UTF8: MSA downconversions

13 files changed, 283 insertions, 98 deletions

diff --git a/src/src/acl.c b/src/src/acl.c
index 3f513c328..c1402a0ff 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -181,17 +181,17 @@ that follows! */
 
 enum {
   CONTROL_AUTH_UNADVERTISED,
-  #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
   CONTROL_BMI_RUN,
-  #endif
+#endif
   CONTROL_DEBUG,
-  #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
   CONTROL_DKIM_VERIFY,
-  #endif
-  #ifdef EXPERIMENTAL_DMARC
+#endif
+#ifdef EXPERIMENTAL_DMARC
   CONTROL_DMARC_VERIFY,
   CONTROL_DMARC_FORENSIC,
-  #endif
+#endif
   CONTROL_DSCP,
   CONTROL_ERROR,
   CONTROL_CASEFUL_LOCAL_PART,
@@ -203,11 +203,14 @@ enum {
   CONTROL_QUEUE_ONLY,
   CONTROL_SUBMISSION,
   CONTROL_SUPPRESS_LOCAL_FIXUPS,
-  #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
   CONTROL_NO_MBOX_UNSPOOL,
-  #endif
+#endif
   CONTROL_FAKEDEFER,
   CONTROL_FAKEREJECT,
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  CONTROL_UTF8_DOWNCONVERT,
+#endif
   CONTROL_NO_MULTILINE,
   CONTROL_NO_PIPELINING,
   CONTROL_NO_DELAY_FLUSH,
@@ -221,17 +224,17 @@ and should be tidied up. */
 
 static uschar *controls[] = {
   US"allow_auth_unadvertised",
-  #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
   US"bmi_run",
-  #endif
+#endif
   US"debug",
-  #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
   US"dkim_disable_verify",
-  #endif
-  #ifdef EXPERIMENTAL_DMARC
+#endif
+#ifdef EXPERIMENTAL_DMARC
   US"dmarc_disable_verify",
   US"dmarc_enable_forensic",
-  #endif
+#endif
   US"dscp",
   US"error",
   US"caseful_local_part",
@@ -243,11 +246,14 @@ static uschar *controls[] = {
   US"queue_only",
   US"submission",
   US"suppress_local_fixups",
-  #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
   US"no_mbox_unspool",
-  #endif
+#endif
   US"fakedefer",
   US"fakereject",
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  US"utf8_downconvert",
+#endif
   US"no_multiline_responses",
   US"no_pipelining",
   US"no_delay_flush",
@@ -600,26 +606,26 @@ static unsigned int control_forbids[] = {
   (unsigned int)
   ~((1<<ACL_WHERE_CONNECT)|(1<<ACL_WHERE_HELO)),   /* allow_auth_unadvertised */
 
-  #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
   0,                                               /* bmi_run */
-  #endif
+#endif
 
   0,                                               /* debug */
 
-  #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
   (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)|      /* dkim_disable_verify */
-  #ifndef DISABLE_PRDR
+# ifndef DISABLE_PRDR
     (1<<ACL_WHERE_PRDR)|
-  #endif
+# endif
     (1<<ACL_WHERE_NOTSMTP_START),
-  #endif
+#endif
 
-  #ifdef EXPERIMENTAL_DMARC
+#ifdef EXPERIMENTAL_DMARC
   (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)|      /* dmarc_disable_verify */
     (1<<ACL_WHERE_NOTSMTP_START),
   (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)|      /* dmarc_enable_forensic */
     (1<<ACL_WHERE_NOTSMTP_START),
-  #endif
+#endif
 
   (1<<ACL_WHERE_NOTSMTP)|
     (1<<ACL_WHERE_NOTSMTP_START)|
@@ -663,30 +669,34 @@ static unsigned int control_forbids[] = {
     (1<<ACL_WHERE_PREDATA)|
     (1<<ACL_WHERE_NOTSMTP_START)),
 
-  #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
   (unsigned int)
   ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)|       /* no_mbox_unspool */
     (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
     // (1<<ACL_WHERE_PRDR)|    /* Not allow one user to freeze for all */
     (1<<ACL_WHERE_MIME)),
-  #endif
+#endif
 
   (unsigned int)
   ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)|       /* fakedefer */
     (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
-  #ifndef DISABLE_PRDR
+#ifndef DISABLE_PRDR
     (1<<ACL_WHERE_PRDR)|
-  #endif
+#endif
     (1<<ACL_WHERE_MIME)),
 
   (unsigned int)
   ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)|       /* fakereject */
     (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
-  #ifndef DISABLE_PRDR
+#ifndef DISABLE_PRDR
     (1<<ACL_WHERE_PRDR)|
-  #endif
+#endif
     (1<<ACL_WHERE_MIME)),
 
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  0,                                               /* utf8_downconvert */
+#endif
+
   (1<<ACL_WHERE_NOTSMTP)|                          /* no_multiline */
     (1<<ACL_WHERE_NOTSMTP_START),
 
@@ -709,37 +719,40 @@ typedef struct control_def {
 } control_def;
 
 static control_def controls_list[] = {
-  { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED, FALSE },
+  { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED,     FALSE },
 #ifdef EXPERIMENTAL_BRIGHTMAIL
-  { US"bmi_run",                 CONTROL_BMI_RUN, FALSE },
+  { US"bmi_run",                 CONTROL_BMI_RUN,               FALSE },
 #endif
-  { US"debug",                   CONTROL_DEBUG, TRUE },
+  { US"debug",                   CONTROL_DEBUG,                 TRUE },
 #ifndef DISABLE_DKIM
-  { US"dkim_disable_verify",     CONTROL_DKIM_VERIFY, FALSE },
+  { US"dkim_disable_verify",     CONTROL_DKIM_VERIFY,           FALSE },
 #endif
 #ifdef EXPERIMENTAL_DMARC
-  { US"dmarc_disable_verify",    CONTROL_DMARC_VERIFY, FALSE },
-  { US"dmarc_enable_forensic",   CONTROL_DMARC_FORENSIC, FALSE },
+  { US"dmarc_disable_verify",    CONTROL_DMARC_VERIFY,          FALSE },
+  { US"dmarc_enable_forensic",   CONTROL_DMARC_FORENSIC,        FALSE },
 #endif
-  { US"dscp",                    CONTROL_DSCP, TRUE },
-  { US"caseful_local_part",      CONTROL_CASEFUL_LOCAL_PART, FALSE },
-  { US"caselower_local_part",    CONTROL_CASELOWER_LOCAL_PART, FALSE },
-  { US"enforce_sync",            CONTROL_ENFORCE_SYNC, FALSE },
-  { US"freeze",                  CONTROL_FREEZE, TRUE },
-  { US"no_callout_flush",        CONTROL_NO_CALLOUT_FLUSH, FALSE },
-  { US"no_delay_flush",          CONTROL_NO_DELAY_FLUSH, FALSE },
-  { US"no_enforce_sync",         CONTROL_NO_ENFORCE_SYNC, FALSE },
-  { US"no_multiline_responses",  CONTROL_NO_MULTILINE, FALSE },
-  { US"no_pipelining",           CONTROL_NO_PIPELINING, FALSE },
-  { US"queue_only",              CONTROL_QUEUE_ONLY, FALSE },
+  { US"dscp",                    CONTROL_DSCP,                  TRUE },
+  { US"caseful_local_part",      CONTROL_CASEFUL_LOCAL_PART,    FALSE },
+  { US"caselower_local_part",    CONTROL_CASELOWER_LOCAL_PART,  FALSE },
+  { US"enforce_sync",            CONTROL_ENFORCE_SYNC,          FALSE },
+  { US"freeze",                  CONTROL_FREEZE,                TRUE },
+  { US"no_callout_flush",        CONTROL_NO_CALLOUT_FLUSH,      FALSE },
+  { US"no_delay_flush",          CONTROL_NO_DELAY_FLUSH,        FALSE },
+  { US"no_enforce_sync",         CONTROL_NO_ENFORCE_SYNC,       FALSE },
+  { US"no_multiline_responses",  CONTROL_NO_MULTILINE,          FALSE },
+  { US"no_pipelining",           CONTROL_NO_PIPELINING,         FALSE },
+  { US"queue_only",              CONTROL_QUEUE_ONLY,            FALSE },
 #ifdef WITH_CONTENT_SCAN
-  { US"no_mbox_unspool",         CONTROL_NO_MBOX_UNSPOOL, FALSE },
+  { US"no_mbox_unspool",         CONTROL_NO_MBOX_UNSPOOL,       FALSE },
 #endif
-  { US"fakedefer",               CONTROL_FAKEDEFER, TRUE },
-  { US"fakereject",              CONTROL_FAKEREJECT, TRUE },
-  { US"submission",              CONTROL_SUBMISSION, TRUE },
+  { US"fakedefer",               CONTROL_FAKEDEFER,             TRUE },
+  { US"fakereject",              CONTROL_FAKEREJECT,            TRUE },
+  { US"submission",              CONTROL_SUBMISSION,            TRUE },
   { US"suppress_local_fixups",   CONTROL_SUPPRESS_LOCAL_FIXUPS, FALSE },
-  { US"cutthrough_delivery",     CONTROL_CUTTHROUGH_DELIVERY, FALSE }
+  { US"cutthrough_delivery",     CONTROL_CUTTHROUGH_DELIVERY,   FALSE },
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  { US"utf8_downconvert",        CONTROL_UTF8_DOWNCONVERT,      TRUE }
+#endif
   };
 
 /* Support data structures for Client SMTP Authorization. acl_verify_csa()
@@ -2080,7 +2093,11 @@ else if (verify_sender_address != NULL)
 
     sender_vaddr = deliver_make_addr(verify_sender_address, TRUE);
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    sender_vaddr->prop.utf8 = message_smtputf8;
+    if ((sender_vaddr->prop.utf8_msg = message_smtputf8))
+      {
+      sender_vaddr->prop.utf8_downcvt =       message_utf8_downconvert == 1;
+      sender_vaddr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+      }
 #endif
     if (no_details) setflag(sender_vaddr, af_sverify_told);
     if (verify_sender_address[0] != 0)
@@ -3377,6 +3394,24 @@ for (; cb != NULL; cb = cb->next)
 					arg, *log_msgptr);
 	  }
 	return ERROR;
+
+    #ifdef EXPERIMENTAL_INTERNATIONAL
+	case CONTROL_UTF8_DOWNCONVERT:
+	if (*p == '/')
+	  {
+	  if (p[1] == '1') { message_utf8_downconvert = 1; p += 2; break; }
+	  if (p[1] == '0') { message_utf8_downconvert = 0; p += 2; break; }
+	  if (p[1] == '-' && p[2] == '1')
+			   { message_utf8_downconvert = -1; p += 3; break; }
+	  *log_msgptr = US"bad option value for control=utf8_downconvert";
+	  }
+	else
+	  {
+	  message_utf8_downconvert = 1; break;
+	  }
+	return ERROR;
+    #endif
+
 	}
       break;
       }
@@ -3390,14 +3425,9 @@ for (; cb != NULL; cb = cb->next)
       /* Run the dcc backend. */
       rc = dcc_process(&ss);
       /* Modify return code based upon the existance of options. */
-      while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size))
-            != NULL) {
+      while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size)))
         if (strcmpic(ss, US"defer_ok") == 0 && rc == DEFER)
-          {
-          /* FAIL so that the message is passed to the next ACL */
-          rc = FAIL;
-          }
-        }
+          rc = FAIL;   /* FAIL so that the message is passed to the next ACL */
       }
     break;
     #endif
@@ -4382,7 +4412,11 @@ if (where == ACL_WHERE_RCPT)
     return DEFER;
     }
 #ifdef EXPERIMENTAL_INTERNATIONAL
-  addr->prop.utf8 = message_smtputf8;
+  if ((addr->prop.utf8_msg = message_smtputf8))
+    {
+    addr->prop.utf8_downcvt =       message_utf8_downconvert == 1;
+    addr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+    }
 #endif
   deliver_domain = addr->domain;
   deliver_localpart = addr->local_part;
diff --git a/src/src/deliver.c b/src/src/deliver.c
index f208f3edd..58b9d3a01 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -822,7 +822,14 @@ if (log_extra_selector & LX_incoming_interface  &&  sending_ip_address)
   /* for the port:  string_sprintf("%d", sending_port) */
 
 if ((log_extra_selector & LX_sender_on_delivery) != 0  ||  msg)
-  s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
+  s = string_append(s, &size, &ptr, 3, US" F=<",
+#ifdef EXPERIMENTAL_INTERNATIONAL
+    testflag(addr, af_utf8_downcvt)
+    ? string_address_utf8_to_alabel(sender_address, NULL)
+    :
+#endif
+      sender_address,
+  US">");
 
 #ifdef EXPERIMENTAL_SRS
 if(addr->prop.srs_sender)
@@ -5595,7 +5602,11 @@ if (process_recipients != RECIP_IGNORE)
       address_item *new = deliver_make_addr(r->address, FALSE);
       new->prop.errors_address = r->errors_to;
 #ifdef EXPERIMENTAL_INTERNATIONAL
-      new->prop.utf8 = message_smtputf8;
+      if ((new->prop.utf8_msg = message_smtputf8))
+	{
+	new->prop.utf8_downcvt =       message_utf8_downconvert == 1;
+	new->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+	}
       DEBUG(D_deliver) if (message_smtputf8) debug_printf("utf8\n");
 #endif
 
diff --git a/src/src/functions.h b/src/src/functions.h
index d1ada3844..1708b7a07 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -422,8 +422,7 @@ extern const uschar *string_printing2(const uschar *, BOOL);
 extern uschar *string_split_message(uschar *);
 extern uschar *string_unprinting(uschar *);
 #ifdef EXPERIMENTAL_INTERNATIONAL
-extern uschar *string_address_alabel_to_utf8(const uschar *, uschar **);
-extern uschar *string_address_utf8_to_alabel(uschar *, uschar **, int *);
+extern uschar *string_address_utf8_to_alabel(const uschar *, uschar **);
 extern uschar *string_domain_alabel_to_utf8(const uschar *, uschar **);
 extern uschar *string_domain_utf8_to_alabel(const uschar *, uschar **);
 extern uschar *string_localpart_alabel_to_utf8(const uschar *, uschar **);
diff --git a/src/src/globals.c b/src/src/globals.c
index 2bf4d0a02..a71c80ed9 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -915,6 +915,7 @@ int     message_size           = 0;
 uschar *message_size_limit     = US"50M";
 #ifdef EXPERIMENTAL_INTERNATIONAL
 BOOL    message_smtputf8       = FALSE;
+int     message_utf8_downconvert = 0;	/* -1 ifneeded; 0 never; 1 always */
 #endif
 uschar  message_subdir[2]      = { 0, 0 };
 uschar *message_reference      = NULL;
diff --git a/src/src/globals.h b/src/src/globals.h
index 7cbf7bfab..8aa69bff6 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -572,6 +572,7 @@ extern int     message_size;           /* Size of message */
 extern uschar *message_size_limit;     /* As it says */
 #ifdef EXPERIMENTAL_INTERNATIONAL
 extern BOOL    message_smtputf8;       /* Internationalized mail handling */
+extern int     message_utf8_downconvert; /* convert from utf8 */
 const extern pcre *regex_UTF8;         /* For recognizing SMTPUTF8 settings */
 #endif
 extern uschar  message_subdir[];       /* Subdirectory for messages */
diff --git a/src/src/routers/redirect.c b/src/src/routers/redirect.c
index 8f1c2c3d2..9e57c8be6 100644
--- a/src/src/routers/redirect.c
+++ b/src/src/routers/redirect.c
@@ -452,7 +452,7 @@ while (generated != NULL)
     }
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    next->prop.utf8 = string_is_utf8(next->address)
+    next->prop.utf8_msg = string_is_utf8(next->address)
       || (sender_address && string_is_utf8(sender_address));
 #endif
 
@@ -476,7 +476,7 @@ while (generated != NULL)
       debug_printf("gid=unset ");
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    if (next->prop.utf8) debug_printf("utf8 ");
+    if (next->prop.utf8_msg) debug_printf("utf8 ");
 #endif
 
     debug_printf("home=%s\n", next->home_dir);
diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 742f4b579..558d955c1 100644
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -301,6 +301,7 @@ spam_score_int = NULL;
 
 #if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
 message_smtputf8 = FALSE;
+message_utf8_downconvert = 0;
 #endif
 
 dsn_ret = 0;
@@ -600,6 +601,15 @@ for (;;)
     break;
 #endif
 
+#if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
+    case 'u':
+    if (Ustrncmp(p, "tf8_downcvt", 11) == 0)
+      message_utf8_downconvert = 1;
+    else if (Ustrncmp(p, "tf8_downcvt_opt", 15) == 0)
+      message_utf8_downconvert = -1;
+    break;
+#endif
+
     default:    /* Present because some compilers complain if all */
     break;      /* possibilities are not covered. */
     }
diff --git a/src/src/spool_out.c b/src/src/spool_out.c
index 6d22bff2c..48f27a8c6 100644
--- a/src/src/spool_out.c
+++ b/src/src/spool_out.c
@@ -246,7 +246,12 @@ if (tls_in.ocsp)	 fprintf(f, "-tls_ocsp %d\n",   tls_in.ocsp);
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-if (message_smtputf8)    fprintf(f, "-smtputf8\n");
+if (message_smtputf8)
+  {
+  fprintf(f, "-smtputf8\n");
+  if (message_utf8_downconvert)
+    fprintf(f, "-utf8_downcvt%s\n", message_utf8_downconvert < 0 ? "_opt" : "");
+  }
 #endif
 
 /* Write the dsn flags to the spool header file */
@@ -508,3 +513,5 @@ return TRUE;
 #endif
 
 /* End of spool_out.c */
+/* vi: aw ai sw=2
+*/
diff --git a/src/src/string.c b/src/src/string.c
index e169a9f05..c50a347c2 100644
--- a/src/src/string.c
+++ b/src/src/string.c
@@ -1547,14 +1547,35 @@ static uschar *
 string_get_localpart(address_item *addr, uschar *yield, int *sizeptr,
   int *ptrptr)
 {
-if (testflag(addr, af_include_affixes) && addr->prefix != NULL)
-  yield = string_cat(yield, sizeptr, ptrptr, addr->prefix,
-    Ustrlen(addr->prefix));
-yield = string_cat(yield, sizeptr, ptrptr, addr->local_part,
-  Ustrlen(addr->local_part));
-if (testflag(addr, af_include_affixes) && addr->suffix != NULL)
-  yield = string_cat(yield, sizeptr, ptrptr, addr->suffix,
-    Ustrlen(addr->suffix));
+uschar * s;
+
+s = addr->prefix;
+if (testflag(addr, af_include_affixes) && s)
+  {
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  if (testflag(addr, af_utf8_downcvt))
+    s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+  yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+  }
+
+s = addr->local_part;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+if (testflag(addr, af_utf8_downcvt))
+  s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+
+s = addr->suffix;
+if (testflag(addr, af_include_affixes) && s)
+  {
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  if (testflag(addr, af_utf8_downcvt))
+    s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+  yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+  }
+
 return yield;
 }
 
@@ -1615,10 +1636,15 @@ else
   {
   if (addr->local_part != NULL)
     {
+    const uschar * s;
     yield = string_get_localpart(addr, yield, &size, &ptr);
     yield = string_cat(yield, &size, &ptr, US"@", 1);
-    yield = string_cat(yield, &size, &ptr, addr->domain,
-      Ustrlen(addr->domain) );
+    s = addr->domain;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+    if (testflag(addr, af_utf8_downcvt))
+      s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+    yield = string_cat(yield, &size, &ptr, s, Ustrlen(s) );
     }
   else
     {
diff --git a/src/src/structs.h b/src/src/structs.h
index 99d65cfae..c181f3f6e 100644
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -460,7 +460,9 @@ typedef struct address_item_propagated {
   uschar *srs_sender;             /* Change return path when delivering */
   #endif
   #ifdef EXPERIMENTAL_INTERNATIONAL
-  BOOL    utf8;			  /* requires SMTPUTF8 processing */
+  BOOL    utf8_msg:1;		  /* requires SMTPUTF8 processing */
+  BOOL	  utf8_downcvt:1;	  /* mandatory downconvert on delivery */
+  BOOL	  utf8_downcvt_maybe:1;	  /* optional downconvert on delivery */
   #endif
 } address_item_propagated;
 
@@ -500,6 +502,9 @@ typedef struct address_item_propagated {
 #ifdef EXPERIMENTAL_DANE
 # define af_dane_verified      0x20000000 /* TLS cert verify done with DANE */
 #endif
+#ifdef EXPERIMENTAL_INTERNATIONAL
+# define af_utf8_downcvt       0x40000000 /* downconvert was done for delivery */
+#endif
 
 /* These flags must be propagated when a child is created */
 
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 65bb1de22..7537e6e4b 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -1366,6 +1366,7 @@ BOOL prdr_offered = FALSE;
 BOOL prdr_active;
 #endif
 #ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_needed = FALSE;
 BOOL utf8_offered = FALSE;
 #endif
 BOOL dsn_all_lasthop = TRUE;
@@ -1642,10 +1643,17 @@ goto SEND_QUIT;
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-  utf8_offered = esmtp
-    && addrlist->prop.utf8
-    && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
-		  PCRE_EOPT, NULL, 0) >= 0;
+  if (addrlist->prop.utf8_msg)
+    {
+    utf8_needed =  !addrlist->prop.utf8_downcvt
+		&& !addrlist->prop.utf8_downcvt_maybe;
+    DEBUG(D_transport) if (!utf8_needed) debug_printf("utf8: %s downconvert\n",
+      addrlist->prop.utf8_downcvt ? "mandatory" : "optional");
+
+    utf8_offered = esmtp
+      && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+		    PCRE_EOPT, NULL, 0) >= 0;
+    }
 #endif
   }
 
@@ -1862,10 +1870,10 @@ if (continue_hostname == NULL
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-  utf8_offered = esmtp
-    && addrlist->prop.utf8
-    && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
-		  PCRE_EOPT, NULL, 0) >= 0;
+  if (addrlist->prop.utf8_msg)
+    utf8_offered = esmtp
+      && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+		    PCRE_EOPT, NULL, 0) >= 0;
 #endif
 
   /* Note if the server supports DSN */
@@ -1896,7 +1904,7 @@ setting_up = FALSE;
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
 /* If this is an international message we need the host to speak SMTPUTF8 */
-if (addrlist->prop.utf8 && !utf8_offered)
+if (utf8_needed && !utf8_offered)
   {
   errno = ERRNO_UTF8_FWD;
   goto RESPONSE_FAILED;
@@ -1980,7 +1988,7 @@ if (prdr_offered)
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-if (addrlist->prop.utf8)
+if (addrlist->prop.utf8_msg && !addrlist->prop.utf8_downcvt && utf8_offered)
   sprintf(CS p, " SMTPUTF8"), p += 9;
 #endif
 
@@ -2037,8 +2045,31 @@ buffer. */
 
 pending_MAIL = TRUE;     /* The block starts with MAIL */
 
-rc = smtp_write_command(&outblock, smtp_use_pipelining,
-	"MAIL FROM:<%s>%s\r\n", return_path, buffer);
+  {
+  uschar * s = return_path;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  uschar * errstr = NULL;
+
+  /* If we must downconvert, do the from-address here.  Remember we had to
+  for the to-addresses (done below), and also (ugly) for re-doing when building
+  the delivery log line. */
+
+  if (addrlist->prop.utf8_msg && (addrlist->prop.utf8_downcvt || !utf8_offered))
+    {
+    if (s = string_address_utf8_to_alabel(return_path, &errstr), errstr)
+      {
+      set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL);
+      yield = ERROR;
+      goto SEND_QUIT;
+      }
+    setflag(addrlist, af_utf8_downcvt);
+    }
+#endif
+
+  rc = smtp_write_command(&outblock, smtp_use_pipelining,
+	  "MAIL FROM:<%s>%s\r\n", s, buffer);
+  }
+
 mail_command = string_copy(big_buffer);  /* Save for later error message */
 
 switch(rc)
@@ -2080,6 +2111,7 @@ for (addr = first_addr;
   {
   int count;
   BOOL no_flush;
+  uschar * rcpt_addr;
 
   addr->dsn_aware = smtp_use_dsn ? dsn_support_yes : dsn_support_no;
 
@@ -2124,8 +2156,24 @@ for (addr = first_addr;
   yield as OK, because this error can often mean that there is a problem with
   just one address, so we don't want to delay the host. */
 
+  rcpt_addr = transport_rcpt_address(addr, tblock->rcpt_include_affixes);
+
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  {
+  uschar * dummy_errstr;
+  if (  testflag(addrlist, af_utf8_downcvt)
+     && (rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, &dummy_errstr),
+	 dummy_errstr
+     )  )
+    {
+    errno = ERRNO_EXPANDFAIL;
+    goto SEND_FAILED;
+    }
+  }
+#endif
+
   count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n",
-    transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr, buffer);
+    rcpt_addr, igquotstr, buffer);
 
   if (count < 0) goto SEND_FAILED;
   if (count > 0)
diff --git a/src/src/utf8.c b/src/src/utf8.c
index 6bc0c2ed5..09ebdf128 100644
--- a/src/src/utf8.c
+++ b/src/src/utf8.c
@@ -24,6 +24,7 @@ return FALSE;
 
 /**************************************************/
 /* Domain conversions */
+/* the *err string pointer should be null before the call */
 
 uschar *
 string_domain_utf8_to_alabel(const uschar * utf8, uschar ** err)
@@ -68,6 +69,7 @@ return s;
 
 /**************************************************/
 /* localpart conversions */
+/* the *err string pointer should be null before the call */
 
 
 uschar *
@@ -126,6 +128,31 @@ return res;
 }
 
 
+/**************************************************/
+/* whole address conversion */
+/* the *err string pointer should be null before the call */
+
+uschar *
+string_address_utf8_to_alabel(const uschar * utf8, uschar ** err)
+{
+const uschar * s;
+uschar * l;
+uschar * d;
+
+for (s = utf8; *s; s++)
+  if (*s == '@')
+    {
+    l = string_copyn(utf8, s - utf8);
+    return   (l = string_localpart_utf8_to_alabel(l, err), err && *err)
+	  || (d = string_domain_utf8_to_alabel(++s, err),  err && *err)
+      ? NULL
+      : string_sprintf("%s@%s", l, d);
+    }
+return string_localpart_utf8_to_alabel(utf8, err);
+}
+
+
+
 /*************************************************
 *         Report the library versions.           *
 *************************************************/
diff --git a/src/src/verify.c b/src/src/verify.c
index 4e9b563fa..28013fa35 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -173,6 +173,9 @@ dbdata_callout_cache new_domain_record;
 dbdata_callout_cache_address new_address_record;
 host_item *host;
 time_t callout_start_time;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_offered = FALSE;
+#endif
 
 new_domain_record.result = ccache_unknown;
 new_domain_record.postmaster_result = ccache_unknown;
@@ -921,22 +924,35 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
       }
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    else if (  addr->prop.utf8
+    else if (  addr->prop.utf8_msg
+	    && !addr->prop.utf8_downcvt
 	    && !(  esmtp
 		&& (  regex_UTF8
 		   || ( (regex_UTF8 = regex_must_compile(
 			  US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE)),
 		      TRUE
 		   )  )
-		&& pcre_exec(regex_UTF8, NULL, CS responsebuffer,
-		    Ustrlen(responsebuffer), 0, PCRE_EOPT, NULL, 0) >= 0
-	    )   )
+		&& (  (utf8_offered = pcre_exec(regex_UTF8, NULL,
+			    CS responsebuffer, Ustrlen(responsebuffer),
+			    0, PCRE_EOPT, NULL, 0) >= 0)
+		   || addr->prop.utf8_downcvt_maybe
+	    )   )  )
       {
       HDEBUG(D_acl|D_v) debug_printf("utf8 required but not offered\n");
       errno = ERRNO_UTF8_FWD;
       setflag(addr, af_verify_nsfail);
       done = FALSE;
       }
+    else if (  addr->prop.utf8_msg
+	    && (addr->prop.utf8_downcvt || !utf8_offered)
+	    && (from_address = string_address_utf8_to_alabel(from_address,
+				      &addr->message), addr->message)
+	    )
+      {
+      errno = ERRNO_EXPANDFAIL;
+      setflag(addr, af_verify_nsfail);
+      done = FALSE;
+      }
 #endif
 
     /* If we haven't authenticated, but are required to, give up. */
@@ -958,7 +974,7 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
     /* Send the MAIL command */
         (smtp_write_command(&outblock, FALSE,
 #ifdef EXPERIMENTAL_INTERNATIONAL
-	  addr->prop.utf8
+	  addr->prop.utf8_msg
 	  ? "MAIL FROM:<%s>%s SMTPUTF8\r\n"
 	  :
 #endif
@@ -1049,7 +1065,7 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
 
             smtp_write_command(&outblock, FALSE,
 #ifdef EXPERIMENTAL_INTERNATIONAL
-	      addr->prop.utf8
+	      addr->prop.utf8_msg
 	      ? "MAIL FROM:<%s> SMTPUTF8\r\n"
 	      :
 #endif