I18N: support IDNA2008. Bug 1911

8 files changed, 264 insertions, 128 deletions

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index d4d091ed2..b783303a1 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -1,6 +1,12 @@
 Change log file for Exim from version 4.21
 -------------------------------------------
 
+Exim version 4.next
+-------------------
+JH/01 Bug 1922: Support IDNA2008.  This has slightly different conversion rules
+      than -2003 did; needs libidn2 in addition to linidn.
+
+
 Exim version 4.88
 -----------------
 JH/01 Use SIZE on MAIL FROM in a cutthrough connection, if the destination
diff --git a/src/src/EDITME b/src/src/EDITME
index 69293467e..1bff9dab2 100644
--- a/src/src/EDITME
+++ b/src/src/EDITME
@@ -937,11 +937,15 @@ ZCAT_COMMAND=/usr/bin/zcat
 #
 # Uncomment the following to include Internationalisation features.  This is the
 # SMTPUTF8 ESMTP extension, and associated facilities for handling UTF8 domain
-# and localparts, per RFCs 5890, 6530 and 6533.
+# and localparts, per RFC 3490 (IDNA2003).
 # You need to have the IDN library installed.
+# If you want IDNA2008 mappings per RFCs 5890, 6530 and 6533, you additionally
+# need libidn2 and SUPPORT_I18N_2008.
 
 # SUPPORT_I18N=yes
 # LDFLAGS += -lidn
+# SUPPORT_I18N_2008=yes
+# LDFLAGS += -lidn -lidn2
 
 
 #------------------------------------------------------------------------------
diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults
index bafdc1ba4..58e181309 100644
--- a/src/src/config.h.defaults
+++ b/src/src/config.h.defaults
@@ -135,6 +135,7 @@ it's a default value. */
 
 #define SUPPORT_CRYPTEQ
 #define SUPPORT_I18N
+#define SUPPORT_I18N_2008
 #define SUPPORT_MAILDIR
 #define SUPPORT_MAILSTORE
 #define SUPPORT_MBX
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index d6ef34eff..d04dfc200 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -2424,17 +2424,14 @@ for (addr = first_addr;
   rcpt_addr = transport_rcpt_address(addr, tblock->rcpt_include_affixes);
 
 #ifdef SUPPORT_I18N
-  {
-  uschar * dummy_errstr;
   if (  testflag(addrlist, af_utf8_downcvt)
-     && (rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, &dummy_errstr),
-	 dummy_errstr
-     )  )
+     && !(rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, NULL))
+     )
     {
+    /*XXX could we use a per-address errstr here? Not fail the whole send? */
     errno = ERRNO_EXPANDFAIL;
     goto SEND_FAILED;
     }
-  }
 #endif
 
   count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n",
diff --git a/src/src/utf8.c b/src/src/utf8.c
index e394db0a8..be5bcb078 100644
--- a/src/src/utf8.c
+++ b/src/src/utf8.c
@@ -2,7 +2,7 @@
 *     Exim - an Internet mail transport agent    *
 *************************************************/
 
-/* Copyright (c) Jeremy Harris 2015 */
+/* Copyright (c) Jeremy Harris 2015, 2016 */
 /* See the file NOTICE for conditions of use and distribution. */
 
 
@@ -10,10 +10,20 @@
 
 #ifdef SUPPORT_I18N
 
-#include <idna.h>
+#ifdef SUPPORT_I18N_2008
+# include <idn2.h>
+#else
+# include <idna.h>
+#endif
+
 #include <punycode.h>
 #include <stringprep.h>
 
+static uschar *
+string_localpart_alabel_to_utf8_(const uschar * alabel, uschar ** err);
+
+/**************************************************/
+
 BOOL
 string_is_utf8(const uschar * s)
 {
@@ -22,17 +32,44 @@ if (s) while ((c = *s++)) if (c & 0x80) return TRUE;
 return FALSE;
 }
 
+static BOOL
+string_is_alabel(const uschar * s)
+{
+return s[0] == 'x' && s[1] == 'n' && s[2] == '-' && s[3] == '-';
+}
+
 /**************************************************/
-/* Domain conversions */
-/* the *err string pointer should be null before the call */
+/* Domain conversions.
+The *err string pointer should be null before the call
+
+Return NULL for error, with optional errstr pointer filled in
+*/
 
 uschar *
 string_domain_utf8_to_alabel(const uschar * utf8, uschar ** err)
 {
-uschar * s1;
-uschar * s;
+uschar * s1, * s;
 int rc;
 
+#ifdef SUPPORT_I18N_2008
+/* Only lowercase is accepted by the library call.  A pity since we lose
+any mixed-case annotation.  This does not really matter for a domain. */
+  {
+  uschar c;
+  for (s1 = s = US utf8; (c = *s1); s1++) if (!(c & 0x80) && isupper(c))
+    {
+    s = string_copy(utf8);
+    for (s1 = s + (s1 - utf8); (c = *s1); s1++) if (!(c & 0x80) && isupper(c))
+      *s1 = tolower(c);
+    break;
+    }
+  }
+if ((rc = idn2_lookup_u8(CCS s, &s1, IDN2_NFC_INPUT)) != IDN2_OK)
+  {
+  if (err) *err = US idn2_strerror(rc);
+  return NULL;
+  }
+#else
 s = US stringprep_utf8_nfkc_normalize(CCS utf8, -1);
 if (  (rc = idna_to_ascii_8z(CCS s, CSS &s1, IDNA_ALLOW_UNASSIGNED))
    != IDNA_SUCCESS)
@@ -42,6 +79,7 @@ if (  (rc = idna_to_ascii_8z(CCS s, CSS &s1, IDNA_ALLOW_UNASSIGNED))
   return NULL;
   }
 free(s);
+#endif
 s = string_copy(s1);
 free(s1);
 return s;
@@ -52,8 +90,23 @@ return s;
 uschar *
 string_domain_alabel_to_utf8(const uschar * alabel, uschar ** err)
 {
-uschar * s1;
-uschar * s;
+#ifdef SUPPORT_I18N_2008
+const uschar * label;
+int sep = '.';
+uschar * s = NULL;
+
+while (label = string_nextinlist(&alabel, &sep, NULL, 0))
+  if (  string_is_alabel(label)
+     && !(label = string_localpart_alabel_to_utf8_(label, err))
+     )
+    return NULL;
+  else
+    s = string_append_listele(s, '.', label);
+return s;
+
+#else
+
+uschar * s1, * s;
 int rc;
 
 if (  (rc = idna_to_unicode_8z8z(CCS alabel, CSS &s1, IDNA_USE_STD3_ASCII_RULES))
@@ -65,6 +118,7 @@ if (  (rc = idna_to_unicode_8z8z(CCS alabel, CSS &s1, IDNA_USE_STD3_ASCII_RULES)
 s = string_copy(s1);
 free(s1);
 return s;
+#endif
 }
 
 /**************************************************/
@@ -103,25 +157,20 @@ return res;
 }
 
 
-uschar *
-string_localpart_alabel_to_utf8(const uschar * alabel, uschar ** err)
+static uschar *
+string_localpart_alabel_to_utf8_(const uschar * alabel, uschar ** err)
 {
-size_t p_len = Ustrlen(alabel);
+size_t p_len;
 punycode_uint * p;
-uschar * s;
-uschar * res;
 int rc;
+uschar * s, * res;
 
-if (alabel[0] != 'x' || alabel[1] != 'n' || alabel[2] != '-' || alabel[3] != '-')
-  {
-  if (err) *err = US"bad alabel prefix";
-  return NULL;
-  }
-
-p_len -= 4;
+DEBUG(D_expand) debug_printf("l_a2u: '%s'\n", alabel);
+alabel += 4;
+p_len = Ustrlen(alabel);
 p = (punycode_uint *) store_get((p_len+1) * sizeof(*p));
 
-if ((rc = punycode_decode(p_len, CCS alabel+4, &p_len, p, NULL)) != PUNYCODE_SUCCESS)
+if ((rc = punycode_decode(p_len, CCS alabel, &p_len, p, NULL)) != PUNYCODE_SUCCESS)
   {
   if (err) *err = US punycode_strerror(rc);
   return NULL;
@@ -134,9 +183,23 @@ return res;
 }
 
 
+uschar *
+string_localpart_alabel_to_utf8(const uschar * alabel, uschar ** err)
+{
+if (string_is_alabel(alabel))
+  return string_localpart_alabel_to_utf8_(alabel, err);
+
+if (err) *err = US"bad alabel prefix";
+return NULL;
+}
+
+
 /**************************************************/
-/* whole address conversion */
-/* the *err string pointer should be null before the call */
+/* Whole address conversion.
+The *err string pointer should be null before the call.
+
+Return NULL on oeeror, with (optional) errstring pointer filled in
+*/
 
 uschar *
 string_address_utf8_to_alabel(const uschar * utf8, uschar ** err)
@@ -153,8 +216,8 @@ for (s = utf8; *s; s++)
   if (*s == '@')
     {
     l = string_copyn(utf8, s - utf8);
-    if (  (l = string_localpart_utf8_to_alabel(l, err), err && *err)
-       || (d = string_domain_utf8_to_alabel(++s, err),  err && *err)
+    if (  !(l = string_localpart_utf8_to_alabel(l, err))
+       || !(d = string_domain_utf8_to_alabel(++s, err))
        )
       return NULL;
     l = string_sprintf("%s@%s", l, d);
@@ -182,10 +245,21 @@ Returns:     nothing
 void
 utf8_version_report(FILE *f)
 {
+#ifdef SUPPORT_I18N_2008
+fprintf(f, "Library version: IDN2: Compile: %s\n"
+           "                       Runtime: %s\n",
+	IDN2_VERSION,
+	idn2_check_version(NULL));
+fprintf(f, "Library version: Stringprep: Compile: %s\n"
+           "                             Runtime: %s\n",
+	STRINGPREP_VERSION,
+	stringprep_check_version(NULL));
+#else
 fprintf(f, "Library version: IDN: Compile: %s\n"
            "                      Runtime: %s\n",
 	STRINGPREP_VERSION,
 	stringprep_check_version(NULL));
+#endif
 }
 
 #endif	/* whole file */
diff --git a/src/src/verify.c b/src/src/verify.c
index 9652a395f..0959b0051 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -940,11 +940,10 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
       }
     else if (  addr->prop.utf8_msg
 	    && (addr->prop.utf8_downcvt || !(peer_offered & PEER_OFFERED_UTF8))
-	    && (setflag(addr, af_utf8_downcvt),
-	        from_address = string_address_utf8_to_alabel(from_address,
-				      &addr->message),
-		addr->message
-	    )  )
+	    && !(setflag(addr, af_utf8_downcvt),
+	         from_address = string_address_utf8_to_alabel(from_address,
+				      &addr->message)
+	    )   )
       {
       errno = ERRNO_EXPANDFAIL;
       setflag(addr, af_verify_nsfail);
@@ -1121,16 +1120,14 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
 
 #ifdef SUPPORT_I18N
 	/*XXX should the conversion be moved into transport_rcpt_address() ? */
-	uschar * dummy_errstr = NULL;
 	if (  testflag(addr, af_utf8_downcvt)
-	   && (rcpt = string_address_utf8_to_alabel(rcpt, &dummy_errstr),
-	       dummy_errstr
-	   )  )
-	{
-	errno = ERRNO_EXPANDFAIL;
-	*failure_ptr = US"recipient";
-	done = FALSE;
-	}
+	   && !(rcpt = string_address_utf8_to_alabel(rcpt, NULL))
+	   )
+	  {
+	  errno = ERRNO_EXPANDFAIL;
+	  *failure_ptr = US"recipient";
+	  done = FALSE;
+	  }
 	else
 #endif
 
diff --git a/test/scripts/4200-International/4200 b/test/scripts/4200-International/4200
index 481a5053f..81fbae865 100644
--- a/test/scripts/4200-International/4200
+++ b/test/scripts/4200-International/4200
@@ -6,95 +6,123 @@ exim -be
 
 utf-8 localpart to a-label:
 
-${utf8_localpart_to_alabel:\xD9\x84}
-xn--ghb
+original:   \xD9\x84
+conversion: ${utf8_localpart_to_alabel:\xD9\x84}
+golden:     xn--ghb
 
-${utf8_localpart_to_alabel:\xD9\x84\xD9\x8A\xD9\x87\xD9\x85\xD8\xA7\xD8\xA8\xD8\xAA\xD9\x83\xD9\x84\
+original:   \xD9\x84\xD9\x8A\xD9\x87\xD9\x85\xD8\xA7\xD8\xA8\xD8\xAA\xD9\x83\xD9\x84\xD9\x85\xD9\x88\xD8\xB4\xD8\xB9\xD8\xB1\xD8\xA8\xD9\x8A\xD8\x9F
+conversion: ${utf8_localpart_to_alabel:\xD9\x84\xD9\x8A\xD9\x87\xD9\x85\xD8\xA7\xD8\xA8\xD8\xAA\xD9\x83\xD9\x84\
 \xD9\x85\xD9\x88\xD8\xB4\xD8\xB9\xD8\xB1\xD8\xA8\xD9\x8A\xD8\x9F}
-xn--egbpdaj6bu4bxfgehfvwxn
+golden:     xn--egbpdaj6bu4bxfgehfvwxn
 
 a-label localpart to utf-8:
 
-${utf8_localpart_from_alabel:xn--ghb}
-${utf8_localpart_from_alabel:xn--egbpdaj6bu4bxfgehfvwxn}
+original:   xn--ghb
+conversion: ${utf8_localpart_from_alabel:xn--ghb}
+golden:     \xD9\x84
+
+original:   xn--egbpdaj6bu4bxfgehfvwxn
+conversion: ${utf8_localpart_from_alabel:xn--egbpdaj6bu4bxfgehfvwxn}
+golden:     \xD9\x84\xD9\x8A\xD9\x87\xD9\x85\xD8\xA7\xD8\xA8\xD8\xAA\xD9\x83\xD9\x84\xD9\x85\xD9\x88\xD8\xB4\xD8\xB9\xD8\xB1\xD8\xA8\xD9\x8A\xD8\x9F
+
+============
 
 utf-8 domain to a-label:
 
-${utf8_domain_to_alabel:bogus.\xD9\x84.com}
-bogus.xn--ghb.com
+original:   bogus.\xD9\x84.com
+conversion: ${utf8_domain_to_alabel:bogus.\xD9\x84.com}
+golden:     bogus.xn--ghb.com
 
-${utf8_domain_to_alabel:arabic.\xD9\x84\xD9\x8A\xD9\x87\xD9\x85\xD8\xA7\xD8\xA8\xD8\xAA\xD9\x83\xD9\x84\
-\xD9\x85\xD9\x88\xD8\xB4\xD8\xB9\xD8\xB1\xD8\xA8\xD9\x8A\xD8\x9F.com}
-arabic.xn--egbpdaj6bu4bxfgehfvwxn.com
+original:   arabic.\xD9\x84\xD9\x8A\xD9\x87\xD9\x85\xD8\xA7\xD8\xA8\xD8\xAA\xD9\x83\xD9\x84\xD9\x85\xD9\x88\xD8\xB4\xD8\xB9\xD8\xB1\xD8\xA8\xD9\x8A.com
+conversion: ${utf8_domain_to_alabel:arabic.\xD9\x84\xD9\x8A\xD9\x87\xD9\x85\xD8\xA7\xD8\xA8\xD8\xAA\xD9\x83\xD9\x84\
+\xD9\x85\xD9\x88\xD8\xB4\xD8\xB9\xD8\xB1\xD8\xA8\xD9\x8A.com}
+golden:     arabic.xn--mgbcah9ar9a4efegftvvn.com
 
-${utf8_domain_to_alabel:simpl.chinese.\xE4\xBB\x96\xE4\xBB\xAC\xE4\xB8\xBA\xE4\xBB\x80\
+original    simpl.chinese.\xE4\xBB\x96\xE4\xBB\xAC\xE4\xB8\xBA\xE4\xBB\x80\xE4\xB9\x88\xE4\xB8\x8D\xE8\xAF\xB4\xE4\xB8\xAD\xE6\x96\x87.com
+conversion: ${utf8_domain_to_alabel:simpl.chinese.\xE4\xBB\x96\xE4\xBB\xAC\xE4\xB8\xBA\xE4\xBB\x80\
 \xE4\xB9\x88\xE4\xB8\x8D\xE8\xAF\xB4\xE4\xB8\xAD\xE6\x96\x87.com}
-simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com
+golden:     simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com
 
-${utf8_domain_to_alabel:trad.chinese.\xE4\xBB\x96\xE5\x80\x91\xE7\x88\xB2\xE4\xBB\x80\
+original    trad.chinese.\xE4\xBB\x96\xE5\x80\x91\xE7\x88\xB2\xE4\xBB\x80\xE9\xBA\xBD\xE4\xB8\x8D\xE8\xAA\xAA\xE4\xB8\xAD\xE6\x96\x87.com
+conversion: ${utf8_domain_to_alabel:trad.chinese.\xE4\xBB\x96\xE5\x80\x91\xE7\x88\xB2\xE4\xBB\x80\
 \xE9\xBA\xBD\xE4\xB8\x8D\xE8\xAA\xAA\xE4\xB8\xAD\xE6\x96\x87.com}
-trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com
+golden:     trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com
 
-${utf8_domain_to_alabel:czech.\x50\x72\x6F\xC4\x8D\x70\x72\x6F\x73\x74\xC4\x9B\x6E\x65\
-\x6D\x6C\x75\x76\xC3\xAD\xC4\x8D\x65\x73\x6B\x79.com}
-czech.xn--Proprostnemluvesky-uyb24dma41a.com
+original    czech.\x50\x72\x6F\xC4\x8D\x70\x72\x6F\x73\x74\xC4\x9B\x6E\x65\x6D\x6C\x75\x76\xC3\xAD\xC4\x8D\x65\x73\x6B\x79.com
+conversion: ${utf8_domain_to_alabel:czech.Pro\xC4\x8Dprost\xC4\x9Bnemluv\xC3\xAD\xC4\x8Desky.com}
+golden:     czech.xn--Proprostnemluvesky-uyb24dma41a.com
 
-${utf8_domain_to_alabel:hebrew.\xD7\x9C\xD7\x9E\xD7\x94\xD7\x94\xD7\x9D\xD7\xA4\xD7\xA9\
+original    hebrew.\xD7\x9C\xD7\x9E\xD7\x94\xD7\x94\xD7\x9D\xD7\xA4\xD7\xA9\xD7\x95\xD7\x98\xD7\x9C\xD7\x90\xD7\x9E\xD7\x93\xD7\x91\xD7\xA8\xD7\x99\xD7\x9D\xD7\xA2\xD7\x91\xD7\xA8\xD7\x99\xD7\xAA.com
+conversion: ${utf8_domain_to_alabel:hebrew.\xD7\x9C\xD7\x9E\xD7\x94\xD7\x94\xD7\x9D\xD7\xA4\xD7\xA9\
 \xD7\x95\xD7\x98\xD7\x9C\xD7\x90\xD7\x9E\xD7\x93\xD7\x91\xD7\xA8\xD7\x99\xD7\x9D\xD7\xA2\
 \xD7\x91\xD7\xA8\xD7\x99\xD7\xAA.com}
-hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com
+golden:     hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com
 
-${utf8_domain_to_alabel:hindi.\xE0\xA4\xAF\xE0\xA4\xB9\xE0\xA4\xB2\xE0\xA5\x8B\xE0\xA4\x97\
+original    hindi.\xE0\xA4\xAF\xE0\xA4\xB9\xE0\xA4\xB2\xE0\xA5\x8B\xE0\xA4\x97\xE0\xA4\xB9\xE0\xA4\xBF\xE0\xA4\xA8\xE0\xA5\x8D\xE0\xA4\xA6\xE0\xA5\x80\xE0\xA4\x95\xE0\xA5\x8D\xE0\xA4\xAF\xE0\xA5\x8B\xE0\xA4\x82\xE0\xA4\xA8\xE0\xA4\xB9\xE0\xA5\x80\xE0\xA4\x82\xE0\xA4\xAC\xE0\xA5\x8B\xE0\xA4\xB2\xE0\xA4\xB8\xE0\xA4\x95\xE0\xA4\xA4\xE0\xA5\x87\xE0\xA4\xB9\xE0\xA5\x88\xE0\xA4\x82.com
+conversion: ${utf8_domain_to_alabel:hindi.\xE0\xA4\xAF\xE0\xA4\xB9\xE0\xA4\xB2\xE0\xA5\x8B\xE0\xA4\x97\
 \xE0\xA4\xB9\xE0\xA4\xBF\xE0\xA4\xA8\xE0\xA5\x8D\xE0\xA4\xA6\xE0\xA5\x80\xE0\xA4\x95\xE0\xA5\x8D\
 \xE0\xA4\xAF\xE0\xA5\x8B\xE0\xA4\x82\xE0\xA4\xA8\xE0\xA4\xB9\xE0\xA5\x80\xE0\xA4\x82\xE0\xA4\xAC\
 \xE0\xA5\x8B\xE0\xA4\xB2\xE0\xA4\xB8\xE0\xA4\x95\xE0\xA4\xA4\xE0\xA5\x87\xE0\xA4\xB9\xE0\xA5\x88\
 \xE0\xA4\x82.com}
-hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com
+golden:     hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com
 
-${utf8_domain_to_alabel:japanese.\xE3\x81\xAA\xE3\x81\x9C\xE3\x81\xBF\xE3\x82\x93\xE3\x81\xAA\
+original    japanese.\xE3\x81\xAA\xE3\x81\x9C\xE3\x81\xBF\xE3\x82\x93\xE3\x81\xAA\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E\xE3\x82\x92\xE8\xA9\xB1\xE3\x81\x97\xE3\x81\xA6\xE3\x81\x8F\xE3\x82\x8C\xE3\x81\xAA\xE3\x81\x84\xE3\x81\xAE\xE3\x81\x8B.com
+conversion: ${utf8_domain_to_alabel:japanese.\xE3\x81\xAA\xE3\x81\x9C\xE3\x81\xBF\xE3\x82\x93\xE3\x81\xAA\
 \xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E\xE3\x82\x92\xE8\xA9\xB1\xE3\x81\x97\xE3\x81\xA6\xE3\x81\x8F\
 \xE3\x82\x8C\xE3\x81\xAA\xE3\x81\x84\xE3\x81\xAE\xE3\x81\x8B.com}
-japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com
+golden:     japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com
 
 # the a-label for the phrase in korean is too long for a domain label (63 byte limit)
+# so this is a localpart conversion
+original    \xEC\x84\xB8\xEA\xB3\x84\xEC\x9D\x98\xEB\xAA\xA8\xEB\x93\xA0\xEC\x82\xAC\xEB\x9E\x8C\xEB\x93\xA4\xEC\x9D\xB4\xED\x95\x9C\xEA\xB5\xAD\xEC\x96\xB4\xEB\xA5\xBC\xEC\x9D\xB4\xED\x95\xB4\xED\x95\x9C\xEB\x8B\xA4\xEB\xA9\xB4\xEC\x96\xBC\xEB\xA7\x88\xEB\x82\x98\xEC\xA2\x8B\xEC\x9D\x84\xEA\xB9\x8C
 korean: ${utf8_localpart_to_alabel:\xEC\x84\xB8\xEA\xB3\x84\xEC\x9D\x98\xEB\xAA\xA8\xEB\x93\xA0\
 \xEC\x82\xAC\xEB\x9E\x8C\xEB\x93\xA4\xEC\x9D\xB4\xED\x95\x9C\xEA\xB5\xAD\xEC\x96\xB4\xEB\xA5\xBC\
 \xEC\x9D\xB4\xED\x95\xB4\xED\x95\x9C\xEB\x8B\xA4\xEB\xA9\xB4\xEC\x96\xBC\xEB\xA7\x88\xEB\x82\x98\
 \xEC\xA2\x8B\xEC\x9D\x84\xEA\xB9\x8C}
-korean: xn--989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c
+golden:     korean: xn--989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c
 
-${utf8_domain_to_alabel:russian.\xD0\xBF\xD0\xBE\xD1\x87\xD0\xB5\xD0\xBC\xD1\x83\xD0\xB6\
+original    russian.\xD0\xBF\xD0\xBE\xD1\x87\xD0\xB5\xD0\xBC\xD1\x83\xD0\xB6\xD0\xB5\xD0\xBE\xD0\xBD\xD0\xB8\xD0\xBD\xD0\xB5\xD0\xB3\xD0\xBE\xD0\xB2\xD0\xBE\xD1\x80\xD1\x8F\xD1\x82\xD0\xBF\xD0\xBE\xD1\x80\xD1\x83\xD1\x81\xD1\x81\xD0\xBA\xD0\xB8.com
+conversion: ${utf8_domain_to_alabel:russian.\xD0\xBF\xD0\xBE\xD1\x87\xD0\xB5\xD0\xBC\xD1\x83\xD0\xB6\
 \xD0\xB5\xD0\xBE\xD0\xBD\xD0\xB8\xD0\xBD\xD0\xB5\xD0\xB3\xD0\xBE\xD0\xB2\xD0\xBE\xD1\x80\
 \xD1\x8F\xD1\x82\xD0\xBF\xD0\xBE\xD1\x80\xD1\x83\xD1\x81\xD1\x81\xD0\xBA\xD0\xB8.com}
-russian.xn--b1abfaaepdrnnbgefbaDotcwatmq2g4l.com
+golden:     russian.xn--b1abfaaepdrnnbgefbaDotcwatmq2g4l.com
 
-${utf8_domain_to_alabel:spanish.\x50\x6F\x72\x71\x75\xC3\xA9\x6E\x6F\x70\x75\x65\x64\x65\
+original    spanish.\x50\x6F\x72\x71\x75\xC3\xA9\x6E\x6F\x70\x75\x65\x64\x65\x6E\x73\x69\x6D\x70\x6C\x65\x6D\x65\x6E\x74\x65\x68\x61\x62\x6C\x61\x72\x65\x6E\x45\x73\x70\x61\xC3\xB1\x6F\x6C.com
+conversion: ${utf8_domain_to_alabel:spanish.\x50\x6F\x72\x71\x75\xC3\xA9\x6E\x6F\x70\x75\x65\x64\x65\
 \x6E\x73\x69\x6D\x70\x6C\x65\x6D\x65\x6E\x74\x65\x68\x61\x62\x6C\x61\x72\x65\x6E\x45\x73\
 \x70\x61\xC3\xB1\x6F\x6C.com}
-spanish.xn--PorqunopuedensimplementehablarenEspaol-fmd56a.com
+golden:     spanish.xn--PorqunopuedensimplementehablarenEspaol-fmd56a.com
 
-${utf8_domain_to_alabel:vietnamese.\x54\xE1\xBA\xA1\x69\x73\x61\x6F\x68\xE1\xBB\x8D\x6B\x68\
+original    vietnamese.\x54\xE1\xBA\xA1\x69\x73\x61\x6F\x68\xE1\xBB\x8D\x6B\x68\xC3\xB4\x6E\x67\x74\x68\xE1\xBB\x83\x63\x68\xE1\xBB\x89\x6E\xC3\xB3\x69\x74\x69\xE1\xBA\xBF\x6E\x67\x56\x69\xE1\xBB\x87\x74.com
+conversion: ${utf8_domain_to_alabel:vietnamese.\x54\xE1\xBA\xA1\x69\x73\x61\x6F\x68\xE1\xBB\x8D\x6B\x68\
 \xC3\xB4\x6E\x67\x74\x68\xE1\xBB\x83\x63\x68\xE1\xBB\x89\x6E\xC3\xB3\x69\x74\x69\xE1\xBA\xBF\
 \x6E\x67\x56\x69\xE1\xBB\x87\x74.com}
-vietnamese.xn--TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g.com
+golden:     vietnamese.xn--TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g.com
+
+original    straße.de
+conversion  ${utf8_domain_to_alabel:straße.de}
+golden:     xn--strae-oqa.de
 
 a-label domain to utf-8:
 
-${utf8_domain_from_alabel:arab.xn--ghb.com}
-${utf8_domain_from_alabel:arab.xn--egbpdaj6bu4bxfgehfvwxn.com}
-${utf8_domain_from_alabel:simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com}
-${utf8_domain_from_alabel:trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com}
-${utf8_domain_from_alabel:czech.xn--Proprostnemluvesky-uyb24dma41a.com}
-${utf8_domain_from_alabel:hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com}
-${utf8_domain_from_alabel:hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com}
-${utf8_domain_from_alabel:japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com}
+conversion: ${utf8_domain_from_alabel:arab.xn--ghb.com}
+conversion: ${utf8_domain_from_alabel:arab.xn--mgbcah9ar9a4efegftvvn.com}
+conversion: ${utf8_domain_from_alabel:simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com}
+conversion: ${utf8_domain_from_alabel:trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com}
+conversion: ${utf8_domain_from_alabel:czech.xn--proprostnemluvesky-uyb24dma41a.com}
+conversion: ${utf8_domain_from_alabel:hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com}
+conversion: ${utf8_domain_from_alabel:hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com}
+conversion: ${utf8_domain_from_alabel:japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com}
 korean: ${utf8_localpart_from_alabel:xn--989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c}
-${utf8_domain_from_alabel:russian.xn--b1abfaaepdrnnbgefbaDotcwatmq2g4l.com}
-${utf8_domain_from_alabel:spanish.xn--PorqunopuedensimplementehablarenEspaol-fmd56a.com}
-${utf8_domain_from_alabel:vietnamese.xn--TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g.com}
+conversion: ${utf8_domain_from_alabel:russian.xn--b1abfaaepdrnnbgefbaDotcwatmq2g4l.com}
+conversion: ${utf8_domain_from_alabel:spanish.xn--PorqunopuedensimplementehablarenEspaol-fmd56a.com}
+conversion: ${utf8_domain_from_alabel:vietnamese.xn--TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g.com}
+conversion: ${utf8_domain_from_alabel:german.xn--strae-oqa.de}
 
 ===========
 
+imapfolder conversions:
+
 ${imapfolder {Foo/Bar}}
 Foo.Bar
 
diff --git a/test/stdout/4200 b/test/stdout/4200
index 9e2c4bbc5..af39676df 100644
--- a/test/stdout/4200
+++ b/test/stdout/4200
@@ -1,73 +1,102 @@
 > 
 > utf-8 localpart to a-label:
 > 
-> xn--ghb
-> xn--ghb
+> original:   ل
+> conversion: xn--ghb
+> golden:     xn--ghb
 > 
-> xn--egbpdaj6bu4bxfgehfvwxn
-> xn--egbpdaj6bu4bxfgehfvwxn
+> original:   ليهمابتكلموشعربي؟
+> conversion: xn--egbpdaj6bu4bxfgehfvwxn
+> golden:     xn--egbpdaj6bu4bxfgehfvwxn
 > 
 > a-label localpart to utf-8:
 > 
-> ل
-> ليهمابتكلموشعربي؟
+> original:   xn--ghb
+> conversion: ل
+> golden:     ل
+> 
+> original:   xn--egbpdaj6bu4bxfgehfvwxn
+> conversion: ليهمابتكلموشعربي؟
+> golden:     ليهمابتكلموشعربي؟
+> 
+> ============
 > 
 > utf-8 domain to a-label:
 > 
-> bogus.xn--ghb.com
-> bogus.xn--ghb.com
+> original:   bogus.ل.com
+> conversion: bogus.xn--ghb.com
+> golden:     bogus.xn--ghb.com
 > 
-> arabic.xn--egbpdaj6bu4bxfgehfvwxn.com
-> arabic.xn--egbpdaj6bu4bxfgehfvwxn.com
+> original:   arabic.ليهمابتكلموشعربي.com
+> conversion: arabic.xn--mgbcah9ar9a4efegftvvn.com
+> golden:     arabic.xn--mgbcah9ar9a4efegftvvn.com
 > 
-> simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com
-> simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com
+> original    simpl.chinese.他们为什么不说中文.com
+> conversion: simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com
+> golden:     simpl.chinese.xn--ihqwcrb4cv8a8dqg056pqjye.com
 > 
-> trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com
-> trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com
+> original    trad.chinese.他們爲什麽不說中文.com
+> conversion: trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com
+> golden:     trad.chinese.xn--ihqwctvzc91f659drss3x8bo0yb.com
 > 
-> czech.xn--proprostnemluvesky-uyb24dma41a.com
-> czech.xn--Proprostnemluvesky-uyb24dma41a.com
+> original    czech.Pročprostěnemluvíčesky.com
+> conversion: czech.xn--proprostnemluvesky-uyb24dma41a.com
+> golden:     czech.xn--Proprostnemluvesky-uyb24dma41a.com
 > 
-> hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com
-> hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com
+> original    hebrew.למההםפשוטלאמדבריםעברית.com
+> conversion: hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com
+> golden:     hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com
 > 
-> hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com
-> hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com
+> original    hindi.यहलोगहिन्दीक्योंनहींबोलसकतेहैं.com
+> conversion: hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com
+> golden:     hindi.xn--i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd.com
 > 
-> japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com
-> japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com
+> original    japanese.なぜみんな日本語を話してくれないのか.com
+> conversion: japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com
+> golden:     japanese.xn--n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa.com
 > 
 > # the a-label for the phrase in korean is too long for a domain label (63 byte limit)
+> # so this is a localpart conversion
+> original    세계의모든사람들이한국어를이해한다면얼마나좋을까
 > korean: xn--989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c
-> korean: xn--989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c
+> golden:     korean: xn--989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c
 > 
-> russian.xn--b1abfaaepdrnnbgefbadotcwatmq2g4l.com
-> russian.xn--b1abfaaepdrnnbgefbaDotcwatmq2g4l.com
+> original    russian.почемужеонинеговорятпорусски.com
+> conversion: russian.xn--b1abfaaepdrnnbgefbadotcwatmq2g4l.com
+> golden:     russian.xn--b1abfaaepdrnnbgefbaDotcwatmq2g4l.com
 > 
-> spanish.xn--porqunopuedensimplementehablarenespaol-fmd56a.com
-> spanish.xn--PorqunopuedensimplementehablarenEspaol-fmd56a.com
+> original    spanish.PorquénopuedensimplementehablarenEspañol.com
+> conversion: spanish.xn--porqunopuedensimplementehablarenespaol-fmd56a.com
+> golden:     spanish.xn--PorqunopuedensimplementehablarenEspaol-fmd56a.com
 > 
-> vietnamese.xn--tisaohkhngthchnitingvit-kjcr8268qyxafd2f1b9g.com
-> vietnamese.xn--TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g.com
+> original    vietnamese.TạisaohọkhôngthểchỉnóitiếngViệt.com
+> conversion: vietnamese.xn--tisaohkhngthchnitingvit-kjcr8268qyxafd2f1b9g.com
+> golden:     vietnamese.xn--TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g.com
+> 
+> original    straße.de
+> conversion  xn--strae-oqa.de
+> golden:     xn--strae-oqa.de
 > 
 > a-label domain to utf-8:
 > 
-> arab.ل.com
-> arab.ليهمابتكلموشعربي؟.com
-> simpl.chinese.他们为什么不说中文.com
-> trad.chinese.他們爲什麽不說中文.com
-> czech.Pročprostěnemluvíčesky.com
-> hebrew.למההםפשוטלאמדבריםעברית.com
-> hindi.यहलोगहिन्दीक्योंनहींबोलसकतेहैं.com
-> japanese.なぜみんな日本語を話してくれないのか.com
+> conversion: arab.ل.com
+> conversion: arab.ليهمابتكلموشعربي.com
+> conversion: simpl.chinese.他们为什么不说中文.com
+> conversion: trad.chinese.他們爲什麽不說中文.com
+> conversion: czech.pročprostěnemluvíčesky.com
+> conversion: hebrew.למההםפשוטלאמדבריםעברית.com
+> conversion: hindi.यहलोगहिन्दीक्योंनहींबोलसकतेहैं.com
+> conversion: japanese.なぜみんな日本語を話してくれないのか.com
 > korean: 세계의모든사람들이한국어를이해한다면얼마나좋을까
-> russian.почемужеонинеговорятпорусски.com
-> spanish.PorquénopuedensimplementehablarenEspañol.com
-> vietnamese.TạisaohọkhôngthểchỉnóitiếngViệt.com
+> conversion: russian.почемужеонинеговорятпорусски.com
+> conversion: spanish.PorquénopuedensimplementehablarenEspañol.com
+> conversion: vietnamese.TạisaohọkhôngthểchỉnóitiếngViệt.com
+> conversion: german.straße.de
 > 
 > ===========
 > 
+> imapfolder conversions:
+> 
 > Foo.Bar
 > Foo.Bar
 >