smtp output

16 files changed, 150 insertions, 32 deletions

diff --git a/TODO b/TODO
index cca33151f..a282dc592 100644
--- a/TODO
+++ b/TODO
@@ -26,7 +26,13 @@ to-Alabel convert of helo name
 --- mua-wrapper
 --- acl control?
 
+++ flag in spool file
+
+retries
+- apply to a-label or utf8 form?
+
 dsn handling				rfc6533
+
 logging
 ++ - international msg
 - presentation of local-part in log
@@ -42,8 +48,6 @@ forwarding checks			rfc6530 7.1 -3-
 - mail-time rejects get 550 mailbox unavailable
 - bounces (see dsn handling)
 
-flag in spool file
-
 ++ expansions for to- and from-Alabel ?	bug1567
 
 enhanced status codes?			rfc5248++
diff --git a/src/src/deliver.c b/src/src/deliver.c
index 1cdecc6e9..cc43c921a 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -5594,6 +5594,10 @@ if (process_recipients != RECIP_IGNORE)
       recipient_item *r = recipients_list + i;
       address_item *new = deliver_make_addr(r->address, FALSE);
       new->p.errors_address = r->errors_to;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+      new->p.utf8 = message_smtputf8;
+      DEBUG(D_deliver) debug_printf("utf8: %c\n", message_smtputf8 ? 'T':'F');
+#endif
 
       if (r->pno >= 0)
         new->onetime_parent = recipients_list[r->pno].address;
@@ -7857,6 +7861,11 @@ if (!regex_PRDR) regex_PRDR =
   regex_must_compile(US"\\n250[\\s\\-]PRDR(\\s|\\n|$)", FALSE, TRUE);
 #endif
 
+#ifdef SUPPORT_TLS
+if (!regex_UTF8) regex_UTF8 =
+  regex_must_compile(US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE);
+#endif
+
 if (!regex_DSN) regex_DSN  =
   regex_must_compile(US"\\n250[\\s\\-]DSN(\\s|\\n|$)", FALSE, TRUE);
 
diff --git a/src/src/functions.h b/src/src/functions.h
index fdd629228..d1ada3844 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -422,6 +422,7 @@ extern const uschar *string_printing2(const uschar *, BOOL);
 extern uschar *string_split_message(uschar *);
 extern uschar *string_unprinting(uschar *);
 #ifdef EXPERIMENTAL_INTERNATIONAL
+extern uschar *string_address_alabel_to_utf8(const uschar *, uschar **);
 extern uschar *string_address_utf8_to_alabel(uschar *, uschar **, int *);
 extern uschar *string_domain_alabel_to_utf8(const uschar *, uschar **);
 extern uschar *string_domain_utf8_to_alabel(const uschar *, uschar **);
diff --git a/src/src/globals.c b/src/src/globals.c
index 2cbafcdce..2bf4d0a02 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -175,6 +175,10 @@ BOOL    prdr_requested         = FALSE;
 const pcre *regex_PRDR         = NULL;
 #endif
 
+#ifdef EXPERIMENTAL_INTERNATIONAL
+const pcre *regex_UTF8         = NULL;
+#endif
+
 /* Input-reading functions for messages, so we can use special ones for
 incoming TCP/IP. The defaults use stdin. We never need these for any
 stand-alone tests. */
@@ -384,6 +388,9 @@ address_item address_defaults = {
 #ifdef EXPERIMENTAL_SRS
     NULL,               /* srs_sender */
 #endif
+#ifdef EXPERIMENTAL_INTERNATIONAL
+    FALSE,		/* utf8 */
+#endif
   }
 };
 
diff --git a/src/src/globals.h b/src/src/globals.h
index d5b34201b..7cbf7bfab 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -572,6 +572,7 @@ extern int     message_size;           /* Size of message */
 extern uschar *message_size_limit;     /* As it says */
 #ifdef EXPERIMENTAL_INTERNATIONAL
 extern BOOL    message_smtputf8;       /* Internationalized mail handling */
+const extern pcre *regex_UTF8;         /* For recognizing SMTPUTF8 settings */
 #endif
 extern uschar  message_subdir[];       /* Subdirectory for messages */
 extern uschar *message_reference;      /* Reference for error messages */
diff --git a/src/src/macros.h b/src/src/macros.h
index 0f893e812..a8ab4f7ae 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -503,7 +503,11 @@ to conflict with system errno values. */
 #define ERRNO_MAIL4XX        (-45)   /* MAIL gave 4xx error */
 #define ERRNO_DATA4XX        (-46)   /* DATA gave 4xx error */
 #define ERRNO_PROXYFAIL      (-47)   /* Negotiation failed for proxy configured host */
-#define ERRNO_AUTHPROB       (-48)   /* Autheticator "other" failure */
+#define ERRNO_AUTHPROB       (-48)   /* Authenticator "other" failure */
+
+#ifdef EXPERIMENTAL_INTERNATIONAL
+# define ERRNO_UTF8_FWD      (-49)   /* target not supporting SMTPUTF8 */
+#endif
 
 /* These must be last, so all retry deferments can easily be identified */
 
diff --git a/src/src/parse.c b/src/src/parse.c
index a648f755a..9e57365be 100644
--- a/src/src/parse.c
+++ b/src/src/parse.c
@@ -740,7 +740,7 @@ if (*s == '<')
   while (bracket_count-- > 0) if (*s++ != '>')
     {
     *errorptr = (s[-1] == 0)? US"'>' missing at end of address" :
-      string_sprintf("malformed address A: %.32s may not follow %.*s",
+      string_sprintf("malformed address: %.32s may not follow %.*s",
         s-1, s - (uschar *)mailbox - 1, mailbox);
     goto PARSE_FAILED;
     }
@@ -793,7 +793,7 @@ if (*s != 0)
     }
   else
     {
-    *errorptr = string_sprintf("malformed address B: %.32s may not follow %.*s",
+    *errorptr = string_sprintf("malformed address: %.32s may not follow %.*s",
       s, s - (uschar *)mailbox, mailbox);
     goto PARSE_FAILED;
     }
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index eb22233ca..a9c7fb25c 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -4409,16 +4409,12 @@ while (done <= 0)
       receive_add_recipient(recipient, -1);
  
       /* Set the dsn flags in the recipients_list */
-      if (orcpt != NULL)
-        recipients_list[recipients_count-1].orcpt = orcpt;
-      else
-        recipients_list[recipients_count-1].orcpt = NULL;
+      recipients_list[recipients_count-1].orcpt = orcpt;
+      recipients_list[recipients_count-1].dsn_flags = flags;
 
-      if (flags != 0)
-        recipients_list[recipients_count-1].dsn_flags = flags;
-      else
-        recipients_list[recipients_count-1].dsn_flags = 0;
-      DEBUG(D_receive) debug_printf("DSN: orcpt: %s  flags: %d\n", recipients_list[recipients_count-1].orcpt, recipients_list[recipients_count-1].dsn_flags);
+      DEBUG(D_receive) debug_printf("DSN: orcpt: %s  flags: %d\n",
+	recipients_list[recipients_count-1].orcpt,
+	recipients_list[recipients_count-1].dsn_flags);
       }
 
     /* The recipient was discarded */
diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 79970cb40..742f4b579 100644
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -299,6 +299,10 @@ tls_in.ocsp = OCSP_NOT_REQ;
 spam_score_int = NULL;
 #endif
 
+#if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
+message_smtputf8 = FALSE;
+#endif
+
 dsn_ret = 0;
 dsn_envid = NULL;
 
@@ -569,6 +573,10 @@ for (;;)
     else if (Ustrncmp(p, "pam_score_int ", 14) == 0)
       spam_score_int = string_copy(big_buffer + 16);
 #endif
+#if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
+    else if (Ustrncmp(p, "mtputf8", 7) == 0)
+      message_smtputf8 = TRUE;
+#endif
     break;
 
 #ifdef SUPPORT_TLS
diff --git a/src/src/spool_out.c b/src/src/spool_out.c
index fc56057c1..6d22bff2c 100644
--- a/src/src/spool_out.c
+++ b/src/src/spool_out.c
@@ -245,6 +245,10 @@ if (tls_in.ourcert)
 if (tls_in.ocsp)	 fprintf(f, "-tls_ocsp %d\n",   tls_in.ocsp);
 #endif
 
+#ifdef EXPERIMENTAL_INTERNATIONAL
+if (message_smtputf8)    fprintf(f, "-smtputf8\n");
+#endif
+
 /* Write the dsn flags to the spool header file */
 DEBUG(D_deliver) debug_printf("DSN: Write SPOOL :-dsn_envid %s\n", dsn_envid);
 if (dsn_envid != NULL) fprintf(f, "-dsn_envid %s\n", dsn_envid);
diff --git a/src/src/structs.h b/src/src/structs.h
index 6ec52e1ec..df19cfad4 100644
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -459,6 +459,9 @@ typedef struct address_item_propagated {
   #ifdef EXPERIMENTAL_SRS
   uschar *srs_sender;             /* Change return path when delivering */
   #endif
+  #ifdef EXPERIMENTAL_INTERNATIONAL
+  BOOL    utf8;			  /* requires SMTPUTF8 processing */
+  #endif
 } address_item_propagated;
 
 /* Bits for the flags field below */
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index b0fe177e9..ffba14662 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -1355,6 +1355,9 @@ BOOL pass_message = FALSE;
 BOOL prdr_offered = FALSE;
 BOOL prdr_active;
 #endif
+#ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_offered = FALSE;
+#endif
 BOOL dsn_all_lasthop = TRUE;
 #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE)
 BOOL dane = FALSE;
@@ -1614,6 +1617,13 @@ goto SEND_QUIT;
   if (prdr_offered)
     {DEBUG(D_transport) debug_printf("PRDR usable\n");}
 #endif
+
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  utf8_offered = esmtp
+    && addrlist->p.utf8
+    && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+		  PCRE_EOPT, NULL, 0) >= 0;
+#endif
   }
 
 /* For continuing deliveries down the same channel, the socket is the standard
@@ -1821,16 +1831,24 @@ if (continue_hostname == NULL
 #ifndef DISABLE_PRDR
   prdr_offered = esmtp
     && pcre_exec(regex_PRDR, NULL, CS buffer, Ustrlen(CS buffer), 0,
-      PCRE_EOPT, NULL, 0) >= 0
+		  PCRE_EOPT, NULL, 0) >= 0
     && verify_check_given_host(&ob->hosts_try_prdr, host) == OK;
 
   if (prdr_offered)
     {DEBUG(D_transport) debug_printf("PRDR usable\n");}
 #endif
 
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  utf8_offered = esmtp
+    && addrlist->p.utf8
+    && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+		  PCRE_EOPT, NULL, 0) >= 0;
+#endif
+
   /* Note if the server supports DSN */
-  smtp_use_dsn = esmtp && pcre_exec(regex_DSN, NULL, CS buffer, (int)Ustrlen(CS buffer), 0,
-       PCRE_EOPT, NULL, 0) >= 0;
+  smtp_use_dsn = esmtp
+    && pcre_exec(regex_DSN, NULL, CS buffer, Ustrlen(CS buffer), 0,
+		  PCRE_EOPT, NULL, 0) >= 0;
   DEBUG(D_transport) debug_printf("use_dsn=%d\n", smtp_use_dsn);
 
   /* Note if the response to EHLO specifies support for the AUTH extension.
@@ -1853,6 +1871,15 @@ message-specific. */
 
 setting_up = FALSE;
 
+#ifdef EXPERIMENTAL_INTERNATIONAL
+/* If this is an international message we need the host to speak SMTPUTF8 */
+if (addrlist->p.utf8 && !utf8_offered)
+  {
+  errno = ERRNO_UTF8_FWD;
+  goto RESPONSE_FAILED;
+  }
+#endif
+
 /* If there is a filter command specified for this transport, we can now
 set it up. This cannot be done until the identify of the host is known. */
 
@@ -1929,18 +1956,25 @@ if (prdr_offered)
   }
 #endif
 
+#ifdef EXPERIMENTAL_INTERNATIONAL
+if (addrlist->p.utf8)
+  sprintf(CS p, " SMTPUTF8"), p += 9;
+#endif
+
 /* check if all addresses have lasthop flag */
 /* do not send RET and ENVID if true */
-dsn_all_lasthop = TRUE;
-for (addr = first_addr;
+for (dsn_all_lasthop = TRUE, addr = first_addr;
      address_count < max_rcpt && addr != NULL;
      addr = addr->next)
   if ((addr->dsn_flags & rf_dsnlasthop) != 1)
+    {
     dsn_all_lasthop = FALSE;
+    break;
+    }
 
 /* Add any DSN flags to the mail command */
 
-if ((smtp_use_dsn) && (dsn_all_lasthop == FALSE))
+if (smtp_use_dsn && !dsn_all_lasthop)
   {
   if (dsn_ret == dsn_ret_hdrs)
     {
@@ -1981,27 +2015,27 @@ buffer. */
 pending_MAIL = TRUE;     /* The block starts with MAIL */
 
 rc = smtp_write_command(&outblock, smtp_use_pipelining,
-       "MAIL FROM:<%s>%s\r\n", return_path, buffer);
+	"MAIL FROM:<%s>%s\r\n", return_path, buffer);
 mail_command = string_copy(big_buffer);  /* Save for later error message */
 
 switch(rc)
   {
   case -1:                /* Transmission error */
-  goto SEND_FAILED;
+    goto SEND_FAILED;
 
   case +1:                /* Block was sent */
-  if (!smtp_read_response(&inblock, buffer, sizeof(buffer), '2',
+    if (!smtp_read_response(&inblock, buffer, sizeof(buffer), '2',
        ob->command_timeout))
-    {
-    if (errno == 0 && buffer[0] == '4')
       {
-      errno = ERRNO_MAIL4XX;
-      addrlist->more_errno |= ((buffer[1] - '0')*10 + buffer[2] - '0') << 8;
+      if (errno == 0 && buffer[0] == '4')
+	{
+	errno = ERRNO_MAIL4XX;
+	addrlist->more_errno |= ((buffer[1] - '0')*10 + buffer[2] - '0') << 8;
+	}
+      goto RESPONSE_FAILED;
       }
-    goto RESPONSE_FAILED;
-    }
-  pending_MAIL = FALSE;
-  break;
+    pending_MAIL = FALSE;
+    break;
   }
 
 /* Pass over all the relevant recipient addresses for this host, which are the
diff --git a/test/confs/4201 b/test/confs/4201
index 7d9af4bf3..3b87cd5fc 100644
--- a/test/confs/4201
+++ b/test/confs/4201
@@ -15,8 +15,10 @@ acl_smtp_rcpt = check_recipient
 trusted_users = CALLER
 log_selector = +received_recipients
 
+.ifdef SERVER
 queue_only
 queue_run_in_order
+.endif
 
 smtputf8_advertise_hosts = *
 
@@ -34,6 +36,8 @@ check_recipient:
 
 begin routers
 
+.ifdef SERVER
+
 fail_remote_domains:
   driver = redirect
   domains = ! +local_domains
@@ -43,6 +47,16 @@ localuser:
   driver = redirect
   data = :blackhole:
 
+.else
+
+rmt:
+  driver = manualroute
+  route_data = <;[127.0.0.1]:PORT_D
+  transport = rmt_smtp
+  self = send
+
+.endif
+
 # ----- Transports -----
 
 begin transports
@@ -57,4 +71,7 @@ local_delivery:
                  X-received-count: $received_count"
   return_path_add
 
+rmt_smtp:
+  driver = smtp
+
 # End
diff --git a/test/log/4201 b/test/log/4201
index 299952090..ebc09936a 100644
--- a/test/log/4201
+++ b/test/log/4201
@@ -1,9 +1,12 @@
 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= someone@some.domain H=(client) [127.0.0.1] P=utf8esmtp S=sss for userx@test.ex
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= ليهمابتكلموشعربي؟@czech.Pročprostěnemluvíčesky.com H=(client) [127.0.0.1] P=utf8esmtp S=sss for userx@test.ex
+1999-03-02 09:44:33 10HmaZ-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for usery@test.ex
+1999-03-02 09:44:33 10HmbA-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com H=localhost (the.local.host.name) [127.0.0.1] P=utf8esmtp S=sss id=E10HmaZ-0005vi-00@the.local.host.name for usery@test.ex
+1999-03-02 09:44:33 10HmaZ-0005vi-00 => usery@test.ex R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbA-0005vi-00"
+1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
 1999-03-02 09:44:33 Start queue run: pid=pppp -qq
 1999-03-02 09:44:33 10HmaX-0005vi-00 => :blackhole: <userx@test.ex> R=localuser
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
 1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <userx@test.ex> R=localuser
 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
-1999-03-02 09:44:33 End queue run: pid=pppp -qq
diff --git a/test/scripts/4200-International/4201 b/test/scripts/4200-International/4201
index bac040f9b..1bb978634 100644
--- a/test/scripts/4200-International/4201
+++ b/test/scripts/4200-International/4201
@@ -57,6 +57,21 @@ QUIT
 ****
 #
 #
+# utf-8 from, -bs input and forwarding
+exim -bs -odi
+EHLO client.bh
+MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
+RCPT TO: <usery@test.ex>
+DATA
+Subject: test
+
+body
+.
+QUIT
+****
+#
+#
+#
 killdaemon
 exim -DSERVER=server -qq
 ****
diff --git a/test/stdout/4201 b/test/stdout/4201
index b37028d59..8b89b2bd3 100644
--- a/test/stdout/4201
+++ b/test/stdout/4201
@@ -68,3 +68,15 @@ Connecting to 127.0.0.1 port 1225 ... connected
 ??? 221
 <<< 221 the.local.host.name closing connection
 End of script
+220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+250-the.local.host.name Hello CALLER at client.bh

+250-SIZE 52428800

+250-8BITMIME

+250-PIPELINING

+250-SMTPUTF8

+250 HELP

+250 OK

+250 Accepted

+354 Enter message, ending with "." on a line by itself

+250 OK id=10HmaZ-0005vi-00

+221 the.local.host.name closing connection