Add macro support to -be expansion test mode. Bug 1623

7 files changed, 208 insertions, 128 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index a9a048ecb..3fe2afb3e 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -2791,6 +2791,13 @@ files or databases you are using, you must exit and restart Exim before trying
 the same lookup again. Otherwise, because each Exim process caches the results
 of lookups, you will just get the same result as before.
 
+.new
+Macro processing is done on lines before string-expansion: new macros can be
+defined and macros will be expanded.
+Because macros in thc config file are often used for secrets, those are only
+available to admin users.
+.wen
+
 .vitem &%-bem%&&~<&'filename'&>
 .oindex "&%-bem%&"
 .cindex "testing" "string expansion"
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index 6d875d5f4..22af13554 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -61,6 +61,9 @@ Version 4.90
 15. TCP Fast Open used, with data-on-SYN, for client SMTP via SOCKS5 proxy,
     for ${readsocket } expansions, and for ClamAV.
 
+16. The "-be" expansion test mode now supports macros.  Macros are expanded
+    in test lines, and new macros can be defined.
+
 
 Version 4.89
 ------------
diff --git a/src/src/exim.c b/src/src/exim.c
index 7dd084534..d71af0ac4 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -1458,6 +1458,39 @@ return TRUE;
 
 
 /*************************************************
+*          Expansion testing			 *
+*************************************************/
+
+/* Expand and print one item, doing macro-processing.
+
+Arguments:
+  item		line for expansion
+*/
+
+static void
+expansion_test_line(uschar * line)
+{
+int len;
+BOOL dummy_macexp;
+
+Ustrncpy(big_buffer, line, big_buffer_size);
+big_buffer[big_buffer_size-1] = '\0';
+len = Ustrlen(big_buffer);
+
+(void) macros_expand(0, &len, &dummy_macexp);
+
+if (isupper(big_buffer[0]))
+  {
+  if (macro_read_assignment(big_buffer))
+    printf("Defined macro '%s'\n", mlast->name);
+  }
+else
+  if ((line = expand_string(big_buffer))) printf("%s\n", CS line);
+  else printf("Failed: %s\n", expand_string_message);
+}
+
+
+/*************************************************
 *          Entry point and high-level code       *
 *************************************************/
 
@@ -4988,7 +5021,7 @@ if (expansion_test)
   /* Read a test message from a file. We fudge it up to be on stdin, saving
   stdin itself for later reading of expansion strings. */
 
-  else if (expansion_test_message != NULL)
+  else if (expansion_test_message)
     {
     int save_stdin = dup(0);
     int fd = Uopen(expansion_test_message, O_RDONLY, 0);
@@ -5008,6 +5041,10 @@ if (expansion_test)
     clearerr(stdin);               /* Required by Darwin */
     }
 
+  /* Only admin users may see config-file macros this way */
+
+  if (!admin_user) macros = mlast = NULL;
+
   /* Allow $recipients for this testing */
 
   enable_dollar_recipients = TRUE;
@@ -5015,15 +5052,8 @@ if (expansion_test)
   /* Expand command line items */
 
   if (recipients_arg < argc)
-    {
     while (recipients_arg < argc)
-      {
-      uschar *s = argv[recipients_arg++];
-      uschar *ss = expand_string(s);
-      if (ss == NULL) printf ("Failed: %s\n", expand_string_message);
-      else printf("%s\n", CS ss);
-      }
-    }
+      expansion_test_line(argv[recipients_arg++]);
 
   /* Read stdin */
 
@@ -5031,25 +5061,18 @@ if (expansion_test)
     {
     char *(*fn_readline)(const char *) = NULL;
     void (*fn_addhist)(const char *) = NULL;
+    uschar * s;
 
-    #ifdef USE_READLINE
+#ifdef USE_READLINE
     void *dlhandle = set_readline(&fn_readline, &fn_addhist);
-    #endif
+#endif
 
-    for (;;)
-      {
-      uschar *ss;
-      uschar *source = get_stdinput(fn_readline, fn_addhist);
-      if (source == NULL) break;
-      ss = expand_string(source);
-      if (ss == NULL)
-        printf ("Failed: %s\n", expand_string_message);
-      else printf("%s\n", CS ss);
-      }
+    while (s = get_stdinput(fn_readline, fn_addhist))
+      expansion_test_line(s);
 
-    #ifdef USE_READLINE
-    if (dlhandle != NULL) dlclose(dlhandle);
-    #endif
+#ifdef USE_READLINE
+    if (dlhandle) dlclose(dlhandle);
+#endif
     }
 
   /* The data file will be open after -Mset */
@@ -5060,7 +5083,7 @@ if (expansion_test)
     deliver_datafile = -1;
     }
 
-  exim_exit(EXIT_SUCCESS, US"main");
+  exim_exit(EXIT_SUCCESS, US"main: expansion test");
   }
 
 
diff --git a/src/src/functions.h b/src/src/functions.h
index 1fe561f56..e50fa6f92 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -257,6 +257,8 @@ extern int     log_create_as_exim(uschar *);
 extern void    log_close_all(void);
 
 extern macro_item * macro_create(const uschar *, const uschar *, BOOL);
+extern BOOL    macro_read_assignment(uschar *);
+extern uschar *macros_expand(int, int *, BOOL *);
 extern void    mainlog_close(void);
 #ifdef WITH_CONTENT_SCAN
 extern int     malware(const uschar *, int);
diff --git a/src/src/readconf.c b/src/src/readconf.c
index b34372c44..8d5f38c58 100644
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -615,7 +615,10 @@ m->namelen = Ustrlen(name);
 m->replen = Ustrlen(val);
 m->name = name;
 m->replacement = val;
-mlast->next = m;
+if (mlast)
+  mlast->next = m;
+else
+  macros = m;
 mlast = m;
 return m;
 }
@@ -630,11 +633,11 @@ non-command line, macros is permitted using '==' instead of '='.
 Arguments:
   s            points to the start of the logical line
 
-Returns:       nothing
+Returns:       FALSE iff fatal error
 */
 
-static void
-read_macro_assignment(uschar *s)
+BOOL
+macro_read_assignment(uschar *s)
 {
 uschar name[64];
 int namelen = 0;
@@ -644,15 +647,21 @@ macro_item *m;
 while (isalnum(*s) || *s == '_')
   {
   if (namelen >= sizeof(name) - 1)
-    log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN,
+    {
+    log_write(0, LOG_PANIC|LOG_CONFIG_IN,
       "macro name too long (maximum is " SIZE_T_FMT " characters)", sizeof(name) - 1);
+    return FALSE;
+    }
   name[namelen++] = *s++;
   }
 name[namelen] = 0;
 
 while (isspace(*s)) s++;
 if (*s++ != '=')
-  log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "malformed macro definition");
+  {
+  log_write(0, LOG_PANIC|LOG_CONFIG_IN, "malformed macro definition");
+  return FALSE;
+  }
 
 if (*s == '=')
   {
@@ -675,15 +684,21 @@ for (m = macros; m; m = m->next)
   if (Ustrcmp(m->name, name) == 0)
     {
     if (!m->command_line && !redef)
-      log_write(0, LOG_CONFIG|LOG_PANIC_DIE, "macro \"%s\" is already "
+      {
+      log_write(0, LOG_CONFIG|LOG_PANIC, "macro \"%s\" is already "
        "defined (use \"==\" if you want to redefine it", name);
+      return FALSE;
+      }
     break;
     }
 
   if (m->namelen < namelen && Ustrstr(name, m->name) != NULL)
-    log_write(0, LOG_CONFIG|LOG_PANIC_DIE, "\"%s\" cannot be defined as "
+    {
+    log_write(0, LOG_CONFIG|LOG_PANIC, "\"%s\" cannot be defined as "
       "a macro because previously defined macro \"%s\" is a substring",
       name, m->name);
+    return FALSE;
+    }
 
   /* We cannot have this test, because it is documented that a substring
   macro is permitted (there is even an example).
@@ -697,7 +712,7 @@ for (m = macros; m; m = m->next)
 
 /* Check for an overriding command-line definition. */
 
-if (m && m->command_line) return;
+if (m && m->command_line) return TRUE;
 
 /* Redefinition must refer to an existing macro. */
 
@@ -708,18 +723,119 @@ if (redef)
     m->replacement = string_copy(s);
     }
   else
-    log_write(0, LOG_CONFIG|LOG_PANIC_DIE, "can't redefine an undefined macro "
+    {
+    log_write(0, LOG_CONFIG|LOG_PANIC, "can't redefine an undefined macro "
       "\"%s\"", name);
+    return FALSE;
+    }
 
 /* We have a new definition. */
 else
   (void) macro_create(string_copy(name), string_copy(s), FALSE);
+return TRUE;
 }
 
 
 
 
 
+/* Process line for macros. The line is in big_buffer starting at offset len.
+Expand big_buffer if needed.  Handle definitions of new macros, and
+imacro expansions, rewriting the line in thw buffer.
+
+Arguments:
+ len		Offset in buffer of start of line
+ newlen		Pointer to offset of end of line, updated on return
+ macro_found	Pointer to return that a macro was expanded
+
+Return: pointer to first nonblank char in line
+*/
+
+uschar *
+macros_expand(int len, int * newlen, BOOL * macro_found)
+{
+uschar * ss = big_buffer + len;
+uschar * s;
+macro_item * m;
+
+/* Find the true start of the physical line - leading spaces are always
+ignored. */
+
+while (isspace(*ss)) ss++;
+
+/* Process the physical line for macros. If this is the start of the logical
+line, skip over initial text at the start of the line if it starts with an
+upper case character followed by a sequence of name characters and an equals
+sign, because that is the definition of a new macro, and we don't do
+replacement therein. */
+
+s = ss;
+if (len == 0 && isupper(*s))
+  {
+  while (isalnum(*s) || *s == '_') s++;
+  while (isspace(*s)) s++;
+  if (*s != '=') s = ss;          /* Not a macro definition */
+  }
+
+/* Skip leading chars which cannot start a macro name, to avoid multiple
+pointless rescans in Ustrstr calls. */
+
+while (*s && !isupper(*s) && *s != '_') s++;
+
+/* For each defined macro, scan the line (from after XXX= if present),
+replacing all occurrences of the macro. */
+
+*macro_found = FALSE;
+for (m = macros; m; m = m->next)
+  {
+  uschar * p, *pp;
+  uschar * t = s;
+
+  while ((p = Ustrstr(t, m->name)) != NULL)
+    {
+    int moveby;
+
+/* fprintf(stderr, "%s: matched '%s' in '%s'\n", __FUNCTION__, m->name, ss); */
+    /* Expand the buffer if necessary */
+
+    while (*newlen - m->namelen + m->replen + 1 > big_buffer_size)
+      {
+      int newsize = big_buffer_size + BIG_BUFFER_SIZE;
+      uschar *newbuffer = store_malloc(newsize);
+      memcpy(newbuffer, big_buffer, *newlen + 1);
+      p = newbuffer  + (p - big_buffer);
+      s = newbuffer  + (s - big_buffer);
+      ss = newbuffer + (ss - big_buffer);
+      t = newbuffer  + (t - big_buffer);
+      big_buffer_size = newsize;
+      store_free(big_buffer);
+      big_buffer = newbuffer;
+      }
+
+    /* Shuffle the remaining characters up or down in the buffer before
+    copying in the replacement text. Don't rescan the replacement for this
+    same macro. */
+
+    pp = p + m->namelen;
+    if ((moveby = m->replen - m->namelen) != 0)
+      {
+      memmove(p + m->replen, pp, (big_buffer + *newlen) - pp + 1);
+      *newlen += moveby;
+      }
+    Ustrncpy(p, m->replacement, m->replen);
+    t = p + m->replen;
+    while (*t && !isupper(*t) && *t != '_') t++;
+    *macro_found = TRUE;
+    }
+  }
+
+/* An empty macro replacement at the start of a line could mean that ss no
+longer points to the first non-blank character. */
+
+while (isspace(*ss)) ss++;
+return ss;
+}
+
 /*************************************************
 *            Read configuration line             *
 *************************************************/
@@ -749,7 +865,6 @@ int startoffset = 0;         /* To first non-blank char in logical line */
 int len = 0;                 /* Of logical line so far */
 int newlen;
 uschar *s, *ss;
-macro_item *m;
 BOOL macro_found;
 
 /* Loop for handling continuation lines, skipping comments, and dealing with
@@ -810,82 +925,7 @@ for (;;)
     newlen += Ustrlen(big_buffer + newlen);
     }
 
-  /* Find the true start of the physical line - leading spaces are always
-  ignored. */
-
-  ss = big_buffer + len;
-  while (isspace(*ss)) ss++;
-
-  /* Process the physical line for macros. If this is the start of the logical
-  line, skip over initial text at the start of the line if it starts with an
-  upper case character followed by a sequence of name characters and an equals
-  sign, because that is the definition of a new macro, and we don't do
-  replacement therein. */
-
-  s = ss;
-  if (len == 0 && isupper(*s))
-    {
-    while (isalnum(*s) || *s == '_') s++;
-    while (isspace(*s)) s++;
-    if (*s != '=') s = ss;          /* Not a macro definition */
-    }
-
-  /* Skip leading chars which cannot start a macro name, to avoid multiple
-  pointless rescans in Ustrstr calls. */
-
-  while (*s && !isupper(*s) && *s != '_') s++;
-
-  /* For each defined macro, scan the line (from after XXX= if present),
-  replacing all occurrences of the macro. */
-
-  macro_found = FALSE;
-  for (m = macros; m; m = m->next)
-    {
-    uschar * p, *pp;
-    uschar * t = s;
-
-    while ((p = Ustrstr(t, m->name)) != NULL)
-      {
-      int moveby;
-
-/* fprintf(stderr, "%s: matched '%s' in '%s'\n", __FUNCTION__, m->name, ss); */
-      /* Expand the buffer if necessary */
-
-      while (newlen - m->namelen + m->replen + 1 > big_buffer_size)
-        {
-        int newsize = big_buffer_size + BIG_BUFFER_SIZE;
-        uschar *newbuffer = store_malloc(newsize);
-        memcpy(newbuffer, big_buffer, newlen + 1);
-        p = newbuffer  + (p - big_buffer);
-        s = newbuffer  + (s - big_buffer);
-        ss = newbuffer + (ss - big_buffer);
-        t = newbuffer  + (t - big_buffer);
-        big_buffer_size = newsize;
-        store_free(big_buffer);
-        big_buffer = newbuffer;
-        }
-
-      /* Shuffle the remaining characters up or down in the buffer before
-      copying in the replacement text. Don't rescan the replacement for this
-      same macro. */
-
-      pp = p + m->namelen;
-      if ((moveby = m->replen - m->namelen) != 0)
-        {
-        memmove(p + m->replen, pp, (big_buffer + newlen) - pp + 1);
-        newlen += moveby;
-        }
-      Ustrncpy(p, m->replacement, m->replen);
-      t = p + m->replen;
-      while (*t && !isupper(*t) && *t != '_') t++;
-      macro_found = TRUE;
-      }
-    }
-
-  /* An empty macro replacement at the start of a line could mean that ss no
-  longer points to the first non-blank character. */
-
-  while (isspace(*ss)) ss++;
+  ss = macros_expand(len, &newlen, &macro_found);
 
   /* Check for comment lines - these are physical lines. */
 
@@ -3277,7 +3317,8 @@ while ((s = get_config_line()))
     log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN,
       "found unexpected BOM (Byte Order Mark)");
 
-  if (isupper(s[0])) read_macro_assignment(s);
+  if (isupper(s[0]))
+    { if (!macro_read_assignment(s)) exim_exit(EXIT_FAILURE, US""); }
 
   else if (Ustrncmp(s, "domainlist", 10) == 0)
     read_named_list(&domainlist_anchor, &domainlist_count,
@@ -3724,7 +3765,7 @@ while ((buffer = get_config_line()) != NULL)
       (d->info->init)(d);
       d = NULL;
       }
-    read_macro_assignment(buffer);
+    if (!macro_read_assignment(buffer)) exim_exit(EXIT_FAILURE, US"");
     continue;
     }
 
@@ -4225,7 +4266,7 @@ between ACLs. */
 
 acl_line = get_config_line();
 
-while(acl_line != NULL)
+while(acl_line)
   {
   uschar name[64];
   tree_node *node;
@@ -4234,7 +4275,7 @@ while(acl_line != NULL)
   p = readconf_readname(name, sizeof(name), acl_line);
   if (isupper(*name) && *p == '=')
     {
-    read_macro_assignment(acl_line);
+    if (!macro_read_assignment(acl_line)) exim_exit(EXIT_FAILURE, US"");
     acl_line = get_config_line();
     continue;
     }
@@ -4278,7 +4319,7 @@ log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "local_scan() options not supported: "
 #else
 
 uschar *p;
-while ((p = get_config_line()) != NULL)
+while ((p = get_config_line()))
   {
   (void) readconf_handle_option(p, local_scan_options, local_scan_options_count,
     NULL, US"local_scan option \"%s\" unknown");
@@ -4368,14 +4409,14 @@ while(next_section[0] != 0)
 void
 readconf_save_config(const uschar *s)
 {
-  save_config_line(string_sprintf("# Exim Configuration (%s)",
-    running_in_test_harness ? US"X" : s));
+save_config_line(string_sprintf("# Exim Configuration (%s)",
+  running_in_test_harness ? US"X" : s));
 }
 
 static void
 save_config_position(const uschar *file, int line)
 {
-  save_config_line(string_sprintf("# %d \"%s\"", line, file));
+save_config_line(string_sprintf("# %d \"%s\"", line, file));
 }
 
 /* Append a pre-parsed logical line to the config lines store,
diff --git a/test/scripts/0000-Basic/0542 b/test/scripts/0000-Basic/0542
index 0c6362bde..1c8c03b5f 100644
--- a/test/scripts/0000-Basic/0542
+++ b/test/scripts/0000-Basic/0542
@@ -7,13 +7,15 @@ Subject: The subject is not the object
 This is the body of the message. Make the line longer than any header.
 ****
 sudo exim -be -Mset $msg1
-From: $h_from:
-Subject: $h_subject:
+from: $h_from:
+subject: $h_subject:
 message_body_size=$message_body_size
 message_id=$message_id
 message_exim_id=$message_exim_id
 max_received_linelength=$max_received_linelength
 recipients=$recipients
+TESTING_MACROS=$recipients
+(TESTING_MACROS)
 ****
 exim -bs
 mail from:<userz@test.ex>
@@ -29,8 +31,8 @@ This is the body of the message. Make the line much longer than any header.
 quit
 ****
 sudo exim -be -Mset $msg2
-From: $h_from:
-Subject: $h_subject:
+from: $h_from:
+subject: $h_subject:
 message_body_size=$message_body_size
 message_id=$message_id
 message_exim_id=$message_exim_id
diff --git a/test/stdout/0542 b/test/stdout/0542
index ab29dc9c0..ce255a744 100644
--- a/test/stdout/0542
+++ b/test/stdout/0542
@@ -1,10 +1,12 @@
-> From: Himself <himself@there.tld>
-> Subject: The subject is not the object
+> from: Himself <himself@there.tld>
+> subject: The subject is not the object
 > message_body_size=71
 > message_id=10HmaX-0005vi-00
 > message_exim_id=10HmaX-0005vi-00
 > max_received_linelength=70
 > recipients=userx@test.x, usery@test.ex
+> Defined macro 'TESTING_MACROS'
+> (userx@test.x, usery@test.ex)
 > 
 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

 250 OK

@@ -13,8 +15,8 @@
 354 Enter message, ending with "." on a line by itself

 250 OK id=10HmaY-0005vi-00

 221 myhost.test.ex closing connection

-> From: Himself <himself@there.tld>
-> Subject: The subject is not the object
+> from: Himself <himself@there.tld>
+> subject: The subject is not the object
 > message_body_size=76
 > message_id=10HmaY-0005vi-00
 > message_exim_id=10HmaY-0005vi-00