Add retry type "lookup". Bug 1566

    Also add dnslookup router option "fail_defer_domains" to support it.

11 files changed, 121 insertions, 52 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index 8bf643632..e124ab018 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -17989,6 +17989,9 @@ There are a few cases where a &(dnslookup)& router will decline to accept
 an address; if such a router is expected to handle "all remaining non-local
 domains", then it is important to set &%no_more%&.
 
+The router will defer rather than decline if the domain
+is found in the &%fail_defer_domains%& router option.
+
 Reasons for a &(dnslookup)& router to decline currently include:
 .ilist
 The domain does not exist in DNS
@@ -18090,6 +18093,17 @@ This applies to all of the SRV, MX, AAAA, A lookup sequence.
 
 
 
+.option fail_defer_domains dnslookup "domain list&!!" unset
+.cindex "MX record" "not found"
+DNS lookups for domains matching &%fail_defer_domains%&
+which find no matching record will cause the router to defer
+rather than the default behaviour of decline.
+This maybe be useful for queueing messages for a newly created
+domain while the DNS configuration is not ready.
+However, it will result in any message with mistyped domains
+also being queued.
+
+
 .option mx_domains dnslookup "domain list&!!" unset
 .cindex "MX record" "required to exist"
 .cindex "SRV record" "required to exist"
@@ -24360,6 +24374,13 @@ A server unexpectedly closed the SMTP connection. There may, of course,
 legitimate reasons for this (host died, network died), but if it repeats a lot
 for the same host, it indicates something odd.
 
+.vitem %&lookup%&
+A DNS lookup for a host failed.
+Note that a &%dnslookup%& router will need to have matched
+its &%fail_defer_domains%& option for this retry type to be usable.
+Also note that a &%manualroute%& router will probably need
+its &%host_find_failed%& option set to &%defer&%.
+
 .vitem &%refused_MX%&
 A connection to a host obtained from an MX record was refused.
 
diff --git a/src/src/exim_dbutil.c b/src/src/exim_dbutil.c
index 48323243d..72eee42e8 100644
--- a/src/src/exim_dbutil.c
+++ b/src/src/exim_dbutil.c
@@ -642,19 +642,6 @@ while (key != NULL)
         printf("\n");
         }
 
-      /* Old-style domain record, without separate timestamps. This code can
-      eventually be thrown away, say in 5 years' time (it's now Feb 2003). */
-
-      else
-        {
-        printf("%s %s callout=%s postmaster=%s random=%s\n",
-          print_time(((dbdata_generic *)value)->time_stamp),
-          keybuffer,
-          print_cache(callout->result),
-          print_cache(callout->postmaster_result),
-          print_cache(callout->random_result));
-        }
-
       break;
 
       case type_ratelimit:
diff --git a/src/src/macros.h b/src/src/macros.h
index 4db2bdab9..0f893e812 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -487,7 +487,7 @@ to conflict with system errno values. */
 #define ERRNO_UIDFAIL        (-29)   /* Failed to get uid */
 #define ERRNO_BADTRANSPORT   (-30)   /* Unset or non-existent transport */
 #define ERRNO_MBXLENGTH      (-31)   /* MBX length mismatch */
-#define ERRNO_UNKNOWNHOST    (-32)   /* Lookup failed in smtp transport */
+#define ERRNO_UNKNOWNHOST    (-32)   /* Lookup failed routing or in smtp tpt */
 #define ERRNO_FORMATUNKNOWN  (-33)   /* Can't match format in appendfile */
 #define ERRNO_BADCREATE      (-34)   /* Creation outside home in appendfile */
 #define ERRNO_LISTDEFER      (-35)   /* Can't check a list; lookup defer */
diff --git a/src/src/readconf.c b/src/src/readconf.c
index 31e04e97e..e2d3c518f 100644
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -3696,7 +3696,8 @@ Returns:       NULL if decoded correctly; else points to error text
 */
 
 uschar *
-readconf_retry_error(const uschar *pp, const uschar *p, int *basic_errno, int *more_errno)
+readconf_retry_error(const uschar *pp, const uschar *p,
+  int *basic_errno, int *more_errno)
 {
 int len;
 const uschar *q = pp;
@@ -3736,24 +3737,19 @@ else if (len == 7 && strncmpic(pp, US"timeout", len) == 0)
       { 'A',   'M',    RTEF_CTOUT,  RTEF_CTOUT|'A', RTEF_CTOUT|'M' };
 
     for (i = 0; i < sizeof(extras)/sizeof(uschar *); i++)
-      {
       if (strncmpic(x, extras[i], xlen) == 0)
         {
         *more_errno = values[i];
         break;
         }
-      }
 
     if (i >= sizeof(extras)/sizeof(uschar *))
-      {
       if (strncmpic(x, US"DNS", xlen) == 0)
-        {
         log_write(0, LOG_MAIN|LOG_PANIC, "\"timeout_dns\" is no longer "
           "available in retry rules (it has never worked) - treated as "
           "\"timeout\"");
-        }
-      else return US"\"A\", \"MX\", or \"connect\" expected after \"timeout\"";
-      }
+      else
+        return US"\"A\", \"MX\", or \"connect\" expected after \"timeout\"";
     }
   }
 
@@ -3780,8 +3776,8 @@ else if (strncmpic(pp, US"mail_4", 6) == 0 ||
     return string_sprintf("%.4s_4 must be followed by xx, dx, or dd, where "
       "x is literal and d is any digit", pp);
 
-  *basic_errno = (*pp == 'm')? ERRNO_MAIL4XX :
-                 (*pp == 'r')? ERRNO_RCPT4XX : ERRNO_DATA4XX;
+  *basic_errno = *pp == 'm' ? ERRNO_MAIL4XX :
+                 *pp == 'r' ? ERRNO_RCPT4XX : ERRNO_DATA4XX;
   *more_errno = x << 8;
   }
 
@@ -3795,6 +3791,9 @@ else if (strncmpic(pp, US"lost_connection", p - pp) == 0)
 else if (strncmpic(pp, US"tls_required", p - pp) == 0)
   *basic_errno = ERRNO_TLSREQUIRED;
 
+else if (strncmpic(pp, US"lookup", p - pp) == 0)
+  *basic_errno = ERRNO_UNKNOWNHOST;
+
 else if (len != 1 || Ustrncmp(pp, "*", 1) != 0)
   return string_sprintf("unknown or malformed retry error \"%.*s\"", (int) (p-pp), pp);
 
@@ -3849,10 +3848,8 @@ if (*p != 0 && !isspace(*p) && *p != ',' && *p != ';')
 *paddr = p;
 switch (type)
   {
-  case 0:
-  return readconf_readtime(pp, *p, FALSE);
-  case 1:
-  return readconf_readfixed(pp, *p);
+  case 0: return readconf_readtime(pp, *p, FALSE);
+  case 1: return readconf_readfixed(pp, *p);
   }
 return 0;    /* Keep picky compilers happy */
 }
@@ -3866,7 +3863,7 @@ retry_config **chain = &retries;
 retry_config *next;
 const uschar *p;
 
-while ((p = get_config_line()) != NULL)
+while ((p = get_config_line()))
   {
   retry_rule **rchain;
   const uschar *pp;
@@ -3890,8 +3887,8 @@ while ((p = get_config_line()) != NULL)
 
   /* Test error names for things we understand. */
 
-  if ((error = readconf_retry_error(pp, p, &(next->basic_errno),
-       &(next->more_errno))) != NULL)
+  if ((error = readconf_retry_error(pp, p, &next->basic_errno,
+       &next->more_errno)))
     log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "%s", error);
 
   /* There may be an optional address list of senders to be used as another
@@ -3928,18 +3925,18 @@ while ((p = get_config_line()) != NULL)
     switch (rule->rule)
       {
       case 'F':   /* Fixed interval */
-      rule->p1 = retry_arg(&p, 0);
-      break;
+	rule->p1 = retry_arg(&p, 0);
+	break;
 
       case 'G':   /* Geometrically increasing intervals */
       case 'H':   /* Ditto, but with randomness */
-      rule->p1 = retry_arg(&p, 0);
-      rule->p2 = retry_arg(&p, 1);
-      break;
+	rule->p1 = retry_arg(&p, 0);
+	rule->p2 = retry_arg(&p, 1);
+	break;
 
       default:
-      log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "unknown retry rule letter");
-      break;
+	log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "unknown retry rule letter");
+	break;
       }
 
     if (rule->timeout <= 0 || rule->p1 <= 0 ||
diff --git a/src/src/routers/dnslookup.c b/src/src/routers/dnslookup.c
index aa5b924c7..8ef4e0da9 100644
--- a/src/src/routers/dnslookup.c
+++ b/src/src/routers/dnslookup.c
@@ -22,6 +22,8 @@ optionlist dnslookup_router_options[] = {
       (void *)(offsetof(dnslookup_router_options_block, dnssec_request_domains)) },
   { "dnssec_require_domains",         opt_stringptr,
       (void *)(offsetof(dnslookup_router_options_block, dnssec_require_domains)) },
+  { "fail_defer_domains", opt_stringptr,
+      (void *)(offsetof(dnslookup_router_options_block, fail_defer_domains)) },
   { "mx_domains",         opt_stringptr,
       (void *)(offsetof(dnslookup_router_options_block, mx_domains)) },
   { "mx_fail_domains",    opt_stringptr,
@@ -59,7 +61,8 @@ dnslookup_router_options_block dnslookup_router_option_defaults = {
   NULL,            /* srv_fail_domains */
   NULL,            /* check_srv */
   NULL,            /* dnssec_request_domains */
-  NULL             /* dnssec_require_domains */
+  NULL,            /* dnssec_require_domains */
+  NULL             /* fail_defer_domains */
 };
 
 
@@ -161,10 +164,10 @@ DEBUG(D_route)
 
 /* If an SRV check is required, expand the service name */
 
-if (ob->check_srv != NULL)
+if (ob->check_srv)
   {
-  srv_service = expand_string(ob->check_srv);
-  if (srv_service == NULL && !expand_string_forcedfail)
+  if (  !(srv_service = expand_string(ob->check_srv))
+     && !expand_string_forcedfail)
     {
     addr->message = string_sprintf("%s router: failed to expand \"%s\": %s",
       rblock->name, ob->check_srv, expand_string_message);
@@ -195,8 +198,8 @@ does not appear in the message header so it is also OK to widen. The
 suppression of widening for sender addresses is silent because it is the
 normal desirable behaviour. */
 
-if (ob->widen_domains != NULL &&
-    (verify != v_sender || !ob->rewrite_headers || addr->parent != NULL))
+if (  ob->widen_domains
+   && (verify != v_sender || !ob->rewrite_headers || addr->parent))
   {
   listptr = ob->widen_domains;
   widen = string_nextinlist(&listptr, &widen_sep, widen_buffer,
@@ -220,12 +223,12 @@ for (;;)
   int flags = whichrrs;
   BOOL removed = FALSE;
 
-  if (pre_widen != NULL)
+  if (pre_widen)
     {
     h.name = pre_widen;
     pre_widen = NULL;
     }
-  else if (widen != NULL)
+  else if (widen)
     {
     h.name = string_sprintf("%s.%s", addr->domain, widen);
     widen = string_nextinlist(&listptr, &widen_sep, widen_buffer,
@@ -233,7 +236,7 @@ for (;;)
     DEBUG(D_route) debug_printf("%s router widened %s to %s\n", rblock->name,
       addr->domain, h.name);
     }
-  else if (post_widen != NULL)
+  else if (post_widen)
     {
     h.name = post_widen;
     post_widen = NULL;
@@ -260,7 +263,7 @@ for (;;)
   subsequently. We use the same logic as that for widen_domains above to avoid
   requesting a header rewrite that cannot work. */
 
-  if (verify != v_sender || !ob->rewrite_headers || addr->parent != NULL)
+  if (verify != v_sender || !ob->rewrite_headers || addr->parent)
     {
     if (ob->qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE;
     if (ob->search_parents) flags |= HOST_FIND_SEARCH_PARENTS;
@@ -277,10 +280,9 @@ for (;;)
   the option is historical. */
 
   if ((rc == HOST_FOUND || rc == HOST_FOUND_LOCAL) && h.mx < 0 &&
-       ob->mx_domains != NULL)
-    {
+       ob->mx_domains)
     switch(match_isinlist(fully_qualified_name,
-          (const uschar **)&(ob->mx_domains), 0,
+          CUSS &(ob->mx_domains), 0,
           &domainlist_anchor, addr->domain_cache, MCL_DOMAIN, TRUE, NULL))
       {
       case DEFER:
@@ -292,7 +294,6 @@ for (;;)
         rblock->name, fully_qualified_name);
       continue;
       }
-    }
 
   /* Deferral returns forthwith, and anything other than failure breaks the
   loop. */
@@ -329,6 +330,7 @@ for (;;)
       addr->message = US"an MX or SRV record indicated no SMTP service";
     else
       {
+      addr->basic_errno = ERRNO_UNKNOWNHOST;
       addr->message = US"all relevant MX records point to non-existent hosts";
       if (!allow_mx_to_ip && string_is_ip_address(h.name, NULL) != 0)
         {
@@ -340,6 +342,22 @@ for (;;)
           addr->message);
         }
       }
+    if (ob->fail_defer_domains)
+      {
+      switch(match_isinlist(fully_qualified_name,
+	    CUSS &ob->fail_defer_domains, 0,
+	    &domainlist_anchor, addr->domain_cache, MCL_DOMAIN, TRUE, NULL))
+	{
+	case DEFER:
+	  addr->message = US"lookup defer for fail_defer_domains";
+	  return DEFER;
+
+	case OK:
+	  DEBUG(D_route) debug_printf("%s router: matched fail_defer_domains\n",
+	    rblock->name);
+	  return DEFER;
+	}
+      }
     return DECLINE;
     }
 
@@ -431,3 +449,5 @@ return rf_queue_add(addr, addr_local, addr_remote, rblock, pw)?
 }
 
 /* End of routers/dnslookup.c */
+/* vi: aw ai sw=2
+*/
diff --git a/src/src/routers/dnslookup.h b/src/src/routers/dnslookup.h
index 518b7f478..907ff0ce3 100644
--- a/src/src/routers/dnslookup.h
+++ b/src/src/routers/dnslookup.h
@@ -19,6 +19,7 @@ typedef struct {
   uschar *check_srv;
   uschar *dnssec_request_domains;
   uschar *dnssec_require_domains;
+  uschar *fail_defer_domains;
 } dnslookup_router_options_block;
 
 /* Data for reading the private options. */
diff --git a/src/src/routers/rf_lookup_hostlist.c b/src/src/routers/rf_lookup_hostlist.c
index 28d4fc4c1..ab2e4ec2c 100644
--- a/src/src/routers/rf_lookup_hostlist.c
+++ b/src/src/routers/rf_lookup_hostlist.c
@@ -168,6 +168,7 @@ for (h = addr->host_list; h != NULL; h = next_h)
     if (hff_code == hff_pass) return PASS;
     if (hff_code == hff_decline) return DECLINE;
 
+    addr->basic_errno = ERRNO_UNKNOWNHOST;
     addr->message =
       string_sprintf("lookup of host \"%s\" failed in %s router%s",
         h->name, rblock->name,
diff --git a/test/confs/0099 b/test/confs/0099
index e54cf628c..c4e1a8339 100644
--- a/test/confs/0099
+++ b/test/confs/0099
@@ -21,12 +21,20 @@ log_selector = +received_recipients
 
 begin routers
 
+bydns:
+  driver = dnslookup
+  domains = mxt2.test.ex
+  transport = smtp
+  fail_defer_domains = *
+
 all:
   driver = manualroute
   route_list = simple  thishost.test.ex  byname \
 	     ; complex thisloop.test.ex  byname \
+	     ; nonexist nonexist.test.ex byname \
 	     ; *       127.0.0.1         byname
   self = send
+  host_find_failed = defer
   transport = smtp
 
 
@@ -50,6 +58,7 @@ begin retry
 rcpt45x.test.ex              rcpt_45x      F,10d,2m
 rcpt463.test.ex              rcpt_463      F,10d,3m
 rcpt4xx.test.ex              rcpt_4xx      F,10d,1m
+*                            lookup        F,10d,30m
 
 *@\N^\d                      *             F,5d,1m
 *@*.abcd.ex                  *             F,5d,2m
diff --git a/test/log/0099 b/test/log/0099
index 952b24041..6fa94331a 100644
--- a/test/log/0099
+++ b/test/log/0099
@@ -21,3 +21,8 @@
 1999-03-02 09:44:33 10HmbB-0005vi-00 H=thisloop.test.ex [ip4.ip4.ip4.ip4] Connection refused
 1999-03-02 09:44:33 10HmbB-0005vi-00 H=thisloop.test.ex [127.0.0.1] Connection refused
 1999-03-02 09:44:33 10HmbB-0005vi-00 == without@complex R=all T=smtp defer (dd): Connection refused
+1999-03-02 09:44:33 10HmbC-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss for userx@nonexist
+1999-03-02 09:44:33 10HmbC-0005vi-00 no IP address found for host nonexist.test.ex
+1999-03-02 09:44:33 10HmbC-0005vi-00 == userx@nonexist R=all defer (-32): lookup of host "nonexist.test.ex" failed in all router
+1999-03-02 09:44:33 10HmbD-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss for userx@mxt2.test.ex
+1999-03-02 09:44:33 10HmbD-0005vi-00 == userx@mxt2.test.ex R=bydns defer (-32): all relevant MX records point to non-existent hosts
diff --git a/test/scripts/0000-Basic/0099 b/test/scripts/0000-Basic/0099
index 6dfc14a04..7ad88b625 100644
--- a/test/scripts/0000-Basic/0099
+++ b/test/scripts/0000-Basic/0099
@@ -38,6 +38,16 @@ Test message
 ****
 dump retry
 #
+# one for a lookup-failure (in manualroute)
+exim -odi userx@nonexist
+Test message
+****
+# one for a lookup-failure (in dnslookup)
+exim -odi userx@mxt2.test.ex
+Test message
+****
+dump retry
+#
 #
 exim -brt x@dark.star.ex
 ****
@@ -69,3 +79,5 @@ exim -f "d@e.f" -brt x@Q.abcd.ex
 ****
 exim -brt x@not.not.ex
 ****
+exim -brt x@nonexist.test.ex lookup
+****
diff --git a/test/stdout/0099 b/test/stdout/0099
index 35b7f54b6..0f6f7f2fa 100644
--- a/test/stdout/0099
+++ b/test/stdout/0099
@@ -40,6 +40,21 @@ first failed = time last try = time2 next try = time2 + 900
 first failed = time last try = time2 next try = time2 + 900
   T:thisloop.test.ex:999 dd 65 Connection refused
 first failed = time last try = time2 next try = time2 + 900
++++++++++++++++++++++++++++
+  R:mxt2.test.ex -32 0 all relevant MX records point to non-existent hosts
+first failed = time last try = time2 next try = time2 + 1800
+  R:nonexist -32 0 lookup of host "nonexist.test.ex" failed in all router
+first failed = time last try = time2 next try = time2 + 1800
+  T:thishost.test.ex:127.0.0.1:999 dd 65 Connection refused
+first failed = time last try = time2 next try = time2 + 900
+  T:thishost.test.ex:999 dd 65 Connection refused
+first failed = time last try = time2 next try = time2 + 900
+  T:thisloop.test.ex:127.0.0.1:999 dd 65 Connection refused
+first failed = time last try = time2 next try = time2 + 900
+  T:thisloop.test.ex:ip4.ip4.ip4.ip4:999 dd 65 Connection refused
+first failed = time last try = time2 next try = time2 + 900
+  T:thisloop.test.ex:999 dd 65 Connection refused
+first failed = time last try = time2 next try = time2 + 900
 Retry rule: *.star.ex  *  F,3d,10m; 
 Retry rule: lsearch*@;TESTSUITE/aux-fixed/0099.rlist  *  F,1d,3m; 
 Retry rule: !*.not.ex  *  F,2d,15m; 
@@ -55,3 +70,4 @@ Retry rule: rcpt463.test.ex  *  F,1w3d,3m;
 Retry rule: *  *  G,1d,1m,1.5; 
 Retry rule: *  *  G,2d,2m,1.5; 
 Retry rule: *  *  F,1w5d,2h30m; 
+Retry rule: *  *  F,1w3d,30m;