DNS: time-limit cached returns, using TTL. Bug 1395

This can matter for fast-changing data such as DNSBLs.

11 files changed, 264 insertions, 2 deletions

diff --git a/test/confs/2200 b/test/confs/2200
index 3bef13363..ae5988cee 100644
--- a/test/confs/2200
+++ b/test/confs/2200
@@ -1,4 +1,5 @@
 # Exim test configuration 2200
+# Check for dnsdb cache TTL handling
 
 exim_path = EXIM_PATH
 host_lookup_order = bydns
@@ -8,4 +9,20 @@ log_file_path = DIR/spool/log/%slog
 gecos_pattern = ""
 gecos_name = CALLER_NAME
 
+# ----- Main settings -----
+
+acl_not_smtp = check_rcpt
+queue_only
+
+begin acl
+
+check_rcpt:
+  warn
+	set acl_m1 =    ${map {<,$recipients} \
+				{${lookup dnsdb{a=${domain:$item}}{$value}fail}}}
+	delay =		4s
+	set acl_m1 =    ${map {<,$recipients} \
+				{${lookup dnsdb{a=${domain:$item}}{$value}fail}}}
+  accept
+
 # End
diff --git a/test/confs/2201 b/test/confs/2201
index ae17b7cb4..7b257cce3 100644
--- a/test/confs/2201
+++ b/test/confs/2201
@@ -12,11 +12,43 @@ gecos_name = CALLER_NAME
 
 trusted_users = CALLER
 
+acl_smtp_rcpt = check_rcpt
+acl_not_smtp = check_sndr
+queue_only
+
+# - ACL --
+begin acl
+
+check_rcpt:
+	# Do not care about result, looking at debug output
+	# expect an original lookup, a cached lookup avoidance
+	# then a TTL-required repeat lookup
+  warn	dnslists =	rbl.test.ex/V4NET.11.12.14
+	dnslists =	rbl.test.ex/V4NET.11.12.14
+	delay =		4s
+	dnslists =	rbl.test.ex/V4NET.11.12.14
+  accept
+
+check_sndr:
+	# Do not care about result, looking at debug output
+	# expect an original lookup, a cached lookup avoidance
+	# then a TTL-required repeat lookup
+  warn	sender_domains = dnsdb;a=$sender_address_domain
+	sender_domains = dnsdb;a=$sender_address_domain
+	delay =		4s
+	sender_domains = dnsdb;a=$sender_address_domain
+  accept
+
 
 # ----- Routers -----
 
 begin routers
 
+r0:
+  driver = accept
+  senders = a@shorthost.test.ex
+  transport = remote_delivery
+
 r1:
   driver = accept
   domains = dnsdb;$domain
@@ -41,6 +73,11 @@ local_delivery:
   file = DIR/test-mail/$local_part
   user = CALLER
 
+remote_delivery:
+  driver = smtp
+  hosts = 127.0.0.1
+  allow_localhost
+  port = PORT_D
 
 # ----- Retry -----
 
diff --git a/test/dnszones-src/db.test.ex b/test/dnszones-src/db.test.ex
index 4cbf0f251..61f274eb0 100644
--- a/test/dnszones-src/db.test.ex
+++ b/test/dnszones-src/db.test.ex
@@ -51,6 +51,10 @@ mx.xn--1xa         A       V4NET.255.255.255
 thishost     A       127.0.0.1
 localhost4   A       127.0.0.1
 
+; A localhost with short TTL
+
+TTL=2 shorthost A    127.0.0.1
+
 
 ; Something that gives both the IP and the loopback
 
@@ -170,7 +174,7 @@ cname4       CNAME   thishost
 
 13.12.11.V4NET.rbl    A   127.0.0.2
                       TXT "This is a test blacklisting message"
-14.12.11.V4NET.rbl    A   127.0.0.2
+TTL=2 14.12.11.V4NET.rbl A 127.0.0.2
                       TXT "This is a test blacklisting message"
 15.12.11.V4NET.rbl    A   127.0.0.2
                       TXT "This is a very long blacklisting message, continuing for ages and ages and certainly being longer than 128 characters which was a previous limit on the length that Exim was prepared to handle."
diff --git a/test/log/2200 b/test/log/2200
new file mode 100644
index 000000000..f59faf8b9
--- /dev/null
+++ b/test/log/2200
@@ -0,0 +1 @@
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
diff --git a/test/log/2201 b/test/log/2201
new file mode 100644
index 000000000..d413ddaf3
--- /dev/null
+++ b/test/log/2201
@@ -0,0 +1,7 @@
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= a@shorthost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
+1999-03-02 09:44:33 Start queue run: pid=pppp
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= a@shorthost.test.ex H=localhost (myhost.test.ex) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@myhost.test.ex
+1999-03-02 09:44:33 10HmaX-0005vi-00 => t@test.ex R=r0 T=remote_delivery H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00"
+1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
+1999-03-02 09:44:33 End queue run: pid=pppp
diff --git a/test/scripts/2200-dnsdb/2200 b/test/scripts/2200-dnsdb/2200
index 40837dbd0..9f6262524 100644
--- a/test/scripts/2200-dnsdb/2200
+++ b/test/scripts/2200-dnsdb/2200
@@ -49,3 +49,16 @@ defer_strict:ten-1         ${lookup dnsdb{defer_strict,a=test.again.dns:ten-1.te
 
 delay1500                  ${lookup dnsdb{retrans_1s,retry_2,a=delay1500.test.ex}}
 ****
+#
+# Cacheability
+exim -d-all+lookup -be
+a=localhost.test.ex        ${lookup dnsdb{a=localhost.test.ex}{$value}fail}
+a=localhost.test.ex        ${lookup dnsdb{a=localhost.test.ex}{$value}fail}
+****
+#
+# TTL-limited noncacheability
+exim -d-all+lookup -odq user@shorthost.test.ex
+****
+#
+no_msglog_check
+#
diff --git a/test/scripts/2200-dnsdb/2201 b/test/scripts/2200-dnsdb/2201
index 600298793..d432ca187 100644
--- a/test/scripts/2200-dnsdb/2201
+++ b/test/scripts/2200-dnsdb/2201
@@ -1,6 +1,24 @@
 # query-style lookup in domains, local_parts, senders
+munge debug_pid
 exim -d -bt test.ex@test.ex unknown@test.ex
 ****
 2
 exim -f a@b.c -bt test.ex@test.ex unknown@test.ex
 ****
+#
+#
+# lookup non/cacheability, lookup done as a list item
+exim -d-all+lookup -odq -f a@shorthost.test.ex t@test.ex
+****
+#
+#
+# lookup non/cacheability, lookup done for a dnslists= ACL condition
+exim -DSERVER=server -d-all+dnsbl -bd -oX PORT_D
+****
+exim -q
+****
+#
+killdaemon
+millisleep 500
+no_msglog_check
+#
diff --git a/test/src/fakens.c b/test/src/fakens.c
index a03f94a07..7e93979ec 100644
--- a/test/src/fakens.c
+++ b/test/src/fakens.c
@@ -61,6 +61,9 @@ Any DNS record line in a zone file can be prefixed with "AA "
 if all the records found by a lookup are marked
 as such then the response will have the "AA" bit set.
 
+Any DNS record line in a zone file can be prefixed with "TTL=" and
+a number of seconds (followed by one space).
+
 */
 
 #include <ctype.h>
@@ -105,6 +108,8 @@ typedef struct tlist {
   int value;
 } tlist;
 
+#define DEFAULT_TTL 3600U
+
 /* On some (older?) operating systems, the standard ns_t_xxx definitions are
 not available, and only the older T_xxx ones exist in nameser.h. If ns_t_a is
 not defined, assume we are in this state. A really old system might not even
@@ -347,6 +352,7 @@ while (fgets(CS buffer, sizeof(buffer), f) != NULL)
   BOOL rr_sec = FALSE;
   BOOL rr_aa = FALSE;
   int delay = 0;
+  uint ttl = DEFAULT_TTL;
 
   p = buffer;
   while (isspace(*p)) p++;
@@ -380,6 +386,12 @@ while (fgets(CS buffer, sizeof(buffer), f) != NULL)
       for (p += 6; *p >= '0' && *p <= '9'; p++) delay = delay*10 + *p - '0';
       if (isspace(*p)) p++;
       }
+    else if (Ustrncmp(p, US"TTL=", 4) == 0)     /* TTL for record */
+      {
+      ttl = 0;
+      for (p += 4; *p >= '0' && *p <= '9'; p++) ttl = ttl*10 + *p - '0';
+      if (isspace(*p)) p++;
+      }
     else
       break;
     }
@@ -459,7 +471,10 @@ while (fgets(CS buffer, sizeof(buffer), f) != NULL)
   *pk++ = 0;
   *pk++ = 1;     /* class = IN */
 
-  pk += 4;       /* TTL field; don't care */
+  *pk++ = (ttl >>24) & 255;
+  *pk++ = (ttl >>16) & 255;
+  *pk++ = (ttl >> 8) & 255;
+  *pk++ = ttl & 255;
 
   rdlptr = pk;   /* remember rdlength field */
   pk += 2;
diff --git a/test/stderr/2200 b/test/stderr/2200
new file mode 100644
index 000000000..4fec895ea
--- /dev/null
+++ b/test/stderr/2200
@@ -0,0 +1,52 @@
+Exim version x.yz ....
+configuration file is TESTSUITE/test-config
+admin user
+search_open: dnsdb "NULL"
+search_find: file="NULL"
+  key="a=localhost.test.ex" partial=-1 affix=NULL starflags=0
+LRU list:
+internal_search_find: file="NULL"
+  type=dnsdb key="a=localhost.test.ex"
+database lookup required for a=localhost.test.ex
+dnsdb key: localhost.test.ex
+lookup yielded: 127.0.0.1
+search_open: dnsdb "NULL"
+  cached open
+search_find: file="NULL"
+  key="a=localhost.test.ex" partial=-1 affix=NULL starflags=0
+LRU list:
+internal_search_find: file="NULL"
+  type=dnsdb key="a=localhost.test.ex"
+cached data used for lookup of a=localhost.test.ex
+lookup yielded: 127.0.0.1
+search_tidyup called
+>>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>>
+Exim version x.yz ....
+configuration file is TESTSUITE/test-config
+admin user
+search_tidyup called
+search_tidyup called
+search_open: dnsdb "NULL"
+search_find: file="NULL"
+  key="a=shorthost.test.ex" partial=-1 affix=NULL starflags=0
+LRU list:
+internal_search_find: file="NULL"
+  type=dnsdb key="a=shorthost.test.ex"
+database lookup required for a=shorthost.test.ex
+dnsdb key: shorthost.test.ex
+lookup yielded: 127.0.0.1
+search_open: dnsdb "NULL"
+  cached open
+search_find: file="NULL"
+  key="a=shorthost.test.ex" partial=-1 affix=NULL starflags=0
+LRU list:
+internal_search_find: file="NULL"
+  type=dnsdb key="a=shorthost.test.ex"
+cached data found but past valid time; database lookup required for a=shorthost.test.ex
+dnsdb key: shorthost.test.ex
+lookup yielded: 127.0.0.1
+LOG: MAIN
+  <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+created log directory TESTSUITE/spool/log
+search_tidyup called
+>>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>>
diff --git a/test/stderr/2201 b/test/stderr/2201
index 925d3c6f7..523ae4395 100644
--- a/test/stderr/2201
+++ b/test/stderr/2201
@@ -3,9 +3,11 @@ changed uid/gid: forcing real = effective
   uid=uuuu gid=CALLER_GID pid=pppp
 seeking password data for user "CALLER": cache not available
 getpwnam() succeeded uid=CALLER_UID gid=CALLER_GID
+tls_validate_require_cipher child ppppp ended: status=0x0
 configuration file is TESTSUITE/test-config
 trusted user
 admin user
+DSN: r0 propagating DSN
 DSN: r1 propagating DSN
 DSN: r2 propagating DSN
 seeking password data for user "CALLER": using cached result
@@ -19,6 +21,12 @@ Testing test.ex@test.ex
 Considering test.ex@test.ex
 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 routing test.ex@test.ex
+--------> r0 router <--------
+local_part=test.ex domain=test.ex
+checking senders
+address match test: subject=CALLER@myhost.test.ex pattern=a@shorthost.test.ex
+CALLER@myhost.test.ex in "a@shorthost.test.ex"? no (end of list)
+r0 router skipped: senders mismatch
 --------> r1 router <--------
 local_part=test.ex domain=test.ex
 checking domains
@@ -62,6 +70,12 @@ Testing unknown@test.ex
 Considering unknown@test.ex
 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 routing unknown@test.ex
+--------> r0 router <--------
+local_part=unknown domain=test.ex
+checking senders
+address match test: subject=CALLER@myhost.test.ex pattern=a@shorthost.test.ex
+CALLER@myhost.test.ex in "a@shorthost.test.ex"? no (end of list)
+r0 router skipped: senders mismatch
 --------> r1 router <--------
 local_part=unknown domain=test.ex
 checking domains
@@ -132,3 +146,84 @@ routed by r2 router
   transport: local_delivery
 search_tidyup called
 >>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>>
+Exim version x.yz ....
+configuration file is TESTSUITE/test-config
+trusted user
+admin user
+search_tidyup called
+search_tidyup called
+search_open: dnsdb "NULL"
+search_find: file="NULL"
+  key="a=shorthost.test.ex" partial=-1 affix=NULL starflags=0
+LRU list:
+internal_search_find: file="NULL"
+  type=dnsdb key="a=shorthost.test.ex"
+database lookup required for a=shorthost.test.ex
+dnsdb key: shorthost.test.ex
+lookup yielded: 127.0.0.1
+search_open: dnsdb "NULL"
+  cached open
+search_find: file="NULL"
+  key="a=shorthost.test.ex" partial=-1 affix=NULL starflags=0
+LRU list:
+internal_search_find: file="NULL"
+  type=dnsdb key="a=shorthost.test.ex"
+cached data used for lookup of a=shorthost.test.ex
+lookup yielded: 127.0.0.1
+search_open: dnsdb "NULL"
+  cached open
+search_find: file="NULL"
+  key="a=shorthost.test.ex" partial=-1 affix=NULL starflags=0
+LRU list:
+internal_search_find: file="NULL"
+  type=dnsdb key="a=shorthost.test.ex"
+cached data found but past valid time; database lookup required for a=shorthost.test.ex
+dnsdb key: shorthost.test.ex
+lookup yielded: 127.0.0.1
+LOG: MAIN
+  <= a@shorthost.test.ex U=CALLER P=local S=sss
+created log directory TESTSUITE/spool/log
+search_tidyup called
+>>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>>
+
+******** SERVER ********
+Exim version x.yz ....
+configuration file is TESTSUITE/test-config
+trusted user
+admin user
+ppppp daemon_smtp_port overridden by -oX:
+ppppp   <: 1225
+ppppp listening on all interfaces (IPv6) port 1225
+ppppp listening on all interfaces (IPv4) port 1225
+ppppp pid written to TESTSUITE/spool/exim-daemon.pid
+ppppp LOG: MAIN
+ppppp   exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
+ppppp daemon running with uid=EXIM_UID gid=EXIM_GID euid=EXIM_UID egid=EXIM_GID
+ppppp Listening...
+ppppp Connection request from 127.0.0.1 port sssss
+ppppp 1 SMTP accept process running
+ppppp Listening...
+ppppp Process ppppp is handling incoming connection from [127.0.0.1]
+ppppp Process ppppp is ready for new message
+ppppp DNS list check: rbl.test.ex/V4NET.11.12.14
+ppppp new DNS lookup for 14.12.11.V4NET.rbl.test.ex
+ppppp DNS lookup for 14.12.11.V4NET.rbl.test.ex succeeded (yielding 127.0.0.2)
+ppppp => that means V4NET.11.12.14 is listed at rbl.test.ex
+ppppp DNS list check: rbl.test.ex/V4NET.11.12.14
+ppppp using result of previous DNS lookup
+ppppp DNS lookup for 14.12.11.V4NET.rbl.test.ex succeeded (yielding 127.0.0.2)
+ppppp => that means V4NET.11.12.14 is listed at rbl.test.ex
+ppppp DNS list check: rbl.test.ex/V4NET.11.12.14
+ppppp cached data found but past valid time; new DNS lookup for 14.12.11.V4NET.rbl.test.ex
+ppppp DNS lookup for 14.12.11.V4NET.rbl.test.ex succeeded (yielding 127.0.0.2)
+ppppp => that means V4NET.11.12.14 is listed at rbl.test.ex
+PDKIM >> Hashed body data, canonicalized >>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+ppppp LOG: MAIN
+ppppp   <= a@shorthost.test.ex H=localhost (myhost.test.ex) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@myhost.test.ex
+ppppp Process ppppp is ready for new message
+ppppp LOG: smtp_connection MAIN
+ppppp   SMTP connection from localhost (myhost.test.ex) [127.0.0.1] closed by QUIT
+ppppp child ppppp ended: status=0x0
+ppppp   normal exit, 0
+ppppp 0 SMTP accept processes now running
+ppppp Listening...
diff --git a/test/stdout/2200 b/test/stdout/2200
index b77594803..71ff12017 100644
--- a/test/stdout/2200
+++ b/test/stdout/2200
@@ -51,3 +51,6 @@ ten-2.test.ex
 > 
 > delay1500                  ip4.ip4.ip4.ip4
 > 
+> a=localhost.test.ex        127.0.0.1
+> a=localhost.test.ex        127.0.0.1
+>