DNS: time-limit cached returns, using TTL. Bug 1395

This can matter for fast-changing data such as DNSBLs.

26 files changed, 143 insertions, 102 deletions

diff --git a/src/src/dns.c b/src/src/dns.c
index a239bec30..f99b470b2 100644
--- a/src/src/dns.c
+++ b/src/src/dns.c
@@ -388,7 +388,8 @@ from the following bytes. */
 
 dnss->aptr += namelen;
 GETSHORT(dnss->srr.type, dnss->aptr); /* Record type */
-dnss->aptr += 6;                      /* Don't want class or TTL */
+dnss->aptr += 2;                      /* Don't want class */
+GETLONG(dnss->srr.ttl, dnss->aptr);   /* TTL */
 GETSHORT(dnss->srr.size, dnss->aptr); /* Size of data portion */
 dnss->srr.data = dnss->aptr;          /* The record's data follows */
 dnss->aptr += dnss->srr.size;         /* Advance to next RR */
diff --git a/src/src/lookupapi.h b/src/src/lookupapi.h
index cdd1c85bf..03de8f675 100644
--- a/src/src/lookupapi.h
+++ b/src/src/lookupapi.h
@@ -34,7 +34,7 @@ typedef struct lookup_info {
     int,                          /* length of key or query */
     uschar **,                    /* for returning answer */
     uschar **,                    /* for error message */
-    BOOL *);                      /* to request cache cleanup */
+    uint *);                      /* cache TTL, sconds */
   void (*close)(                  /* close function */
     void *);                      /* handle */
   void (*tidy)(void);             /* tidy function */
@@ -46,9 +46,10 @@ typedef struct lookup_info {
 } lookup_info;
 
 /* This magic number is used by the following lookup_module_info structure
-   for checking API compatibility. It's equivalent to the string"LMM2" */
-#define LOOKUP_MODULE_INFO_MAGIC 0x4c4d4d32
+   for checking API compatibility. It used to be equivalent to the string"LMM3" */
+#define LOOKUP_MODULE_INFO_MAGIC 0x4c4d4933
 /* Version 2 adds: version_report */
+/* Version 3 change: non/cache becomes TTL in seconds */
 
 typedef struct lookup_module_info {
   uint magic;
diff --git a/src/src/lookups/README b/src/src/lookups/README
index 98905dc5c..31fea6448 100644
--- a/src/src/lookups/README
+++ b/src/src/lookups/README
@@ -122,12 +122,15 @@ DEFER. The arguments are:
   uschar **errmsg     where to put an error message on failure;
                       this is initially set to "", and should be left
                       as that for a standard "entry not found" error
-  BOOL *do_cache      the lookup should set this to FALSE when it changes data.
-                      This is TRUE by default. When set to FALSE the cache tree
+  uint *do_cache      the lookup should set this to 0 when it changes data.
+                      This is MAXINT by default. When set to 0 the cache tree
                       of the current search handle will be cleaned and the
                       current result will NOT be cached. Currently the mysql
                       and pgsql lookups use this when UPDATE/INSERT queries are
                       executed.
+                      If set to a nonzero number of seconds, the cached value
+                      becomes unusable after this time. Currently the dnsdb
+                      lookup uses this to support the TTL value.
 
 Even though the key is zero-terminated, the length is passed because in the
 common case it has been computed already and is often needed.
diff --git a/src/src/lookups/cdb.c b/src/src/lookups/cdb.c
index ea017def1..ba925dc12 100644
--- a/src/src/lookups/cdb.c
+++ b/src/src/lookups/cdb.c
@@ -279,7 +279,7 @@ cdb_find(void *handle,
         int  key_len,
         uschar **result,
         uschar **errmsg,
-        BOOL *do_cache)
+        uint *do_cache)
 {
   struct cdb_state * cdbp = handle;
   uint32 item_key_len,
diff --git a/src/src/lookups/dbmdb.c b/src/src/lookups/dbmdb.c
index 03248e490..b8c42d596 100644
--- a/src/src/lookups/dbmdb.c
+++ b/src/src/lookups/dbmdb.c
@@ -87,7 +87,7 @@ the keylength in order to include the terminating zero. */
 
 static int
 dbmdb_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 EXIM_DB *d = (EXIM_DB *)handle;
 EXIM_DATUM key, data;
@@ -120,7 +120,7 @@ return FAIL;
 
 int
 static dbmnz_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 return dbmdb_find(handle, filename, keystring, length-1, result, errmsg,
   do_cache);
@@ -140,7 +140,7 @@ return dbmdb_find(handle, filename, keystring, length-1, result, errmsg,
 
 static int
 dbmjz_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 uschar *key_item, *key_buffer, *key_p;
 const uschar *key_elems = keystring;
diff --git a/src/src/lookups/dnsdb.c b/src/src/lookups/dnsdb.c
index e3de279e2..70e6c8c63 100644
--- a/src/src/lookups/dnsdb.c
+++ b/src/src/lookups/dnsdb.c
@@ -131,7 +131,7 @@ separator, as always, is colon. */
 
 static int
 dnsdb_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int rc;
 int size = 256;
@@ -388,6 +388,9 @@ while ((domain = string_nextinlist(&keystring, &sep, NULL, 0)))
       {
       if (rr->type != searchtype) continue;
 
+      if (*do_cache > rr->ttl)
+	*do_cache = rr->ttl;
+
       if (type == T_A || type == T_AAAA || type == T_ADDRESSES)
         {
         dns_address *da;
diff --git a/src/src/lookups/dsearch.c b/src/src/lookups/dsearch.c
index f8c592adb..9f7dd8da0 100644
--- a/src/src/lookups/dsearch.c
+++ b/src/src/lookups/dsearch.c
@@ -67,7 +67,7 @@ for us. */
 
 int
 static dsearch_find(void *handle, uschar *dirname, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 struct stat statbuf;
 int save_errno;
diff --git a/src/src/lookups/ibase.c b/src/src/lookups/ibase.c
index 23e1dea60..7fd53d011 100644
--- a/src/src/lookups/ibase.c
+++ b/src/src/lookups/ibase.c
@@ -451,7 +451,7 @@ deferred with a retryable error. */
 
 static int
 ibase_find(void *handle, uschar * filename, uschar * query, int length,
-           uschar ** result, uschar ** errmsg, BOOL *do_cache)
+           uschar ** result, uschar ** errmsg, uint *do_cache)
 {
     int sep = 0;
     uschar *server;
diff --git a/src/src/lookups/ldap.c b/src/src/lookups/ldap.c
index a56eff326..b870df147 100644
--- a/src/src/lookups/ldap.c
+++ b/src/src/lookups/ldap.c
@@ -1339,7 +1339,7 @@ The handle and filename arguments are not used. */
 
 static int
 eldap_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
@@ -1348,7 +1348,7 @@ return(control_ldap_search(ldap_url, SEARCH_LDAP_SINGLE, result, errmsg));
 
 static int
 eldapm_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
@@ -1357,7 +1357,7 @@ return(control_ldap_search(ldap_url, SEARCH_LDAP_MULTIPLE, result, errmsg));
 
 static int
 eldapdn_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
@@ -1366,7 +1366,7 @@ return(control_ldap_search(ldap_url, SEARCH_LDAP_DN, result, errmsg));
 
 int
 eldapauth_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
diff --git a/src/src/lookups/lf_functions.h b/src/src/lookups/lf_functions.h
index 73e93037e..d2487d362 100644
--- a/src/src/lookups/lf_functions.h
+++ b/src/src/lookups/lf_functions.h
@@ -12,7 +12,7 @@ extern int     lf_check_file(int, uschar *, int, int, uid_t *, gid_t *,
 extern uschar *lf_quote(uschar *, uschar *, int, uschar *, int *, int *);
 extern int     lf_sqlperform(const uschar *, const uschar *, const uschar *,
 		 const uschar *, uschar **,
-                 uschar **, BOOL *, int(*)(const uschar *, uschar *, uschar **,
-                 uschar **, BOOL *, BOOL *));
+                 uschar **, uint *, int(*)(const uschar *, uschar *, uschar **,
+                 uschar **, BOOL *, uint *));
 
 /* End of lf_functions.h */
diff --git a/src/src/lookups/lf_sqlperform.c b/src/src/lookups/lf_sqlperform.c
index 2d7f32684..6d4f7a798 100644
--- a/src/src/lookups/lf_sqlperform.c
+++ b/src/src/lookups/lf_sqlperform.c
@@ -27,7 +27,7 @@ Arguments:
   query          the query
   result         where to pass back the result
   errmsg         where to pass back an error message
-  do_cache       to be set FALSE if data is changed
+  do_cache       to be set zero if data is changed
   func           the lookup function to call
 
 Returns:         the return from the lookup function, or DEFER
@@ -36,8 +36,8 @@ Returns:         the return from the lookup function, or DEFER
 int
 lf_sqlperform(const uschar *name, const uschar *optionname,
   const uschar *optserverlist, const uschar *query,
-  uschar **result, uschar **errmsg, BOOL *do_cache,
-  int(*fn)(const uschar *, uschar *, uschar **, uschar **, BOOL *, BOOL *))
+  uschar **result, uschar **errmsg, uint *do_cache,
+  int(*fn)(const uschar *, uschar *, uschar **, uschar **, BOOL *, uint *))
 {
 int sep, rc;
 uschar *server;
diff --git a/src/src/lookups/lsearch.c b/src/src/lookups/lsearch.c
index 3883d4ba0..eb70a45fa 100644
--- a/src/src/lookups/lsearch.c
+++ b/src/src/lookups/lsearch.c
@@ -323,7 +323,7 @@ return FAIL;
 
 static int
 lsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 return internal_lsearch_find(handle, filename, keystring, length, result,
@@ -340,7 +340,7 @@ return internal_lsearch_find(handle, filename, keystring, length, result,
 
 static int
 wildlsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 return internal_lsearch_find(handle, filename, keystring, length, result,
@@ -357,7 +357,7 @@ return internal_lsearch_find(handle, filename, keystring, length, result,
 
 static int
 nwildlsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 return internal_lsearch_find(handle, filename, keystring, length, result,
@@ -375,7 +375,7 @@ return internal_lsearch_find(handle, filename, keystring, length, result,
 
 static int
 iplsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 if ((length == 1 && keystring[0] == '*') ||
diff --git a/src/src/lookups/mysql.c b/src/src/lookups/mysql.c
index 8dff86ad1..1ce8831e8 100644
--- a/src/src/lookups/mysql.c
+++ b/src/src/lookups/mysql.c
@@ -74,7 +74,7 @@ Arguments:
   resultptr    where to store the result
   errmsg       where to point an error message
   defer_break  TRUE if no more servers are to be tried after DEFER
-  do_cache     set false if data is changed
+  do_cache     set zero if data is changed
 
 The server string is of the form "host/dbname/user/password". The host can be
 host:port. This string is in a nextinlist temporary buffer, so can be
@@ -85,7 +85,7 @@ Returns:       OK, FAIL, or DEFER
 
 static int
 perform_mysql_search(const uschar *query, uschar *server, uschar **resultptr,
-  uschar **errmsg, BOOL *defer_break, BOOL *do_cache)
+  uschar **errmsg, BOOL *defer_break, uint *do_cache)
 {
 MYSQL *mysql_handle = NULL;        /* Keep compilers happy */
 MYSQL_RES *mysql_result = NULL;
@@ -225,7 +225,7 @@ can be detected by calling mysql_field_count(). If its result is zero, no data
 was expected (this is all explained clearly in the MySQL manual). In this case,
 we return the number of rows affected by the command. In this event, we do NOT
 want to cache the result; also the whole cache for the handle must be cleaned
-up. Setting do_cache FALSE requests this. */
+up. Setting do_cache zero requests this. */
 
 if ((mysql_result = mysql_use_result(mysql_handle)) == NULL)
   {
@@ -233,7 +233,7 @@ if ((mysql_result = mysql_use_result(mysql_handle)) == NULL)
     {
     DEBUG(D_lookup) debug_printf("MYSQL: query was not one that returns data\n");
     result = string_sprintf("%d", mysql_affected_rows(mysql_handle));
-    *do_cache = FALSE;
+    *do_cache = 0;
     goto MYSQL_EXIT;
     }
   *errmsg = string_sprintf("MYSQL: lookup result failed: %s\n",
@@ -341,7 +341,7 @@ shared with other SQL lookups. */
 
 static int
 mysql_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 return lf_sqlperform(US"MySQL", US"mysql_servers", mysql_servers, query,
   result, errmsg, do_cache, perform_mysql_search);
diff --git a/src/src/lookups/nis.c b/src/src/lookups/nis.c
index 7b012b14c..1faa884a1 100644
--- a/src/src/lookups/nis.c
+++ b/src/src/lookups/nis.c
@@ -42,7 +42,7 @@ code. */
 
 static int
 nis_find(void *handle, uschar *filename, uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int rc;
 uschar *nis_data;
@@ -68,7 +68,7 @@ return (rc == YPERR_KEY || rc == YPERR_MAP)? FAIL : DEFER;
 
 static int
 nis0_find(void *handle, uschar *filename, uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int rc;
 uschar *nis_data;
diff --git a/src/src/lookups/nisplus.c b/src/src/lookups/nisplus.c
index 8895ceeec..a4a7a2d5b 100644
--- a/src/src/lookups/nisplus.c
+++ b/src/src/lookups/nisplus.c
@@ -43,7 +43,7 @@ equals sign. */
 
 static int
 nisplus_find(void *handle, uschar *filename, uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int i;
 int ssize = 0;
diff --git a/src/src/lookups/oracle.c b/src/src/lookups/oracle.c
index 1f2520ac0..adb17b4da 100644
--- a/src/src/lookups/oracle.c
+++ b/src/src/lookups/oracle.c
@@ -517,7 +517,7 @@ deferred with a retryable error. */
 
 static int
 oracle_find(void *handle, uschar *filename, uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int sep = 0;
 uschar *server;
diff --git a/src/src/lookups/passwd.c b/src/src/lookups/passwd.c
index e726f3e8e..315677ffa 100644
--- a/src/src/lookups/passwd.c
+++ b/src/src/lookups/passwd.c
@@ -34,7 +34,7 @@ return (void *)(-1);     /* Just return something non-null */
 
 static int
 passwd_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 struct passwd *pw;
 
diff --git a/src/src/lookups/pgsql.c b/src/src/lookups/pgsql.c
index c86ac23ed..4be3d98f1 100644
--- a/src/src/lookups/pgsql.c
+++ b/src/src/lookups/pgsql.c
@@ -119,7 +119,7 @@ Returns:       OK, FAIL, or DEFER
 
 static int
 perform_pgsql_search(const uschar *query, uschar *server, uschar **resultptr,
-  uschar **errmsg, BOOL *defer_break, BOOL *do_cache)
+  uschar **errmsg, BOOL *defer_break, uint *do_cache)
 {
 PGconn *pg_conn = NULL;
 PGresult *pg_result = NULL;
@@ -290,10 +290,10 @@ else
     /* The command was successful but did not return any data since it was
      * not SELECT but either an INSERT, UPDATE or DELETE statement. Tell the
      * high level code to not cache this query, and clean the current cache for
-     * this handle by setting *do_cache FALSE. */
+     * this handle by setting *do_cache zero. */
     result = string_copy(US PQcmdTuples(pg_result));
     offset = Ustrlen(result);
-    *do_cache = FALSE;
+    *do_cache = 0;
     DEBUG(D_lookup) debug_printf("PGSQL: command does not return any data "
       "but was successful. Rows affected: %s\n", result);
 
@@ -399,7 +399,7 @@ shared with other SQL lookups. */
 
 static int
 pgsql_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 return lf_sqlperform(US"PostgreSQL", US"pgsql_servers", pgsql_servers, query,
   result, errmsg, do_cache, perform_pgsql_search);
diff --git a/src/src/lookups/redis.c b/src/src/lookups/redis.c
index ac4d0ec30..18cd3a0af 100644
--- a/src/src/lookups/redis.c
+++ b/src/src/lookups/redis.c
@@ -65,7 +65,7 @@ redis_tidy(void)
  */
 static int
 perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
-  uschar **errmsg, BOOL *defer_break, BOOL *do_cache)
+  uschar **errmsg, BOOL *defer_break, uint *do_cache)
 {
        redisContext *redis_handle = NULL;        /* Keep compilers happy */
        redisReply *redis_reply = NULL;
@@ -197,7 +197,7 @@ perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
        case REDIS_REPLY_ERROR:
                *errmsg = string_sprintf("REDIS: lookup result failed: %s\n", redis_reply->str);
                *defer_break = FALSE;
-               *do_cache = FALSE;
+               *do_cache = 0;
                goto REDIS_EXIT;
                /* NOTREACHED */
 
@@ -205,7 +205,7 @@ perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
        case REDIS_REPLY_NIL:
                DEBUG(D_lookup) debug_printf("REDIS: query was not one that returned any data\n");
                result = string_sprintf("");
-               *do_cache = FALSE;
+               *do_cache = 0;
                goto REDIS_EXIT;
                /* NOTREACHED */
 
@@ -304,7 +304,7 @@ perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
 
 static int
 redis_find(void *handle __attribute__((unused)), uschar *filename __attribute__((unused)),
-           uschar *command, int length, uschar **result, uschar **errmsg, BOOL *do_cache)
+           uschar *command, int length, uschar **result, uschar **errmsg, uint *do_cache)
 {
        return lf_sqlperform(US"Redis", US"redis_servers", redis_servers, command,
          result, errmsg, do_cache, perform_redis_search);
diff --git a/src/src/lookups/spf.c b/src/src/lookups/spf.c
index 23ad2addd..2671fc9c4 100644
--- a/src/src/lookups/spf.c
+++ b/src/src/lookups/spf.c
@@ -31,7 +31,9 @@ static void dummy(int x) { dummy2(x-1); }
 #include <spf2/spf_dns_resolv.h>
 #include <spf2/spf_dns_cache.h>
 
-static void *spf_open(uschar *filename, uschar **errmsg) {
+static void *
+spf_open(uschar *filename, uschar **errmsg)
+{
   SPF_server_t *spf_server = NULL;
   spf_server = SPF_server_new(SPF_DNS_CACHE, 0);
   if (spf_server == NULL) {
@@ -41,13 +43,17 @@ static void *spf_open(uschar *filename, uschar **errmsg) {
   return (void *) spf_server;
 }
 
-static void spf_close(void *handle) {
+static void
+spf_close(void *handle)
+{
   SPF_server_t *spf_server = handle;
   if (spf_server) SPF_server_free(spf_server);
 }
 
-static int spf_find(void *handle, uschar *filename, uschar *keystring, int key_len,
-             uschar **result, uschar **errmsg, BOOL *do_cache) {
+static int
+spf_find(void *handle, uschar *filename, uschar *keystring, int key_len,
+             uschar **result, uschar **errmsg, uint *do_cache)
+{
   SPF_server_t *spf_server = handle;
   SPF_request_t *spf_request = NULL;
   SPF_response_t *spf_response = NULL;
diff --git a/src/src/lookups/sqlite.c b/src/src/lookups/sqlite.c
index bb92c8c18..e2330f920 100644
--- a/src/src/lookups/sqlite.c
+++ b/src/src/lookups/sqlite.c
@@ -81,7 +81,7 @@ return 0;
 
 static int
 sqlite_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int ret;
 struct strbuf res = { NULL, 0, 0 };
@@ -93,7 +93,7 @@ if (ret != SQLITE_OK)
   return FAIL;
   }
 
-if (res.string == NULL) *do_cache = FALSE;
+if (res.string == NULL) *do_cache = 0;
 
 *result = res.string;
 return OK;
diff --git a/src/src/lookups/testdb.c b/src/src/lookups/testdb.c
index c82fa7f3e..401f7c8bf 100644
--- a/src/src/lookups/testdb.c
+++ b/src/src/lookups/testdb.c
@@ -38,7 +38,7 @@ return (void *)(1);    /* Just return something non-null */
 
 static int
 testdb_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 handle = handle;          /* Keep picky compilers happy */
 filename = filename;
@@ -57,7 +57,7 @@ if (Ustrcmp(query, "defer") == 0)
   return DEFER;
   }
 
-if (Ustrcmp(query, "nocache") == 0) *do_cache = FALSE;
+if (Ustrcmp(query, "nocache") == 0) *do_cache = 0;
 
 *result = string_copy(query);
 return OK;
diff --git a/src/src/lookups/whoson.c b/src/src/lookups/whoson.c
index 4166089bd..9ac5a3a43 100644
--- a/src/src/lookups/whoson.c
+++ b/src/src/lookups/whoson.c
@@ -36,7 +36,7 @@ return (void *)(1);    /* Just return something non-null */
 
 static int
 whoson_find(void *handle, uschar *filename, uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 uschar buffer[80];
 handle = handle;          /* Keep picky compilers happy */
diff --git a/src/src/search.c b/src/src/search.c
index a05529196..cd522dae8 100644
--- a/src/src/search.c
+++ b/src/src/search.c
@@ -466,6 +466,7 @@ internal_search_find(void *handle, uschar *filename, uschar *keystring)
 {
 tree_node *t = (tree_node *)handle;
 search_cache *c = (search_cache *)(t->data.ptr);
+expiring_data *e;
 uschar *data = NULL;
 int search_type = t->name[0] - '0';
 int old_pool = store_pool;
@@ -491,18 +492,27 @@ store_pool = POOL_SEARCH;
 /* Look up the data for the key, unless it is already in the cache for this
 file. No need to check c->item_cache for NULL, tree_search will do so. */
 
-if ((t = tree_search(c->item_cache, keystring)) == NULL)
+if (  (t = tree_search(c->item_cache, keystring))
+   && (!(e = t->data.ptr)->expiry || e->expiry > time(NULL))
+   )
+  { /* Data was in the cache already; set the pointer from the tree node */
+  data = e->ptr;
+  DEBUG(D_lookup) debug_printf("cached data used for lookup of %s%s%s\n",
+    keystring,
+    filename ? US"\n  in " : US"", filename ? filename : US"");
+  }
+else
   {
-  BOOL do_cache = TRUE;
+  uint do_cache = UINT_MAX;
   int keylength = Ustrlen(keystring);
 
   DEBUG(D_lookup)
     {
-    if (filename != NULL)
-      debug_printf("file lookup required for %s\n  in %s\n",
-        keystring, filename);
-    else
-      debug_printf("database lookup required for %s\n", keystring);
+    if (t) debug_printf("cached data found but past valid time; ");
+    debug_printf("%s lookup required for %s%s%s\n",
+      filename ? US"file" : US"database",
+      keystring,
+      filename ? US"\n  in " : US"", filename ? filename : US"");
     }
 
   /* Call the code for the different kinds of search. DEFER is handled
@@ -511,9 +521,7 @@ if ((t = tree_search(c->item_cache, keystring)) == NULL)
 
   if (lookup_list[search_type]->find(c->handle, filename, keystring, keylength,
       &data, &search_error_message, &do_cache) == DEFER)
-    {
     search_find_defer = TRUE;
-    }
 
   /* A record that has been found is now in data, which is either NULL
   or points to a bit of dynamic store. Cache the result of the lookup if
@@ -524,10 +532,22 @@ if ((t = tree_search(c->item_cache, keystring)) == NULL)
   else if (do_cache)
     {
     int len = keylength + 1;
-    t = store_get(sizeof(tree_node) + len);
-    memcpy(t->name, keystring, len);
-    t->data.ptr = data;
-    tree_insertnode(&c->item_cache, t);
+
+    if (t)	/* Previous, out-of-date cache entry.  Update with the */
+      { 	/* new result and forget the old one */
+      e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
+      e->ptr = data;
+      }
+    else
+      {
+      t = store_get(sizeof(tree_node) + len + sizeof(expiring_data));
+      e = (expiring_data *)((char *)t + sizeof(tree_node) + len);
+      e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
+      e->ptr = data;
+      memcpy(t->name, keystring, len);
+      t->data.ptr = e;
+      tree_insertnode(&c->item_cache, t);
+      }
     }
 
   /* If caching was disabled, empty the cache tree. We just set the cache
@@ -540,34 +560,19 @@ if ((t = tree_search(c->item_cache, keystring)) == NULL)
     }
   }
 
-/* Data was in the cache already; set the pointer from the tree node */
-
-else
-  {
-  data = US t->data.ptr;
-  DEBUG(D_lookup) debug_printf("cached data used for lookup of %s%s%s\n",
-    keystring,
-    (filename == NULL)? US"" : US"\n  in ",
-    (filename == NULL)? US"" : filename);
-  }
-
-/* Debug: output the answer */
-
 DEBUG(D_lookup)
   {
-  if (data == NULL)
-    {
-    if (search_find_defer) debug_printf("lookup deferred: %s\n",
-      search_error_message);
-    else debug_printf("lookup failed\n");
-    }
-  else debug_printf("lookup yielded: %s\n", data);
+  if (data)
+    debug_printf("lookup yielded: %s\n", data);
+  else if (search_find_defer)
+    debug_printf("lookup deferred: %s\n", search_error_message);
+  else debug_printf("lookup failed\n");
   }
 
 /* Return it in new dynamic store in the regular pool */
 
 store_pool = old_pool;
-return (data == NULL)? NULL : string_copy(data);
+return data ? string_copy(data) : NULL;
 }
 
 
diff --git a/src/src/structs.h b/src/src/structs.h
index db9e843ac..c36d08ca7 100644
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -662,6 +662,16 @@ typedef struct tree_node {
   uschar  name[1];                /* node name - variable length */
 } tree_node;
 
+/* Structure for holding time-limited data such as DNS returns.
+We use this rather than extending tree_node to avoid wasting
+space for most tree use (variables...) at the cost of complexity
+for the lookups cache */
+
+typedef struct expiring_data {
+  time_t expiry;		  /* if nonzero, data invalid after this time */
+  void   *ptr;			  /* pointer to data */
+} expiring_data;
+
 /* Structure for holding the handle and the cached last lookup for searches.
 This block is pointed to by the tree entry for the file. The file can get
 closed if too many are opened at once. There is a LRU chain for deciding which
@@ -681,6 +691,7 @@ uncompressed, but the data pointer is into the raw data. */
 typedef struct {
   uschar  name[DNS_MAXNAME];      /* domain name */
   int     type;                   /* record type */
+  unsigned short ttl;		  /* time-to-live, seconds */
   int     size;                   /* size of data */
   uschar *data;                   /* pointer to data */
 } dns_record;
diff --git a/src/src/verify.c b/src/src/verify.c
index 7992d58fc..dc9c58224 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -21,6 +21,7 @@ uschar ctbuffer[8192];
 /* Structure for caching DNSBL lookups */
 
 typedef struct dnsbl_cache_block {
+  time_t expiry;
   dns_address *rhs;
   uschar *text;
   int rc;
@@ -3584,21 +3585,37 @@ if (!string_format(query, sizeof(query), "%s.%s", prepend, domain))
 
 /* Look for this query in the cache. */
 
-t = tree_search(dnsbl_cache, query);
+if (  (t = tree_search(dnsbl_cache, query))
+   && (cb = t->data.ptr)->expiry > time(NULL)
+   )
+
+/* Previous lookup was cached */
+
+  {
+  HDEBUG(D_dnsbl) debug_printf("using result of previous DNS lookup\n");
+  }
 
 /* If not cached from a previous lookup, we must do a DNS lookup, and
 cache the result in permanent memory. */
 
-if (t == NULL)
+else
   {
+  uint ttl = UINT_MAX;
+
   store_pool = POOL_PERM;
 
-  /* Set up a tree entry to cache the lookup */
+  if (t)
+    {
+    HDEBUG(D_dnsbl) debug_printf("cached data found but past valid time; ");
+    }
 
-  t = store_get(sizeof(tree_node) + Ustrlen(query));
-  Ustrcpy(t->name, query);
-  t->data.ptr = cb = store_get(sizeof(dnsbl_cache_block));
-  (void)tree_insertnode(&dnsbl_cache, t);
+  else
+    {	/* Set up a tree entry to cache the lookup */
+    t = store_get(sizeof(tree_node) + Ustrlen(query));
+    Ustrcpy(t->name, query);
+    t->data.ptr = cb = store_get(sizeof(dnsbl_cache_block));
+    (void)tree_insertnode(&dnsbl_cache, t);
+    }
 
   /* Do the DNS loopup . */
 
@@ -3634,6 +3651,7 @@ if (t == NULL)
           *addrp = da;
           while (da->next != NULL) da = da->next;
           addrp = &(da->next);
+	  if (ttl > rr->ttl) ttl = rr->ttl;
           }
         }
       }
@@ -3645,17 +3663,10 @@ if (t == NULL)
     if (cb->rhs == NULL) cb->rc = DNS_NODATA;
     }
 
+  cb->expiry = time(NULL)+ttl;
   store_pool = old_pool;
   }
 
-/* Previous lookup was cached */
-
-else
-  {
-  HDEBUG(D_dnsbl) debug_printf("using result of previous DNS lookup\n");
-  cb = t->data.ptr;
-  }
-
 /* We now have the result of the DNS lookup, either newly done, or cached
 from a previous call. If the lookup succeeded, check against the address
 list if there is one. This may be a positive equality list (introduced by