summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/doc-txt/NewStuff2
-rw-r--r--doc/doc-txt/experimental-spec.txt44
-rwxr-xr-xsrc/scripts/MakeLinks2
-rwxr-xr-xsrc/scripts/lookups-Makefile5
-rw-r--r--src/src/EDITME7
-rw-r--r--src/src/config.h.defaults1
-rw-r--r--src/src/drtables.c7
-rw-r--r--src/src/exim.c6
-rw-r--r--src/src/lookups/Makefile2
-rw-r--r--src/src/lookups/lmdb.c160
-rw-r--r--test/aux-fixed/2800.lmdb-mkdb-dump.py15
-rw-r--r--test/aux-fixed/2800.mdbbin0 -> 12288 bytes
-rw-r--r--test/aux-fixed/2800.mdb.src3
-rw-r--r--test/confs/28009
-rw-r--r--test/scripts/2800-lmdb/280011
-rw-r--r--test/scripts/2800-lmdb/REQUIRES1
-rw-r--r--test/stdout/28009
17 files changed, 283 insertions, 1 deletions
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index 719bedf31..cf1cf6d56 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -33,6 +33,8 @@ Version 4.88
chunking_advertise_hosts, and smtp transport option hosts_try_chunking
for control.
+ 8. LMDB lookup support, as Experimental. Patch supplied by Andrew Colin Kissa.
+
Version 4.87
------------
diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index 993b5b05c..2e72def27 100644
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -967,6 +967,50 @@ Rationale:
Note that non-RFC-documented field names and data types are used.
+LMDB Lookup support
+-------------------
+LMDB is an ultra-fast, ultra-compact, crash-proof key-value embedded data store.
+It is modeled loosely on the BerkeleyDB API. You shoul read about the feature
+set as well as operation modes at https://symas.com/products/lightning-memory-mapped-database/
+
+LMDB single key lookup support is provided by linking to the LMDB C library.
+The current implementation does not support writing to the LMDB database.
+
+Visit https://github.com/LMDB/lmdb to download the library or find it in your
+operating systems package repository.
+
+If building from source, this description assumes that headers will be in
+/usr/local/include, and that the libraries are in /usr/local/lib.
+
+1. In order to build exim with LMDB lookup support add or uncomment
+
+EXPERIMENTAL_LMDB=yes
+
+to your Local/Makefile. (Re-)build/install exim. exim -d should show
+Experimental_LMDB in the line "Support for:".
+
+EXPERIMENTAL_LMDB=yes
+LDFLAGS += -llmdb
+# CFLAGS += -I/usr/local/include
+# LDFLAGS += -L/usr/local/lib
+
+The first line sets the feature to include the correct code, and
+the second line says to link the LMDB libraries into the
+exim binary. The commented out lines should be uncommented if you
+built LMDB from source and installed in the default location.
+Adjust the paths if you installed them elsewhere, but you do not
+need to uncomment them if an rpm (or you) installed them in the
+package controlled locations (/usr/include and /usr/lib).
+
+2. Create your LMDB files, you can use the mdb_load utility which is
+part of the LMDB distribution our your favourite language bindings.
+
+3. Add the single key lookups to your exim.conf file, example lookups
+are below.
+
+${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}}
+${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}fail}
+${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}}
--------------------------------------------------------------
diff --git a/src/scripts/MakeLinks b/src/scripts/MakeLinks
index 7a5649ef8..86f748c15 100755
--- a/src/scripts/MakeLinks
+++ b/src/scripts/MakeLinks
@@ -31,7 +31,7 @@ mkdir lookups
cd lookups
# Makefile is generated
for f in README cdb.c dbmdb.c dnsdb.c dsearch.c ibase.c ldap.h ldap.c \
- lsearch.c mysql.c redis.c nis.c nisplus.c oracle.c passwd.c \
+ lmdb.c lsearch.c mysql.c redis.c nis.c nisplus.c oracle.c passwd.c \
pgsql.c spf.c sqlite.c testdb.c whoson.c \
lf_functions.h lf_check_file.c lf_quote.c lf_sqlperform.c
do
diff --git a/src/scripts/lookups-Makefile b/src/scripts/lookups-Makefile
index 4e69a9a77..db2d184a7 100755
--- a/src/scripts/lookups-Makefile
+++ b/src/scripts/lookups-Makefile
@@ -177,6 +177,11 @@ fi
OBJ="${OBJ} spf.o"
+if want_experimental LMDB
+then
+ OBJ="${OBJ} lmdb.o"
+fi
+
echo "MODS = $MODS"
echo "OBJ = $OBJ"
diff --git a/src/src/EDITME b/src/src/EDITME
index ac8c642bc..10e5ccacb 100644
--- a/src/src/EDITME
+++ b/src/src/EDITME
@@ -488,6 +488,13 @@ EXIM_MONITOR=eximon.bin
# Uncomment the following to include extra information in fail DSN message (bounces)
# EXPERIMENTAL_DSN_INFO=yes
+# Uncomment the following to add LMDB lookup support
+# You need to have LMDB installed on your system (https://github.com/LMDB/lmdb)
+# Depending on where it is installed you may have to edit the CFLAGS and LDFLAGS lines.
+# EXPERIMENTAL_LMDB=yes
+# CFLAGS += -I/usr/local/include
+# LDFLAGS += -llmdb
+
###############################################################################
# THESE ARE THINGS YOU MIGHT WANT TO SPECIFY #
###############################################################################
diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults
index 9d5f4c402..fe5878aaf 100644
--- a/src/src/config.h.defaults
+++ b/src/src/config.h.defaults
@@ -177,6 +177,7 @@ it's a default value. */
#define EXPERIMENTAL_DCC
#define EXPERIMENTAL_DSN_INFO
#define EXPERIMENTAL_DMARC
+#define EXPERIMENTAL_LMDB
#define EXPERIMENTAL_SPF
#define EXPERIMENTAL_SRS
diff --git a/src/src/drtables.c b/src/src/drtables.c
index c83012944..c807d86c7 100644
--- a/src/src/drtables.c
+++ b/src/src/drtables.c
@@ -500,6 +500,9 @@ extern lookup_module_info pgsql_lookup_module_info;
#if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2
extern lookup_module_info redis_lookup_module_info;
#endif
+#if defined(EXPERIMENTAL_LMDB)
+extern lookup_module_info lmdb_lookup_module_info;
+#endif
#if defined(EXPERIMENTAL_SPF)
extern lookup_module_info spf_lookup_module_info;
#endif
@@ -585,6 +588,10 @@ init_lookup_list(void)
addlookupmodule(NULL, &redis_lookup_module_info);
#endif
+#ifdef EXPERIMENTAL_LMDB
+ addlookupmodule(NULL, &lmdb_lookup_module_info);
+#endif
+
#ifdef EXPERIMENTAL_SPF
addlookupmodule(NULL, &spf_lookup_module_info);
#endif
diff --git a/src/src/exim.c b/src/src/exim.c
index 14e0b9d67..1ad76dea2 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -842,6 +842,9 @@ fprintf(f, "Support for:");
#ifdef SUPPORT_SOCKS
fprintf(f, " SOCKS");
#endif
+#ifdef EXPERIMENTAL_LMDB
+ fprintf(f, " Experimental_LMDB");
+#endif
#ifdef EXPERIMENTAL_SPF
fprintf(f, " Experimental_SPF");
#endif
@@ -887,6 +890,9 @@ fprintf(f, "Lookups (built-in):");
#if defined(LOOKUP_LDAP) && LOOKUP_LDAP!=2
fprintf(f, " ldap ldapdn ldapm");
#endif
+#ifdef EXPERIMENTAL_LMDB
+ fprintf(f, " lmdb");
+#endif
#if defined(LOOKUP_MYSQL) && LOOKUP_MYSQL!=2
fprintf(f, " mysql");
#endif
diff --git a/src/src/lookups/Makefile b/src/src/lookups/Makefile
index 6ba0cb169..7c9700690 100644
--- a/src/src/lookups/Makefile
+++ b/src/src/lookups/Makefile
@@ -34,6 +34,7 @@ dnsdb.o: $(PHDRS) dnsdb.c
dsearch.o: $(PHDRS) dsearch.c
ibase.o: $(PHDRS) ibase.c
ldap.o: $(PHDRS) ldap.c
+lmdb.o: $(PHDRS) lmdb.c
lsearch.o: $(PHDRS) lsearch.c
mysql.o: $(PHDRS) mysql.c
nis.o: $(PHDRS) nis.c
@@ -53,6 +54,7 @@ dnsdb.so: $(PHDRS) dnsdb.c
dsearch.so: $(PHDRS) dsearch.c
ibase.so: $(PHDRS) ibase.c
ldap.so: $(PHDRS) ldap.c
+lmdb.so: $(PHDRS) lmdb.c
lsearch.so: $(PHDRS) lsearch.c
mysql.so: $(PHDRS) mysql.c
nis.so: $(PHDRS) nis.c
diff --git a/src/src/lookups/lmdb.c b/src/src/lookups/lmdb.c
new file mode 100644
index 000000000..8b0ffd2dd
--- /dev/null
+++ b/src/src/lookups/lmdb.c
@@ -0,0 +1,160 @@
+/*************************************************
+* Exim - an Internet mail transport agent *
+*************************************************/
+
+/* Copyright (c) University of Cambridge 2016 */
+/* See the file NOTICE for conditions of use and distribution. */
+
+#include "../exim.h"
+
+#ifdef EXPERIMENTAL_LMDB
+
+#include <lmdb.h>
+
+typedef struct lmdbstrct
+{
+MDB_txn *txn;
+MDB_dbi db_dbi;
+} Lmdbstrct;
+
+
+/*************************************************
+* Open entry point *
+*************************************************/
+
+static void *
+lmdb_open(uschar * filename, uschar ** errmsg)
+{
+MDB_env * db_env = NULL;
+Lmdbstrct * lmdb_p;
+int ret, save_errno;
+const uschar * errstr;
+
+lmdb_p = store_get(sizeof(Lmdbstrct));
+lmdb_p->txn = NULL;
+
+if ((ret = mdb_env_create(&db_env)))
+ {
+ errstr = US"create environment";
+ goto bad;
+ }
+
+if ((ret = mdb_env_open(db_env, CS filename, MDB_NOSUBDIR|MDB_RDONLY, 0660)))
+ {
+ errstr = US"open environment";
+ goto bad;
+ }
+
+if ((ret = mdb_txn_begin(db_env, NULL, MDB_RDONLY, &lmdb_p->txn)))
+ {
+ errstr = US"start transaction";
+ goto bad;
+ }
+
+if ((ret = mdb_open(lmdb_p->txn, NULL, 0, &lmdb_p->db_dbi)))
+ {
+ errstr = US"open database";
+ goto bad;
+ }
+
+return lmdb_p;
+
+bad:
+ save_errno = errno;
+ if (lmdb_p->txn) mdb_txn_abort(lmdb_p->txn);
+ if (db_env) mdb_env_close(db_env);
+ *errmsg = string_sprintf("LMDB: Unable to %s: %s", errstr, mdb_strerror(ret));
+ errno = save_errno;
+ return NULL;
+}
+
+
+/*************************************************
+* Find entry point *
+*************************************************/
+
+static int
+lmdb_find(void * handle, uschar * filename,
+ const uschar * keystring, int length, uschar ** result, uschar ** errmsg,
+ uint * do_cache)
+{
+int ret;
+MDB_val dbkey, data;
+Lmdbstrct * lmdb_p = handle;
+
+dbkey.mv_data = CS keystring;
+dbkey.mv_size = length;
+
+DEBUG(D_lookup) debug_printf("LMDB: lookup key: %s\n", (char *)keystring);
+
+if ((ret = mdb_get(lmdb_p->txn, lmdb_p->db_dbi, &dbkey, &data)) == 0)
+ {
+ *result = string_copyn(US data.mv_data, data.mv_size);
+ DEBUG(D_lookup) debug_printf("LMDB: lookup result: %s\n", *result);
+ return OK;
+ }
+else if (ret == MDB_NOTFOUND)
+ {
+ *errmsg = US"LMDB: lookup, no data found";
+ DEBUG(D_lookup) debug_printf("%s\n", *errmsg);
+ return FAIL;
+ }
+else
+ {
+ *errmsg = string_sprintf("LMDB: lookup error: %s", mdb_strerror(ret));
+ DEBUG(D_lookup) debug_printf("%s\n", *errmsg);
+ return DEFER;
+ }
+}
+
+
+/*************************************************
+* Close entry point *
+*************************************************/
+
+static void
+lmdb_close(void * handle)
+{
+Lmdbstrct * lmdb_p = handle;
+MDB_env * db_env = mdb_txn_env(lmdb_p->txn);
+mdb_txn_abort(lmdb_p->txn);
+mdb_env_close(db_env);
+}
+
+
+/*************************************************
+* Version reporting entry point *
+*************************************************/
+
+#include "../version.h"
+
+void
+lmdb_version_report(FILE * f)
+{
+fprintf(f, "Library version: LMDB: Compile: %d.%d.%d\n",
+ MDB_VERSION_MAJOR, MDB_VERSION_MINOR, MDB_VERSION_PATCH);
+#ifdef DYNLOOKUP
+fprintf(f, " Exim version %s\n", EXIM_VERSION_STR);
+#endif
+}
+
+static lookup_info lmdb_lookup_info = {
+ US"lmdb", /* lookup name */
+ lookup_absfile, /* query-style lookup */
+ lmdb_open, /* open function */
+ NULL, /* no check function */
+ lmdb_find, /* find function */
+ lmdb_close, /* close function */
+ NULL, /* tidy function */
+ NULL, /* quoting function */
+ lmdb_version_report /* version reporting */
+};
+
+#ifdef DYNLOOKUP
+# define lmdb_lookup_module_info _lookup_module_info
+#endif /* DYNLOOKUP */
+
+static lookup_info *_lookup_list[] = { &lmdb_lookup_info };
+lookup_module_info lmdb_lookup_module_info = { LOOKUP_MODULE_INFO_MAGIC, _lookup_list, 1 };
+
+#endif /* EXPERIMENTAL_LMDB */
diff --git a/test/aux-fixed/2800.lmdb-mkdb-dump.py b/test/aux-fixed/2800.lmdb-mkdb-dump.py
new file mode 100644
index 000000000..3de6ba13b
--- /dev/null
+++ b/test/aux-fixed/2800.lmdb-mkdb-dump.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+import os
+import lmdb
+
+if os.path.exists('2800.mdb'):
+ os.unlink('2800.mdb')
+
+env = lmdb.open('2800.mdb', subdir=False);
+with env.begin(write=True) as txn:
+ txn.put('first', 'data for first')
+ txn.put('second', 'A=1 B=2')
+ txn.put('third', 'A1:B2:C3')
+ cursor = txn.cursor()
+ for key, value in cursor:
+ print key, "=>", value
diff --git a/test/aux-fixed/2800.mdb b/test/aux-fixed/2800.mdb
new file mode 100644
index 000000000..2002ee1b5
--- /dev/null
+++ b/test/aux-fixed/2800.mdb
Binary files differ
diff --git a/test/aux-fixed/2800.mdb.src b/test/aux-fixed/2800.mdb.src
new file mode 100644
index 000000000..957952720
--- /dev/null
+++ b/test/aux-fixed/2800.mdb.src
@@ -0,0 +1,3 @@
+first: data for first
+second: A=1 B=2
+third: A1:B2:C3
diff --git a/test/confs/2800 b/test/confs/2800
new file mode 100644
index 000000000..607fcf5c7
--- /dev/null
+++ b/test/confs/2800
@@ -0,0 +1,9 @@
+# Exim test configuration 2800
+
+.include DIR/aux-var/std_conf_prefix
+
+primary_hostname = myhost.test.ex
+
+# ----- Main settings -----
+
+# End
diff --git a/test/scripts/2800-lmdb/2800 b/test/scripts/2800-lmdb/2800
new file mode 100644
index 000000000..a074ae628
--- /dev/null
+++ b/test/scripts/2800-lmdb/2800
@@ -0,0 +1,11 @@
+# lmdb lookup
+exim -be
+${lookup{first}lmdb{DIR/aux-fixed/TESTNUM.mdb}}
+${lookup{first}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}
+${lookup{fail}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}{failure value}}
+${lookup{fail}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}
+${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}
+${extract{A}{${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}}}
+${extract{B}{${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}}}
+${extract{C}{${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}}{$value}fail}
+****
diff --git a/test/scripts/2800-lmdb/REQUIRES b/test/scripts/2800-lmdb/REQUIRES
new file mode 100644
index 000000000..c160c4640
--- /dev/null
+++ b/test/scripts/2800-lmdb/REQUIRES
@@ -0,0 +1 @@
+lookup lmdb
diff --git a/test/stdout/2800 b/test/stdout/2800
new file mode 100644
index 000000000..30e14b09d
--- /dev/null
+++ b/test/stdout/2800
@@ -0,0 +1,9 @@
+> data for first
+> data for first
+> failure value
+> Failed: "lookup" failed and "fail" requested
+> A=1 B=2
+> 1
+> 2
+> Failed: "extract" failed and "fail" requested
+>
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-11 23:38:55 +0000