From 7adc9ca07a9a870f92a14d16740abfecde0bdfa4 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 9 May 2020 21:04:49 +0100 Subject: Move the native SRS implementation from Experimental to mainline --- doc/doc-docbook/spec.xfpt | 117 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index f1940bb1e..748f81cc0 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -10597,6 +10597,14 @@ ${sort {${lookup dnsdb{>:,,mx=example.com}}} {<} {${listextract{1}{<,$item}}}} will sort an MX lookup into priority order. + +.new +.vitem &*${srs_encode&~{*&<&'secret'&>&*}{*&<&'return&~path'&>&*}{*&<&'original&~domain'&>&*}}*& +SRS encoding. See SECT &<>& for details. +.wen + + + .vitem &*${substr{*&<&'string1'&>&*}{*&<&'string2'&>&*}{*&<&'string3'&>&*}}*& .cindex "&%substr%& expansion item" .cindex "substring extraction" @@ -11659,6 +11667,13 @@ includes the case of letters, whereas for &%gti%& the comparison is case-independent. Case and collation order are defined per the system C locale. + +.new +.vitem &*inbound_srs&~{*&<&'local&~part'&>&*}{*&<&'secret'&>&*}*& +SRS decode. See SECT &<>& for details. +.wen + + .vitem &*inlist&~{*&<&'string1'&>&*}{*&<&'string2'&>&*}*& &&& &*inlisti&~{*&<&'string1'&>&*}{*&<&'string2'&>&*}*& .cindex "string" "comparison" @@ -41078,6 +41093,108 @@ The lookup will return the same result strings as can appear in +.section "SRS (Sender Rewriting Scheme)" SECTSRS +.cindex SRS "sender rewriting scheme" + +.new +SRS can be used to modify sender addresses when forwarding so that +SPF verification does not object to them. +It operates by encoding the original envelope sender in a new +sender local part and using a domain run by the forwarding site +as the new domain for the sender. Any DSN message should be returned +to this new sender at the forwarding site, which can extract the +original sender from the coded local part and forward the DSN to +the originator. + +This is a way of avoiding the breakage that SPF does to forwarding. +The constructed local-part will be longer than the original, +leading to possible problems with very long addresses. +The changing of the sender address also hinders the tracing of mail +problems. + +Exim can be built to include native SRS support. To do this +SUPPORT_SRS=yes must be defined in &_Local/Makefile_&. +If this has been done, the macros _HAVE_SRS and _HAVE_NATIVE_SRS +will be defined. +The support is limited to SRS0-encoding; SRS1 is not supported. + +.cindex SRS excoding +To encode an address use this expansion item: +.vlist +.vitem &*${srs_encode&~{*&<&'secret'&>&*}{*&<&'return&~path'&>&*}{*&<&'original&~domain'&>&*}}*& +.cindex "&%srs_encode%& expansion item" +.cindex SRS "expansion item" +The first argument should be a secret known and used by all systems +handling the recipient domain for the original message. +There is no need to periodically change this key; a timestamp is also +encoded. +The second argument should be given as the envelope sender address before this +encoding operation. +The third argument should be the recipient domain of the message when +it arrived at this system. +.endlist + +.cindex SRS decoding +To decode an address use this expansion condition: +.vlist +.vitem &*inbound_srs&~{*&<&'local&~part'&>&*}{*&<&'secret'&>&*}*& +The first argument should be the recipient local prt as is was received. +The second argument is the site secret. + +If the messages is not for an SRS-encoded recipient the condition will +return false. If it is, the condition will return true and the variable +&$srs_recipient$& will be set to the decoded (original) value. +.endlist + +Example usage: +.code + #macro + SRS_SECRET = + + #routers + + outbound: + driver = dnslookup + # if outbound, and forwarding has been done, use an alternate transport + domains = ! +my_domains + transport = ${if eq {$local_part@$domain} \ + {$original_local_part@$original_domain} \ + {remote_smtp} {remote_forwarded_smtp}} + + inbound_srs: + driver = redirect + senders = : + domains = +my_domains + # detect inbound bounces which are SRS'd, and decode them + condition = ${if inbound_srs {$local_part} {SRS_SECRET}} + data = $srs_recipient + + inbound_srs_failure: + driver = redirect + senders = : + domains = +my_domains + # detect inbound bounces which look SRS'd but are invalid + condition = ${if inbound_srs {$local_part} {}} + allow_fail + data = :fail: Invalid SRS recipient address + + #... further routers here + + + # transport; should look like the non-forward outbound + # one, plus the max_rcpt and return_path options + remote_forwarded_smtp: + driver = smtp + # modify the envelope from, for mails that we forward + max_rcpt = 1 + return_path = ${srs_encode {SRS_SECRET} {$return_path} {$original_domain}} +.endd + + +.wen + + + .section DMARC SECDMARC .cindex DMARC verification -- cgit v1.2.3 From 2983e1a616058c03b57f1ab32a691f8b8ff9764e Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 10 May 2020 12:08:50 +0100 Subject: Move the TLS resumption support from Experimental to mainline --- doc/doc-docbook/spec.xfpt | 134 ++++++++++++++++++++++++++++++++++++++ doc/doc-txt/NewStuff | 3 + doc/doc-txt/OptionLists.txt | 2 + doc/doc-txt/experimental-spec.txt | 59 ----------------- src/src/EDITME | 6 +- src/src/auths/gsasl_exim.c | 4 +- src/src/config.h.defaults | 2 +- src/src/configure.default | 13 +++- src/src/deliver.c | 4 +- src/src/exim.c | 6 +- src/src/expand.c | 4 +- src/src/globals.c | 2 +- src/src/globals.h | 4 +- src/src/macro_predef.c | 2 +- src/src/readconf.c | 2 +- src/src/receive.c | 2 +- src/src/smtp_in.c | 2 +- src/src/spool_in.c | 2 +- src/src/spool_out.c | 2 +- src/src/structs.h | 2 +- src/src/tls-gnu.c | 28 ++++---- src/src/tls-openssl.c | 30 ++++----- src/src/transports/smtp.c | 6 +- src/src/transports/smtp.h | 2 +- 24 files changed, 207 insertions(+), 116 deletions(-) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 748f81cc0..5f5538e64 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -13766,6 +13766,17 @@ the value of the Distinguished Name of the certificate is made available in the If certificate verification fails it may refer to a failing chain element which is not the leaf. + +.new +.vitem &$tls_in_resumption$& &&& + &$tls_out_resumption$& +.vindex &$tls_in_resumption$& +.vindex &$tls_out_resumption$& +.cindex TLS resumption +Observability for TLS session resumption. See &<>& for details. +.wen + + .vitem &$tls_in_sni$& .vindex "&$tls_in_sni$&" .vindex "&$tls_sni$&" @@ -18274,6 +18285,14 @@ preference order of the available ciphers. Details are given in sections &<>& and &<>&. +.new +.option tls_resumption_hosts main "host list&!!" unset +.cindex TLS resumption +This option controls which connections to offer the TLS resumption feature. +See &<>& for details. +.wen + + .option tls_try_verify_hosts main "host list&!!" unset .cindex "TLS" "client certificate verification" .cindex "certificate" "verification of client" @@ -25594,6 +25613,14 @@ is used in different ways by OpenSSL and GnuTLS (see sections ciphers is a preference order. +.new +.option tls_resumption_hosts smtp "host list&!!" unset +.cindex TLS resumption +This option controls which connections to use the TLS resumption feature. +See &<>& for details. +.wen + + .option tls_sni smtp string&!! unset .cindex "TLS" "Server Name Indication" @@ -29424,6 +29451,100 @@ Open-source PKI book, available online at .ecindex IIDencsmtp2 +.new +.section "TLS Resumption" "SECTresumption" +.cindex TLS resumption +TLS Session Resumption for TLS 1.2 and TLS 1.3 connections can be used (defined +in RFC 5077 for 1.2). The support for this requires GnuTLS 3.6.3 or OpenSSL 1.1.1 +(or later). + +Session resumption (this is the "stateless" variant) involves the server sending +a "session ticket" to the client on one connection, which can be stored by the +client and used for a later session. The ticket contains sufficient state for +the server to reconstruct the TLS session, avoiding some expensive crypto +calculation and one full packet roundtrip time. + +.ilist +Operational cost/benefit: + + The extra data being transmitted costs a minor amount, and the client has + extra costs in storing and retrieving the data. + + In the Exim/Gnutls implementation the extra cost on an initial connection + which is TLS1.2 over a loopback path is about 6ms on 2017-laptop class hardware. + The saved cost on a subsequent connection is about 4ms; three or more + connections become a net win. On longer network paths, two or more + connections will have an average lower startup time thanks to the one + saved packet roundtrip. TLS1.3 will save the crypto cpu costs but not any + packet roundtrips. + +.cindex "hints database" tls + Since a new hints DB is used on the TLS client, + the hints DB maintenance should be updated to additionally handle "tls". + +.next +Security aspects: + + The session ticket is encrypted, but is obviously an additional security + vulnarability surface. An attacker able to decrypt it would have access + all connections using the resumed session. + The session ticket encryption key is not committed to storage by the server + and is rotated regularly (OpenSSL: 1hr, and one previous key is used for + overlap; GnuTLS 6hr but does not specify any overlap). + Tickets have limited lifetime (2hr, and new ones issued after 1hr under + OpenSSL. GnuTLS 2hr, appears to not do overlap). + + There is a question-mark over the security of the Diffie-Helman parameters + used for session negotiation. + +.next +Observability: + + The &%log_selector%& "tls_resumption" appends an asterisk to the tls_cipher "X=" + element. + + The variables &$tls_in_resumption$& and &$tls_out_resumption$& + have bits 0-4 indicating respectively + support built, client requested ticket, client offered session, + server issued ticket, resume used. A suitable decode list is provided + in the builtin macro _RESUME_DECODE for in &%listextract%& expansions. + +.next: +Control: + +The &%tls_resumption_hosts%& main option specifies a hostlist for which +exim, operating as a server, will offer resumption to clients. +Current best practice is to not offer the feature to MUA connection. +Commonly this can be done like this: +.code +tls_resumption_hosts = ${if inlist {$received_port}{587:465} {:}{*}} +.endd +If the peer host matches the list after expansion then resumption +is offered and/or accepted. + +The &%tls_resumption_hosts% smtp transport option performs the +equivalent function for operation as a client. +If the peer host matches the list after expansion then resumption +is attempted (if a stored session is available) or the information +stored (if supplied by the peer). + + +.next +Issues: + + In a resumed session: +.ilist + The variables &$tls_{in,out}_cipher$& will have values different + to the original (under GnuTLS). +.next + The variables &$tls_{in,out}_ocsp$& will be "not requested" or "no response", + and the &%hosts_require_ocsp%& smtp trasnport option will fail. +. XXX need to do something with that hosts_require_ocsp +.endlist + +.endlist +.wen + .section DANE "SECDANE" .cindex DANE @@ -38043,6 +38164,7 @@ selection marked by asterisks: &`*tls_certificate_verified `& certificate verification status &`*tls_cipher `& TLS cipher suite on <= and => lines &` tls_peerdn `& TLS peer DN on <= and => lines +&` tls_resumption `& append * to cipher field &` tls_sni `& TLS SNI on <= lines &` unknown_in_list `& DNS lookup failed in list match @@ -38444,6 +38566,14 @@ connection, the cipher suite used is added to the log line, preceded by X=. connection, and a certificate is supplied by the remote host, the peer DN is added to the log line, preceded by DN=. .next +.cindex "log" "TLS resumption" +.cindex "TLS" "logging session resumption" +.new +&%tls_resumption%&: When a message is sent or received over an encrypted +connection and the TLS session resumed one used on a previous TCP connection, +an asterisk is appended to the X= cipher field in the log line. +.wen +.next .cindex "log" "TLS SNI" .cindex "TLS" "logging SNI" &%tls_sni%&: When a message is received over an encrypted connection, and @@ -38989,6 +39119,10 @@ for remote hosts .next &'ratelimit'&: the data for implementing the ratelimit ACL condition .next +.new +&'tls'&: TLS session resumption data +.wen +.next &'misc'&: other hints data .endlist diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 53d7b5c9a..f3bf3e8e5 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -15,6 +15,9 @@ Version 4.95 2. The native SRS support, previously experimental, is now supported. It is not built unless specified in the Local/Makefile. + 3. TLS resumption support, previously experimental, is now supported and + included in default builds. + Version 4.94 ------------ diff --git a/doc/doc-txt/OptionLists.txt b/doc/doc-txt/OptionLists.txt index f15ccd553..39827b6ee 100644 --- a/doc/doc-txt/OptionLists.txt +++ b/doc/doc-txt/OptionLists.txt @@ -602,6 +602,8 @@ tls_privatekey string* unset main tls_remember_emstp boolean false main 4.21 tls_require_ciphers string* unset smtp 4.00 replaces tls_verify_ciphers string* unset main 4.33 +tls_resumption_hosts host list* unset main 4.95 + host list* unset smtp 4.95 tls_sni string* unset main 4.80 tls_tempfail_tryclear boolean true smtp 4.05 tls_try_verify_hosts host list unset main 4.00 diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 2b6d01f33..599d2bb1b 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -684,65 +684,6 @@ used via the transport in question. - -TLS Session Resumption ----------------------- -TLS Session Resumption for TLS 1.2 and TLS 1.3 connections can be used (defined -in RFC 5077 for 1.2). The support for this can be included by building with -EXPERIMENTAL_TLS_RESUME defined. This requires GnuTLS 3.6.3 or OpenSSL 1.1.1 -(or later). - -Session resumption (this is the "stateless" variant) involves the server sending -a "session ticket" to the client on one connection, which can be stored by the -client and used for a later session. The ticket contains sufficient state for -the server to reconstruct the TLS session, avoiding some expensive crypto -calculation and one full packet roundtrip time. - -Operational cost/benefit: - The extra data being transmitted costs a minor amount, and the client has - extra costs in storing and retrieving the data. - - In the Exim/Gnutls implementation the extra cost on an initial connection - which is TLS1.2 over a loopback path is about 6ms on 2017-laptop class hardware. - The saved cost on a subsequent connection is about 4ms; three or more - connections become a net win. On longer network paths, two or more - connections will have an average lower startup time thanks to the one - saved packet roundtrip. TLS1.3 will save the crypto cpu costs but not any - packet roundtrips. - - Since a new hints DB is used, the hints DB maintenance should be updated - to additionally handle "tls". - -Security aspects: - The session ticket is encrypted, but is obviously an additional security - vulnarability surface. An attacker able to decrypt it would have access - all connections using the resumed session. - The session ticket encryption key is not committed to storage by the server - and is rotated regularly (OpenSSL: 1hr, and one previous key is used for - overlap; GnuTLS 6hr but does not specify any overlap). - Tickets have limited lifetime (2hr, and new ones issued after 1hr under - OpenSSL. GnuTLS 2hr, appears to not do overlap). - - There is a question-mark over the security of the Diffie-Helman parameters - used for session negotiation. TBD. q-value; cf bug 1895 - -Observability: - New log_selector "tls_resumption", appends an asterisk to the tls_cipher "X=" - element. - - Variables $tls_{in,out}_resumption have bits 0-4 indicating respectively - support built, client requested ticket, client offered session, - server issued ticket, resume used. A suitable decode list is provided - in the builtin macro _RESUME_DECODE for ${listextract {}{}}. - -Issues: - In a resumed session: - $tls_{in,out}_cipher will have values different to the original (under GnuTLS) - $tls_{in,out}_ocsp will be "not requested" or "no response", and - hosts_require_ocsp will fail - - - Dovecot authenticator via inet socket ------------------------------------ If Dovecot is configured similar to :- diff --git a/src/src/EDITME b/src/src/EDITME index cf671afd1..e198a9cc9 100644 --- a/src/src/EDITME +++ b/src/src/EDITME @@ -277,6 +277,9 @@ SPOOL_DIRECTORY=/var/spool/exim # specified in INCLUDE. +# Uncomment the following line to remove support for TLS Resumption +# DISABLE_TLS_RESUME=yes + ############################################################################### # THESE ARE THINGS YOU PROBABLY WANT TO SPECIFY # @@ -633,9 +636,6 @@ DISABLE_MAL_MKS=yes # Uncomment the following line to add queuefile transport support # EXPERIMENTAL_QUEUEFILE=yes -# Uncomment the following line to include support for TLS Resumption -# EXPERIMENTAL_TLS_RESUME=yes - ############################################################################### # THESE ARE THINGS YOU MIGHT WANT TO SPECIFY # ############################################################################### diff --git a/src/src/auths/gsasl_exim.c b/src/src/auths/gsasl_exim.c index a3aaf1fa6..708957f04 100644 --- a/src/src/auths/gsasl_exim.c +++ b/src/src/auths/gsasl_exim.c @@ -365,7 +365,7 @@ HDEBUG(D_auth) #ifndef DISABLE_TLS if (tls_in.channelbinding && ob->server_channelbinding) { -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME if (!tls_in.ext_master_secret && tls_in.resumption == RESUME_USED) { /* per RFC 7677 section 4 */ HDEBUG(D_auth) debug_printf( @@ -814,7 +814,7 @@ HDEBUG(D_auth) #ifndef DISABLE_TLS if (tls_out.channelbinding && ob->client_channelbinding) { -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME if (!tls_out.ext_master_secret && tls_out.resumption == RESUME_USED) { /* per RFC 7677 section 4 */ string_format(buffer, buffsize, "%s", diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults index a91c3c3aa..1a1091f9c 100644 --- a/src/src/config.h.defaults +++ b/src/src/config.h.defaults @@ -54,6 +54,7 @@ Do not put spaces between # and the 'define'. #define DISABLE_PRDR #define DISABLE_QUEUE_RAMP #define DISABLE_TLS +#define DISABLE_TLS_RESUME #define DISABLE_D_OPTION #define ENABLE_DISABLE_FSYNC @@ -206,7 +207,6 @@ Do not put spaces between # and the 'define'. #define EXPERIMENTAL_LMDB #define EXPERIMENTAL_QUEUEFILE #define EXPERIMENTAL_SRS_ALT -#define EXPERIMENTAL_TLS_RESUME /* For developers */ diff --git a/src/src/configure.default b/src/src/configure.default index b758c8950..57af99c14 100644 --- a/src/src/configure.default +++ b/src/src/configure.default @@ -169,7 +169,14 @@ acl_smtp_data = acl_check_data # tls_privatekey = /etc/ssl/exim.pem # For OpenSSL, prefer EC- over RSA-authenticated ciphers -# tls_require_ciphers = ECDSA:RSA:!COMPLEMENTOFDEFAULT +.ifdef _HAVE_OPENSSL +tls_require_ciphers = ECDSA:RSA:!COMPLEMENTOFDEFAULT +.endif + +# Don't offer resumption to (most) MUAs, who we don't want to reuse +# tickets. Once the TLS extension for vended ticket numbers comes +# though, re-examine since resumption on a single-use ticket is still a benefit. +tls_resumption_hosts = ${if inlist {$received_port}{587:465} {:}{*}} # In order to support roaming users who wish to send email from anywhere, # you may want to make Exim listen on other ports as well as port 25, in @@ -808,6 +815,9 @@ begin transports remote_smtp: driver = smtp message_size_limit = ${if > {$max_received_linelength}{998} {1}{0}} +.ifdef _HAVE_TLS + tls_resumption_hosts = * +#endif .ifdef _HAVE_PRDR hosts_try_prdr = * .endif @@ -848,6 +858,7 @@ smarthost_smtp: .ifdef _HAVE_GNUTLS tls_require_ciphers = SECURE192:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1 .endif + tls_resumption_hosts = * .endif .ifdef _HAVE_PRDR hosts_try_prdr = * diff --git a/src/src/deliver.c b/src/src/deliver.c index b681584d5..3dcd7f949 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -817,7 +817,7 @@ d_tlslog(gstring * g, address_item * addr) if (LOGGING(tls_cipher) && addr->cipher) { g = string_append(g, 2, US" X=", addr->cipher); -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME if (LOGGING(tls_resumption) && testflag(addr, af_tls_resume)) g = string_catn(g, US"*", 1); #endif @@ -4773,7 +4773,7 @@ all pipes, so I do not see a reason to use non-blocking IO here #ifdef SUPPORT_DANE if (tls_out.dane_verified) setflag(addr, af_dane_verified); #endif -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME if (tls_out.resumption & RESUME_USED) setflag(addr, af_tls_resume); # endif diff --git a/src/src/exim.c b/src/src/exim.c index ab2d673dd..f61fe208f 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -931,6 +931,9 @@ g = string_cat(NULL, US"Support for:"); #ifdef USE_OPENSSL g = string_cat(g, US" OpenSSL"); #endif +#ifndef DISABLE_TLS_RESUME + g = string_cat(g, US" TLS_resume"); +#endif #ifdef SUPPORT_TRANSLATE_IP_ADDRESS g = string_cat(g, US" translate_ip_address"); #endif @@ -1007,9 +1010,6 @@ g = string_cat(NULL, US"Support for:"); #if defined(EXPERIMENTAL_SRS_ALT) g = string_cat(g, US" Experimental_SRS"); #endif -#ifdef EXPERIMENTAL_TLS_RESUME - g = string_cat(g, US" Experimental_TLS_resume"); -#endif g = string_cat(g, US"\n"); g = string_cat(g, US"Lookups (built-in):"); diff --git a/src/src/expand.c b/src/src/expand.c index 4abde0af6..732e20f97 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -779,7 +779,7 @@ static var_entry var_table[] = { { "tls_in_ourcert", vtype_cert, &tls_in.ourcert }, { "tls_in_peercert", vtype_cert, &tls_in.peercert }, { "tls_in_peerdn", vtype_stringptr, &tls_in.peerdn }, -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME { "tls_in_resumption", vtype_int, &tls_in.resumption }, #endif #ifndef DISABLE_TLS @@ -797,7 +797,7 @@ static var_entry var_table[] = { { "tls_out_ourcert", vtype_cert, &tls_out.ourcert }, { "tls_out_peercert", vtype_cert, &tls_out.peercert }, { "tls_out_peerdn", vtype_stringptr, &tls_out.peerdn }, -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME { "tls_out_resumption", vtype_int, &tls_out.resumption }, #endif #ifndef DISABLE_TLS diff --git a/src/src/globals.c b/src/src/globals.c index 561054981..1a4e3c8ed 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -137,7 +137,7 @@ uschar *tls_ocsp_file = NULL; uschar *tls_privatekey = NULL; BOOL tls_remember_esmtp = FALSE; uschar *tls_require_ciphers = NULL; -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME uschar *tls_resumption_hosts = NULL; # endif uschar *tls_try_verify_hosts = NULL; diff --git a/src/src/globals.h b/src/src/globals.h index fffe74441..1bdf338a2 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -107,7 +107,7 @@ typedef struct { OCSP_FAILED, /* verify failed */ OCSP_VFIED /* verified */ } ocsp; /* Stapled OCSP status */ -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME unsigned resumption; /* Session resumption */ BOOL host_resumable:1; BOOL ticket_received:1; @@ -134,7 +134,7 @@ extern uschar *tls_ocsp_file; /* OCSP stapling proof file */ extern uschar *tls_privatekey; /* Private key file */ extern BOOL tls_remember_esmtp; /* For YAEB */ extern uschar *tls_require_ciphers; /* So some can be avoided */ -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME extern uschar *tls_resumption_hosts; /* TLS session resumption */ # endif extern uschar *tls_try_verify_hosts; /* Optional client verification */ diff --git a/src/src/macro_predef.c b/src/src/macro_predef.c index f6cfcb14c..aeb234af7 100644 --- a/src/src/macro_predef.c +++ b/src/src/macro_predef.c @@ -204,7 +204,7 @@ due to conflicts with other common macros. */ #ifdef EXPERIMENTAL_DSN_INFO builtin_macro_create(US"_HAVE_DSN_INFO"); #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME builtin_macro_create(US"_HAVE_TLS_RESUME"); #endif diff --git a/src/src/readconf.c b/src/src/readconf.c index 3921ccffb..7b474f51d 100644 --- a/src/src/readconf.c +++ b/src/src/readconf.c @@ -378,7 +378,7 @@ static optionlist optionlist_config[] = { { "tls_privatekey", opt_stringptr, {&tls_privatekey} }, { "tls_remember_esmtp", opt_bool, {&tls_remember_esmtp} }, { "tls_require_ciphers", opt_stringptr, {&tls_require_ciphers} }, -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME { "tls_resumption_hosts", opt_stringptr, {&tls_resumption_hosts} }, # endif { "tls_try_verify_hosts", opt_stringptr, {&tls_try_verify_hosts} }, diff --git a/src/src/receive.c b/src/src/receive.c index 0fbd35f82..df8719ec6 100644 --- a/src/src/receive.c +++ b/src/src/receive.c @@ -3994,7 +3994,7 @@ g = add_host_info_for_log(g); if (LOGGING(tls_cipher) && tls_in.cipher) { g = string_append(g, 2, US" X=", tls_in.cipher); -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME if (LOGGING(tls_resumption) && tls_in.resumption & RESUME_USED) g = string_catn(g, US"*", 1); # endif diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index b3d1acbf9..412ef4df0 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -1801,7 +1801,7 @@ s_tlslog(gstring * g) if (LOGGING(tls_cipher) && tls_in.cipher) { g = string_append(g, 2, US" X=", tls_in.cipher); -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME if (LOGGING(tls_resumption) && tls_in.resumption & RESUME_USED) g = string_catn(g, US"*", 1); #endif diff --git a/src/src/spool_in.c b/src/src/spool_in.c index a0147d5ee..4b70780bc 100644 --- a/src/src/spool_in.c +++ b/src/src/spool_in.c @@ -683,7 +683,7 @@ for (;;) tls_in.sni = string_unprinting(string_copy_taint(q+4, tainted)); else if (Ustrncmp(q, "ocsp", 4) == 0) tls_in.ocsp = q[5] - '0'; -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME else if (Ustrncmp(q, "resumption", 10) == 0) tls_in.resumption = q[11] - 'A'; # endif diff --git a/src/src/spool_out.c b/src/src/spool_out.c index 539ad3d72..5d658fd74 100644 --- a/src/src/spool_out.c +++ b/src/src/spool_out.c @@ -261,7 +261,7 @@ if (tls_in.ourcert) fprintf(fp, "-tls_ourcert %s\n", CS big_buffer); } if (tls_in.ocsp) fprintf(fp, "-tls_ocsp %d\n", tls_in.ocsp); -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME fprintf(fp, "-tls_resumption %c\n", 'A' + tls_in.resumption); # endif if (tls_in.ver) spool_var_write(fp, US"tls_ver", tls_in.ver); diff --git a/src/src/structs.h b/src/src/structs.h index e1d989af2..9aab603f8 100644 --- a/src/src/structs.h +++ b/src/src/structs.h @@ -644,7 +644,7 @@ typedef struct address_item { #ifdef SUPPORT_I18N BOOL af_utf8_downcvt:1; /* downconvert was done for delivery */ #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME BOOL af_tls_resume:1; /* TLS used a resumed session */ #endif } flags; diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c index 24114f05e..c8017a733 100644 --- a/src/src/tls-gnu.c +++ b/src/src/tls-gnu.c @@ -111,7 +111,7 @@ require current GnuTLS, then we'll drop support for the ancient libraries). # endif #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME # if GNUTLS_VERSION_NUMBER < 0x030603 # error GNUTLS version too early for session-resumption # endif @@ -131,7 +131,7 @@ require current GnuTLS, then we'll drop support for the ancient libraries). void options_tls(void) { -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME builtin_macro_create_var(US"_RESUME_DECODE", RESUME_DECODE_STRING ); # endif # ifdef EXIM_HAVE_TLS1_3 @@ -266,7 +266,7 @@ static BOOL gnutls_buggy_ocsp = FALSE; static BOOL exim_testharness_disable_ocsp_validity_check = FALSE; #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME static gnutls_datum_t server_sessticket_key; #endif @@ -326,7 +326,7 @@ static void exim_gnutls_logger_cb(int level, const char *message); static int exim_sni_handling_cb(gnutls_session_t session); -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME static int tls_server_ticket_cb(gnutls_session_t sess, u_int htype, unsigned when, unsigned incoming, const gnutls_datum_t * msg); @@ -337,7 +337,7 @@ tls_server_ticket_cb(gnutls_session_t sess, u_int htype, unsigned when, void tls_daemon_init(void) { -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME /* We are dependent on the GnuTLS implementation of the Session Ticket encryption; both the strength and the key rotation period. We hope that the strength at least matches that of the ciphersuite (but GnuTLS does not @@ -1003,7 +1003,7 @@ So we need to spot the Certificate handshake message, parse it and spot any stat This is different to tls1.2 - where it is a separate record (wireshake term) / handshake message (gnutls term). */ -#if defined(EXPERIMENTAL_TLS_RESUME) || defined(SUPPORT_GNUTLS_EXT_RAW_PARSE) +#if !defined(DISABLE_TLS_RESUME) || defined(SUPPORT_GNUTLS_EXT_RAW_PARSE) /* Callback for certificate-status, on server. We sent stapled OCSP. */ static int tls_server_certstatus_cb(gnutls_session_t session, unsigned int htype, @@ -1035,7 +1035,7 @@ switch (htype) # endif case GNUTLS_HANDSHAKE_CERTIFICATE_STATUS: return tls_server_certstatus_cb(sess, htype, when, incoming, msg); -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME case GNUTLS_HANDSHAKE_NEW_SESSION_TICKET: return tls_server_ticket_cb(sess, htype, when, incoming, msg); # endif @@ -2328,7 +2328,7 @@ else } -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME static int tls_server_ticket_cb(gnutls_session_t sess, u_int htype, unsigned when, unsigned incoming, const gnutls_datum_t * msg) @@ -2442,7 +2442,7 @@ DEBUG(D_tls) debug_printf("initialising GnuTLS as a server\n"); #endif } -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_server_resume_prehandshake(state); #endif @@ -2550,7 +2550,7 @@ if (gnutls_session_get_flags(state->session) & GNUTLS_SFLAGS_EXT_MASTER_SECRET) tls_in.ext_master_secret = TRUE; #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_server_resume_posthandshake(state); #endif @@ -2683,7 +2683,7 @@ return TRUE; -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME /* On the client, get any stashed session for the given IP from hints db and apply it to the ssl-connection for attempted resumption. Although there is a gnutls_session_ticket_enable_client() interface it is @@ -2816,7 +2816,7 @@ if (gnutls_session_is_resumed(state->session)) tls_save_session(tlsp, state->session, host); } -#endif /* EXPERIMENTAL_TLS_RESUME */ +#endif /* !DISABLE_TLS_RESUME */ /************************************************* @@ -2970,7 +2970,7 @@ if (request_ocsp) } #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_client_resume_prehandshake(state, tlsp, host, ob); #endif @@ -3070,7 +3070,7 @@ if (request_ocsp) } #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_client_resume_posthandshake(state, tlsp, host); #endif diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c index 0caf1c0ff..673768a56 100644 --- a/src/src/tls-openssl.c +++ b/src/src/tls-openssl.c @@ -116,7 +116,7 @@ change this guard and punt the issue for a while longer. */ # define DISABLE_OCSP #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME # if OPENSSL_VERSION_NUMBER < 0x0101010L # error OpenSSL version too old for session-resumption # endif @@ -292,7 +292,7 @@ for (struct exim_openssl_option * o = exim_openssl_options; builtin_macro_create(buf); } -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME builtin_macro_create_var(US"_RESUME_DECODE", RESUME_DECODE_STRING ); # endif # ifdef SSL_OP_NO_TLSv1_3 @@ -422,7 +422,7 @@ static int tls_server_stapling_cb(SSL *s, void *arg); /* Daemon-called, before every connection, key create/rotate */ -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME static void tk_init(void); static int tls_exdata_idx = -1; #endif @@ -430,7 +430,7 @@ static int tls_exdata_idx = -1; void tls_daemon_init(void) { -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tk_init(); #endif return; @@ -891,7 +891,7 @@ fclose(fp); #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME /* Manage the keysets used for encrypting the session tickets, on the server. */ typedef struct { /* Session ticket encryption key */ @@ -2176,12 +2176,12 @@ availability of the option value macros from OpenSSL. */ if (!tls_openssl_options_parse(openssl_options, &init_options)) return tls_error(US"openssl_options parsing failed", host, NULL, errstr); -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tlsp->resumption = RESUME_SUPPORTED; #endif if (init_options) { -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME /* Should the server offer session resumption? */ if (!host && verify_check_host(&tls_resumption_hosts) == OK) { @@ -2685,12 +2685,12 @@ else if (verify_check_host(&tls_try_verify_hosts) == OK) server_verify_optional = TRUE; } -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME SSL_CTX_set_tlsext_ticket_key_cb(server_ctx, ticket_key_callback); /* despite working, appears to always return failure, so ignoring */ #endif #ifdef OPENSSL_HAVE_NUM_TICKETS -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME SSL_CTX_set_num_tickets(server_ctx, tls_in.host_resumable ? 1 : 0); # else SSL_CTX_set_num_tickets(server_ctx, 0); /* send no TLS1.3 stateful-tickets */ @@ -2796,7 +2796,7 @@ DEBUG(D_tls) debug_printf("SSL_accept was successful\n"); ERR_clear_error(); /* Even success can leave errors in the stack. Seen with anon-authentication ciphersuite negotiated. */ -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME if (SSL_session_reused(server_ssl)) { tls_in.resumption |= RESUME_USED; @@ -2983,7 +2983,7 @@ return DEFER; -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME /* On the client, get any stashed session for the given IP from hints db and apply it to the ssl-connection for attempted resumption. */ @@ -3145,7 +3145,7 @@ if (SSL_session_reused(exim_client_ctx->ssl)) tlsp->resumption |= RESUME_USED; } } -#endif /* EXPERIMENTAL_TLS_RESUME */ +#endif /* !DISABLE_TLS_RESUME */ /************************************************* @@ -3294,7 +3294,7 @@ else client_static_cbinfo, errstr) != OK) return FALSE; -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_client_ctx_resume_prehandshake(exim_client_ctx, tlsp, ob, host); #endif @@ -3365,7 +3365,7 @@ if (request_ocsp) } #endif -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME if (!tls_client_ssl_resume_prehandshake(exim_client_ctx->ssl, tlsp, host, errstr)) return FALSE; @@ -3406,7 +3406,7 @@ DEBUG(D_tls) #endif } -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_client_resume_posthandshake(exim_client_ctx, tlsp); #endif diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 8492a7f25..f47c6d92f 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -127,7 +127,7 @@ optionlist smtp_transport_options[] = { { "tls_dh_min_bits", opt_int, LOFF(tls_dh_min_bits) }, { "tls_privatekey", opt_stringptr, LOFF(tls_privatekey) }, { "tls_require_ciphers", opt_stringptr, LOFF(tls_require_ciphers) }, -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME { "tls_resumption_hosts", opt_stringptr, LOFF(tls_resumption_hosts) }, # endif { "tls_sni", opt_stringptr, LOFF(tls_sni) }, @@ -233,7 +233,7 @@ smtp_transport_options_block smtp_transport_option_defaults = { .tls_verify_certificates = US"system", .tls_dh_min_bits = EXIM_CLIENT_DH_DEFAULT_MIN_BITS, .tls_tempfail_tryclear = TRUE, -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME .tls_resumption_hosts = NULL, # endif .tls_verify_hosts = NULL, @@ -1970,7 +1970,7 @@ tls_out.peerdn = NULL; tls_out.sni = NULL; #endif tls_out.ocsp = OCSP_NOT_REQ; -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_out.resumption = 0; #endif tls_out.ver = NULL; diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h index 6e63a002d..037105a00 100644 --- a/src/src/transports/smtp.h +++ b/src/src/transports/smtp.h @@ -84,7 +84,7 @@ typedef struct { uschar *tls_crl; uschar *tls_privatekey; uschar *tls_require_ciphers; -# ifdef EXPERIMENTAL_TLS_RESUME +# ifndef DISABLE_TLS_RESUME uschar *tls_resumption_hosts; # endif uschar *tls_sni; -- cgit v1.2.3 From 7dd52d1ff8cd5e8914172d3131412bf1c2ceaf92 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 10 May 2020 15:57:39 +0100 Subject: Fix build with older GnuTLS Broken-by: 2983e1a616 --- doc/doc-docbook/spec.xfpt | 4 ++-- src/src/tls-gnu.c | 30 ++++++++++++++++-------------- test/scripts/5890-Resume-GnuTLS/REQUIRES | 2 +- test/scripts/5891-Resume-OpenSSL/REQUIRES | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 5f5538e64..be3e7c925 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -29462,7 +29462,7 @@ Session resumption (this is the "stateless" variant) involves the server sending a "session ticket" to the client on one connection, which can be stored by the client and used for a later session. The ticket contains sufficient state for the server to reconstruct the TLS session, avoiding some expensive crypto -calculation and one full packet roundtrip time. +calculation and (on TLS1.2) one full packet roundtrip time. .ilist Operational cost/benefit: @@ -29509,7 +29509,7 @@ Observability: server issued ticket, resume used. A suitable decode list is provided in the builtin macro _RESUME_DECODE for in &%listextract%& expansions. -.next: +.next Control: The &%tls_resumption_hosts%& main option specifies a hostlist for which diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c index c8017a733..a351c34d6 100644 --- a/src/src/tls-gnu.c +++ b/src/src/tls-gnu.c @@ -112,8 +112,10 @@ require current GnuTLS, then we'll drop support for the ancient libraries). #endif #ifndef DISABLE_TLS_RESUME -# if GNUTLS_VERSION_NUMBER < 0x030603 -# error GNUTLS version too early for session-resumption +# if GNUTLS_VERSION_NUMBER >= 0x030603 +# define EXIM_HAVE_TLS_RESUME +# else +# warning "GnuTLS library version too old; resumption unsupported" # endif #endif @@ -266,7 +268,7 @@ static BOOL gnutls_buggy_ocsp = FALSE; static BOOL exim_testharness_disable_ocsp_validity_check = FALSE; #endif -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME static gnutls_datum_t server_sessticket_key; #endif @@ -326,7 +328,7 @@ static void exim_gnutls_logger_cb(int level, const char *message); static int exim_sni_handling_cb(gnutls_session_t session); -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME static int tls_server_ticket_cb(gnutls_session_t sess, u_int htype, unsigned when, unsigned incoming, const gnutls_datum_t * msg); @@ -337,7 +339,7 @@ tls_server_ticket_cb(gnutls_session_t sess, u_int htype, unsigned when, void tls_daemon_init(void) { -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME /* We are dependent on the GnuTLS implementation of the Session Ticket encryption; both the strength and the key rotation period. We hope that the strength at least matches that of the ciphersuite (but GnuTLS does not @@ -1000,10 +1002,10 @@ return gnutls_ext_raw_parse(NULL, tls_server_servercerts_ext, msg, 0); "Handshake Protocol: Certificate" record. So we need to spot the Certificate handshake message, parse it and spot any status_request extension(s) -This is different to tls1.2 - where it is a separate record (wireshake term) / handshake message (gnutls term). +This is different to tls1.2 - where it is a separate record (wireshark term) / handshake message (gnutls term). */ -#if !defined(DISABLE_TLS_RESUME) || defined(SUPPORT_GNUTLS_EXT_RAW_PARSE) +#if defined(EXIM_HAVE_TLS_RESUME) || defined(SUPPORT_GNUTLS_EXT_RAW_PARSE) /* Callback for certificate-status, on server. We sent stapled OCSP. */ static int tls_server_certstatus_cb(gnutls_session_t session, unsigned int htype, @@ -1035,7 +1037,7 @@ switch (htype) # endif case GNUTLS_HANDSHAKE_CERTIFICATE_STATUS: return tls_server_certstatus_cb(sess, htype, when, incoming, msg); -# ifndef DISABLE_TLS_RESUME +# ifdef EXIM_HAVE_TLS_RESUME case GNUTLS_HANDSHAKE_NEW_SESSION_TICKET: return tls_server_ticket_cb(sess, htype, when, incoming, msg); # endif @@ -2328,7 +2330,7 @@ else } -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME static int tls_server_ticket_cb(gnutls_session_t sess, u_int htype, unsigned when, unsigned incoming, const gnutls_datum_t * msg) @@ -2442,7 +2444,7 @@ DEBUG(D_tls) debug_printf("initialising GnuTLS as a server\n"); #endif } -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME tls_server_resume_prehandshake(state); #endif @@ -2550,7 +2552,7 @@ if (gnutls_session_get_flags(state->session) & GNUTLS_SFLAGS_EXT_MASTER_SECRET) tls_in.ext_master_secret = TRUE; #endif -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME tls_server_resume_posthandshake(state); #endif @@ -2683,7 +2685,7 @@ return TRUE; -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME /* On the client, get any stashed session for the given IP from hints db and apply it to the ssl-connection for attempted resumption. Although there is a gnutls_session_ticket_enable_client() interface it is @@ -2970,7 +2972,7 @@ if (request_ocsp) } #endif -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME tls_client_resume_prehandshake(state, tlsp, host, ob); #endif @@ -3070,7 +3072,7 @@ if (request_ocsp) } #endif -#ifndef DISABLE_TLS_RESUME +#ifdef EXIM_HAVE_TLS_RESUME tls_client_resume_posthandshake(state, tlsp, host); #endif diff --git a/test/scripts/5890-Resume-GnuTLS/REQUIRES b/test/scripts/5890-Resume-GnuTLS/REQUIRES index 2f12f2742..7f34d4f81 100644 --- a/test/scripts/5890-Resume-GnuTLS/REQUIRES +++ b/test/scripts/5890-Resume-GnuTLS/REQUIRES @@ -1,3 +1,3 @@ support GnuTLS running IPv4 -support Experimental_TLS_resume +support TLS_resume diff --git a/test/scripts/5891-Resume-OpenSSL/REQUIRES b/test/scripts/5891-Resume-OpenSSL/REQUIRES index 027b4dcab..8e3a29f53 100644 --- a/test/scripts/5891-Resume-OpenSSL/REQUIRES +++ b/test/scripts/5891-Resume-OpenSSL/REQUIRES @@ -1,3 +1,3 @@ support OpenSSL running IPv4 -support Experimental_TLS_resume +support TLS_resume -- cgit v1.2.3 From 4a90d4879978fe062e0539da0942c3f9176f5b9b Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 10 May 2020 19:00:24 +0100 Subject: Move the LMDB lokup support from Experimental to mainline --- doc/doc-docbook/spec.xfpt | 23 ++++++++++++++++++++ doc/doc-txt/NewStuff | 3 +++ doc/doc-txt/experimental-spec.txt | 46 --------------------------------------- src/src/EDITME | 12 ++++------ src/src/config.h.defaults | 2 +- src/src/drtables.c | 4 ++-- src/src/exim.c | 5 +---- src/src/lookups/lmdb.c | 4 ++-- src/src/macro_predef.c | 7 +++--- 9 files changed, 40 insertions(+), 66 deletions(-) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index be3e7c925..6dcb69d47 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -6858,6 +6858,29 @@ The final resulting element can be a simple JSON type or a JSON object or array; for the latter two a string-representation of the JSON is returned. For elements of type string, the returned value is de-quoted. + + +.new +.next +.cindex LMDB +.cindex lookup lmdb +.cindex database lmdb +&(lmdb)&: The given file is an LMDB database. +LMDB is a memory-mapped key-value store, +with API modeled loosely on that of BerkeleyDB. +See &url(https://symas.com/products/lightning-memory-mapped-database/) +for the feature set and operation modes. + +Exim provides read-only access via the LMDB C library. +The library can be obtained from &url(https://github.com/LMDB/lmdb) +or your operating system package repository. +To enable LMDB support in Exim set LOOKUP_LMDB=yes in &_Local/Makefile_&. + +You will need to separately create the LMDB database file, +possibly using the &"mdb_load"& utility. +.wen + + .next .cindex "linear search" .cindex "lookup" "lsearch" diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index f3bf3e8e5..253eae2b7 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -18,6 +18,9 @@ Version 4.95 3. TLS resumption support, previously experimental, is now supported and included in default builds. + 4. Single-key LMDB lookups, previously experimental, are now supported. + The support is not built unless specified in the Local/Makefile. + Version 4.94 ------------ diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 599d2bb1b..8c9a866a2 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -466,52 +466,6 @@ Rationale: Note that non-RFC-documented field names and data types are used. -LMDB Lookup support -------------------- -LMDB is an ultra-fast, ultra-compact, crash-proof key-value embedded data store. -It is modeled loosely on the BerkeleyDB API. You should read about the feature -set as well as operation modes at https://symas.com/products/lightning-memory-mapped-database/ - -LMDB single key lookup support is provided by linking to the LMDB C library. -The current implementation does not support writing to the LMDB database. - -Visit https://github.com/LMDB/lmdb to download the library or find it in your -operating systems package repository. - -If building from source, this description assumes that headers will be in -/usr/local/include, and that the libraries are in /usr/local/lib. - -1. In order to build exim with LMDB lookup support add or uncomment - -EXPERIMENTAL_LMDB=yes - -to your Local/Makefile. (Re-)build/install exim. exim -d should show -Experimental_LMDB in the line "Support for:". - -EXPERIMENTAL_LMDB=yes -LDFLAGS += -llmdb -# CFLAGS += -I/usr/local/include -# LDFLAGS += -L/usr/local/lib - -The first line sets the feature to include the correct code, and -the second line says to link the LMDB libraries into the -exim binary. The commented out lines should be uncommented if you -built LMDB from source and installed in the default location. -Adjust the paths if you installed them elsewhere, but you do not -need to uncomment them if an rpm (or you) installed them in the -package controlled locations (/usr/include and /usr/lib). - -2. Create your LMDB files, you can use the mdb_load utility which is -part of the LMDB distribution our your favourite language bindings. - -3. Add the single key lookups to your exim.conf file, example lookups -are below. - -${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}} -${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}fail} -${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}} - - Queuefile transport ------------------- Queuefile is a pseudo transport which does not perform final delivery. diff --git a/src/src/EDITME b/src/src/EDITME index e198a9cc9..d7be01cd6 100644 --- a/src/src/EDITME +++ b/src/src/EDITME @@ -415,6 +415,8 @@ LOOKUP_DNSDB=yes # LOOKUP_IBASE=yes # LOOKUP_JSON=yes # LOOKUP_LDAP=yes +# LOOKUP_LMDB=yes + # LOOKUP_MYSQL=yes # LOOKUP_MYSQL_PC=mariadb # LOOKUP_NIS=yes @@ -491,7 +493,8 @@ SUPPORT_DANE=yes # You do not need to use this for any lookup information added via pkg-config. # LOOKUP_INCLUDE=-I /usr/local/ldap/include -I /usr/local/mysql/include -I /usr/local/pgsql/include -# LOOKUP_LIBS=-L/usr/local/lib -lldap -llber -lmysqlclient -lpq -lgds -lsqlite3 +# LOOKUP_INCLUDE +=-I /usr/local/include +# LOOKUP_LIBS=-L/usr/local/lib -lldap -llber -lmysqlclient -lpq -lgds -lsqlite3 -llmdb #------------------------------------------------------------------------------ @@ -626,13 +629,6 @@ DISABLE_MAL_MKS=yes # Uncomment the following to include extra information in fail DSN message (bounces) # EXPERIMENTAL_DSN_INFO=yes -# Uncomment the following to add LMDB lookup support -# You need to have LMDB installed on your system (https://github.com/LMDB/lmdb) -# Depending on where it is installed you may have to edit the CFLAGS and LDFLAGS lines. -# EXPERIMENTAL_LMDB=yes -# CFLAGS += -I/usr/local/include -# LDFLAGS += -llmdb - # Uncomment the following line to add queuefile transport support # EXPERIMENTAL_QUEUEFILE=yes diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults index 1a1091f9c..07c0ecf81 100644 --- a/src/src/config.h.defaults +++ b/src/src/config.h.defaults @@ -99,6 +99,7 @@ Do not put spaces between # and the 'define'. #define LOOKUP_IBASE #define LOOKUP_JSON #define LOOKUP_LDAP +#define LOOKUP_LMDB #define LOOKUP_LSEARCH #define LOOKUP_MYSQL #define LOOKUP_NIS @@ -204,7 +205,6 @@ Do not put spaces between # and the 'define'. #define EXPERIMENTAL_BRIGHTMAIL #define EXPERIMENTAL_DCC #define EXPERIMENTAL_DSN_INFO -#define EXPERIMENTAL_LMDB #define EXPERIMENTAL_QUEUEFILE #define EXPERIMENTAL_SRS_ALT diff --git a/src/src/drtables.c b/src/src/drtables.c index 0ca369cfc..67a2b8f52 100644 --- a/src/src/drtables.c +++ b/src/src/drtables.c @@ -602,7 +602,7 @@ extern lookup_module_info pgsql_lookup_module_info; #if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2 extern lookup_module_info redis_lookup_module_info; #endif -#if defined(EXPERIMENTAL_LMDB) +#if defined(LOOKUP_LMDB) extern lookup_module_info lmdb_lookup_module_info; #endif #if defined(SUPPORT_SPF) @@ -698,7 +698,7 @@ addlookupmodule(NULL, &pgsql_lookup_module_info); addlookupmodule(NULL, &redis_lookup_module_info); #endif -#ifdef EXPERIMENTAL_LMDB +#ifdef LOOKUP_LMDB addlookupmodule(NULL, &lmdb_lookup_module_info); #endif diff --git a/src/src/exim.c b/src/src/exim.c index f61fe208f..da4341421 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -1001,9 +1001,6 @@ g = string_cat(NULL, US"Support for:"); #ifdef EXPERIMENTAL_DSN_INFO g = string_cat(g, US" Experimental_DSN_info"); #endif -#ifdef EXPERIMENTAL_LMDB - g = string_cat(g, US" Experimental_LMDB"); -#endif #ifdef EXPERIMENTAL_QUEUEFILE g = string_cat(g, US" Experimental_QUEUEFILE"); #endif @@ -1037,7 +1034,7 @@ g = string_cat(g, US"Lookups (built-in):"); #if defined(LOOKUP_LDAP) && LOOKUP_LDAP!=2 g = string_cat(g, US" ldap ldapdn ldapm"); #endif -#ifdef EXPERIMENTAL_LMDB +#ifdef LOOKUP_LMDB g = string_cat(g, US" lmdb"); #endif #if defined(LOOKUP_MYSQL) && LOOKUP_MYSQL!=2 diff --git a/src/src/lookups/lmdb.c b/src/src/lookups/lmdb.c index 406675198..d9cf25d54 100644 --- a/src/src/lookups/lmdb.c +++ b/src/src/lookups/lmdb.c @@ -8,7 +8,7 @@ #include "../exim.h" -#ifdef EXPERIMENTAL_LMDB +#ifdef LOOKUP_LMDB #include @@ -158,4 +158,4 @@ static lookup_info lmdb_lookup_info = { static lookup_info *_lookup_list[] = { &lmdb_lookup_info }; lookup_module_info lmdb_lookup_module_info = { LOOKUP_MODULE_INFO_MAGIC, _lookup_list, 1 }; -#endif /* EXPERIMENTAL_LMDB */ +#endif /* LOOKUP_LMDB */ diff --git a/src/src/macro_predef.c b/src/src/macro_predef.c index aeb234af7..9c3d34a96 100644 --- a/src/src/macro_predef.c +++ b/src/src/macro_predef.c @@ -180,9 +180,6 @@ due to conflicts with other common macros. */ #ifdef TCP_FASTOPEN builtin_macro_create(US"_HAVE_TCP_FASTOPEN"); #endif -#ifdef EXPERIMENTAL_LMDB - builtin_macro_create(US"_HAVE_LMDB"); -#endif #ifdef SUPPORT_SPF builtin_macro_create(US"_HAVE_SPF"); #endif @@ -226,6 +223,10 @@ due to conflicts with other common macros. */ #ifdef LOOKUP_IBASE builtin_macro_create(US"_HAVE_LOOKUP_IBASE"); #endif +#ifdef LOOKUP_LMDB + builtin_macro_create(US"_HAVE_LMDB"); + builtin_macro_create(US"_HAVE_LOOKUP_LMDB"); +#endif #ifdef LOOKUP_LDAP builtin_macro_create(US"_HAVE_LOOKUP_JSON"); #endif -- cgit v1.2.3 From 77afbf0a11c2a6e532ac860243e48414c294541b Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Tue, 12 May 2020 12:56:03 +0100 Subject: Docs: fast-ramp two-phase queue run --- doc/doc-docbook/spec.xfpt | 19 ++++++++++++++++++- doc/doc-txt/OptionLists.txt | 1 + doc/doc-txt/experimental-spec.txt | 26 +------------------------- 3 files changed, 20 insertions(+), 26 deletions(-) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 6dcb69d47..0c4df93a7 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -4518,6 +4518,7 @@ appear in the correct order. Each flag is described in a separate item below. .cindex "queue" "routing" .cindex "routing" "whole queue before delivery" .cindex "first pass routing" +.cindex "queue runner" "two phase" An option starting with &%-qq%& requests a two-stage queue run. In the first stage, the queue is scanned as if the &%queue_smtp_domains%& option matched every domain. Addresses are routed, local deliveries happen, but no remote @@ -4525,6 +4526,10 @@ transports are run. .new Performance will be best if the &%queue_run_in_order%& option is false. +If that is so and the &%queue_fast_ramp%& option is true then +in the first phase of the run, +once a threshold number of messages are routed for a given host, +a delivery process is forked in parallel with the rest of the scan. .wen .cindex "hints database" "remembering routing" @@ -14800,6 +14805,7 @@ See also the &'Policy controls'& section above. .row &%hold_domains%& "hold delivery for these domains" .row &%local_interfaces%& "for routing checks" .row &%queue_domains%& "no immediate delivery for these" +.row &%queue_fast ramp%& "parallel delivery with 2-phase queue run" .row &%queue_only%& "no immediate delivery at all" .row &%queue_only_file%& "no immediate delivery if file exists" .row &%queue_only_load%& "no immediate delivery if load is high" @@ -16976,6 +16982,17 @@ domains that do not match are processed. All other deliveries wait until the next queue run. See also &%hold_domains%& and &%queue_smtp_domains%&. +.new +.option queue_fast_ramp main boolean false +.cindex "queue runner" "two phase" +.cindex "queue" "double scanning" +If set to true, two-phase queue runs, initiated using &%-qq%& on the +command line, may start parallel delivery processes during their first +phase. This will be done when a threshold number of messages have been +routed for a single host. +.wen + + .option queue_list_requires_admin main boolean true .cindex "restricting access to features" .oindex "&%-bp%&" @@ -29545,7 +29562,7 @@ tls_resumption_hosts = ${if inlist {$received_port}{587:465} {:}{*}} If the peer host matches the list after expansion then resumption is offered and/or accepted. -The &%tls_resumption_hosts% smtp transport option performs the +The &%tls_resumption_hosts%& smtp transport option performs the equivalent function for operation as a client. If the peer host matches the list after expansion then resumption is attempted (if a stored session is available) or the information diff --git a/doc/doc-txt/OptionLists.txt b/doc/doc-txt/OptionLists.txt index 39827b6ee..5de6d6936 100644 --- a/doc/doc-txt/OptionLists.txt +++ b/doc/doc-txt/OptionLists.txt @@ -443,6 +443,7 @@ qualify_recipient string + main qualify_single boolean true dnslookup 4.00 query string* + iplookup 4.00 queue_domains domain list unset main 4.00 +queue_fasst_ramp boolean false main 4.95 queue_list_requires_admin boolean true main 1.95 queue_only boolean false main queue_only_file string unset main 2.05 diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 8c9a866a2..68366a4a9 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -639,7 +639,7 @@ used via the transport in question. Dovecot authenticator via inet socket ------------------------------------- +-------------------------------------------------------------- If Dovecot is configured similar to :- service auth { @@ -666,30 +666,6 @@ and a whitespace-separated port number must be given. -Twophase queue run fast ramp ----------------------------- -To include this feature, add to Local/Makefile: - EXPERIMENTAL_QUEUE_RAMP=yes - -If the (added for this feature) main-section option "queue_fast_ramp" (boolean) -is set, and a two-phase ("-qq") queue run finds, during the first phase, a -suitably large number of message routed for a given host - then (subject to -the usual queue-runner resource limits) delivery for that host is initiated -immediately, overlapping with the remainder of the first phase. - -This is incompatible with queue_run_in_order. - -The result should be a faster startup of deliveries when a large queue is -present and reasonable numbers of messages are routed to common hosts; this -could be a smarthost case, or delivery onto the Internet where a large proportion -of recipients hapen to be on a Gorilla-sized provider. - -As usual, the presence of a configuration option is associated with a -predefined macro, making it possible to write portable configurations. -For this one, the macro is _OPT_MAIN_QUEUE_FAST_RAMP. - - - -------------------------------------------------------------- End of file -------------------------------------------------------------- -- cgit v1.2.3 From 64dfd5e0cec10edded40f4669a706e6564aa8e07 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Tue, 12 May 2020 21:18:36 +0100 Subject: Docs: fix typo --- doc/doc-docbook/spec.xfpt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 0c4df93a7..30f5f2867 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -14805,7 +14805,7 @@ See also the &'Policy controls'& section above. .row &%hold_domains%& "hold delivery for these domains" .row &%local_interfaces%& "for routing checks" .row &%queue_domains%& "no immediate delivery for these" -.row &%queue_fast ramp%& "parallel delivery with 2-phase queue run" +.row &%queue_fast_ramp%& "parallel delivery with 2-phase queue run" .row &%queue_only%& "no immediate delivery at all" .row &%queue_only_file%& "no immediate delivery if file exists" .row &%queue_only_load%& "no immediate delivery if load is high" -- cgit v1.2.3 From 0de9945258c2e7910c35f715caf07c5e9270aa1b Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Tue, 12 May 2020 23:10:08 +0100 Subject: smtp transport: message_linelength_limit option. Bug 1684 --- doc/doc-docbook/spec.xfpt | 14 ++++++++++++++ doc/doc-txt/NewStuff | 3 +++ doc/doc-txt/OptionLists.txt | 1 + src/src/configure.default | 5 ----- src/src/deliver.c | 24 ++++++++++++++++++----- src/src/smtp_in.c | 8 ++++---- src/src/spool_out.c | 6 +++--- src/src/transports/smtp.c | 19 +++++++++++++++++++ src/src/transports/smtp.h | 1 + test/confs/0588 | 42 +++++++++++++++++++++++++++++++++++++++++ test/log/0588 | 15 +++++++++++++++ test/scripts/0000-Basic/0588 | 45 ++++++++++++++++++++++++++++++++++++++++++++ test/stdout/0572 | 1 + test/stdout/0588 | 40 +++++++++++++++++++++++++++++++++++++++ 14 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 test/confs/0588 create mode 100644 test/log/0588 create mode 100644 test/scripts/0000-Basic/0588 create mode 100644 test/stdout/0588 (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 30f5f2867..5c6955e58 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -25472,6 +25472,20 @@ so can cause parallel connections to the same host if &%remote_max_parallel%& permits this. +.new +.option message_linelength_limit smtp integer 998 +.cindex "line length" limit +This option sets the maximum line length, in bytes, that the transport +will send. Any messages with lines exceeding the given value +will fail and a failure-DSN ("bounce") message will if possible be returned +to the sender. +The default value is that defined by the SMTP standards. + +It is generally wise to also check in the data ACL so that messages +received via SMTP can be refused without producing a bounce. +.wen + + .option multi_domain smtp boolean&!! true .vindex "&$domain$&" When this option is set, the &(smtp)& transport can handle a number of diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 253eae2b7..82f1c5c18 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -21,6 +21,9 @@ Version 4.95 4. Single-key LMDB lookups, previously experimental, are now supported. The support is not built unless specified in the Local/Makefile. + 5. Option "message_linelength_limit" on the smtp transport to enforce (by + default) the RFC 998 character limit. + Version 4.94 ------------ diff --git a/doc/doc-txt/OptionLists.txt b/doc/doc-txt/OptionLists.txt index 5de6d6936..cfb3cd0f0 100644 --- a/doc/doc-txt/OptionLists.txt +++ b/doc/doc-txt/OptionLists.txt @@ -382,6 +382,7 @@ message_body_newlines boolean false main message_body_visible integer 500 main message_id_header_domain string* unset main 4.11 message_id_header_text string* unset main +message_linelength_limit integer 998 smtp 4.94 message_logs boolean true main 4.10 message_prefix string* + appendfile 4.00 replaces prefix string* unset pipe 4.00 replaces prefix diff --git a/src/src/configure.default b/src/src/configure.default index 57af99c14..7d54e11eb 100644 --- a/src/src/configure.default +++ b/src/src/configure.default @@ -808,13 +808,9 @@ begin transports # This transport is used for delivering messages over SMTP connections. -# Refuse to send any message with over-long lines, which could have -# been received other than via SMTP. The use of message_size_limit to -# enforce this is a red herring. remote_smtp: driver = smtp - message_size_limit = ${if > {$max_received_linelength}{998} {1}{0}} .ifdef _HAVE_TLS tls_resumption_hosts = * #endif @@ -832,7 +828,6 @@ remote_smtp: smarthost_smtp: driver = smtp - message_size_limit = ${if > {$max_received_linelength}{998} {1}{0}} multi_domain # .ifdef _HAVE_TLS diff --git a/src/src/deliver.c b/src/src/deliver.c index 3dcd7f949..67d711b7e 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -5380,7 +5380,8 @@ Returns: nothing static void print_dsn_diagnostic_code(const address_item *addr, FILE *f) { -uschar *s = testflag(addr, af_pass_message) ? addr->message : NULL; +uschar * s = testflag(addr, af_pass_message) ? addr->message : NULL; +unsigned cnt; /* af_pass_message and addr->message set ? print remote host answer */ if (s) @@ -5392,19 +5393,32 @@ if (s) if (!(s = Ustrstr(addr->message, ": "))) return; /* not found, bail out */ s += 2; /* skip ": " */ - fprintf(f, "Diagnostic-Code: smtp; "); + cnt = fprintf(f, "Diagnostic-Code: smtp; "); } /* no message available. do nothing */ else return; while (*s) + { + if (cnt > 950) /* RFC line length limit: 998 */ + { + DEBUG(D_deliver) debug_printf("print_dsn_diagnostic_code() truncated line\n"); + fputs("[truncated]", f); + break; + } + if (*s == '\\' && s[1] == 'n') { fputs("\n ", f); /* as defined in RFC 3461 */ s += 2; + cnt += 2; } else + { fputc(*s++, f); + cnt++; + } + } fputc('\n', f); } @@ -7831,11 +7845,11 @@ wording. */ fprintf(fp, "Remote-MTA: X-ip; [%s]%s\n", hu->address, p); } if ((s = addr->smtp_greeting) && *s) - fprintf(fp, "X-Remote-MTA-smtp-greeting: X-str; %s\n", s); + fprintf(fp, "X-Remote-MTA-smtp-greeting: X-str; %.900s\n", s); if ((s = addr->helo_response) && *s) - fprintf(fp, "X-Remote-MTA-helo-response: X-str; %s\n", s); + fprintf(fp, "X-Remote-MTA-helo-response: X-str; %.900s\n", s); if ((s = addr->message) && *s) - fprintf(fp, "X-Exim-Diagnostic: X-str; %s\n", s); + fprintf(fp, "X-Exim-Diagnostic: X-str; %.900s\n", s); } #endif print_dsn_diagnostic_code(addr, fp); diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index 412ef4df0..2b4323bec 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -5173,9 +5173,9 @@ while (done <= 0) recipients_list[recipients_count-1].orcpt = orcpt; recipients_list[recipients_count-1].dsn_flags = dsn_flags; - DEBUG(D_receive) debug_printf("DSN: orcpt: %s flags: %d\n", + /* DEBUG(D_receive) debug_printf("DSN: orcpt: %s flags: %d\n", recipients_list[recipients_count-1].orcpt, - recipients_list[recipients_count-1].dsn_flags); + recipients_list[recipients_count-1].dsn_flags); */ } /* The recipient was discarded */ @@ -5190,8 +5190,8 @@ while (done <= 0) discarded = TRUE; log_write(0, LOG_MAIN|LOG_REJECT, "%s F=<%s> RCPT %s: " "discarded by %s ACL%s%s", host_and_ident(TRUE), - sender_address_unrewritten? sender_address_unrewritten : sender_address, - smtp_cmd_argument, f.recipients_discarded? "MAIL" : "RCPT", + sender_address_unrewritten ? sender_address_unrewritten : sender_address, + smtp_cmd_argument, f.recipients_discarded ? "MAIL" : "RCPT", log_msg ? US": " : US"", log_msg ? log_msg : US""); } diff --git a/src/src/spool_out.c b/src/src/spool_out.c index 5d658fd74..9a514b331 100644 --- a/src/src/spool_out.c +++ b/src/src/spool_out.c @@ -277,9 +277,9 @@ if (message_smtputf8) #endif /* Write the dsn flags to the spool header file */ -DEBUG(D_deliver) debug_printf("DSN: Write SPOOL: -dsn_envid %s\n", dsn_envid); +/* DEBUG(D_deliver) debug_printf("DSN: Write SPOOL: -dsn_envid %s\n", dsn_envid); */ if (dsn_envid) fprintf(fp, "-dsn_envid %s\n", dsn_envid); -DEBUG(D_deliver) debug_printf("DSN: Write SPOOL :-dsn_ret %d\n", dsn_ret); +/* DEBUG(D_deliver) debug_printf("DSN: Write SPOOL: -dsn_ret %d\n", dsn_ret); */ if (dsn_ret) fprintf(fp, "-dsn_ret %d\n", dsn_ret); /* To complete the envelope, write out the tree of non-recipients, followed by @@ -293,7 +293,7 @@ for (int i = 0; i < recipients_count; i++) { recipient_item *r = recipients_list + i; - DEBUG(D_deliver) debug_printf("DSN: Flags: 0x%x\n", r->dsn_flags); + /* DEBUG(D_deliver) debug_printf("DSN: Flags: 0x%x\n", r->dsn_flags); */ if (r->pno < 0 && !r->errors_to && r->dsn_flags == 0) fprintf(fp, "%s\n", r->address); diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index f47c6d92f..12a199464 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -111,6 +111,7 @@ optionlist smtp_transport_options[] = { { "lmtp_ignore_quota", opt_bool, LOFF(lmtp_ignore_quota) }, { "max_rcpt", opt_int | opt_public, OPT_OFF(transport_instance, max_addresses) }, + { "message_linelength_limit", opt_int, LOFF(message_linelength_limit) }, { "multi_domain", opt_expand_bool | opt_public, OPT_OFF(transport_instance, multi_domain) }, { "port", opt_stringptr, LOFF(port) }, @@ -207,6 +208,7 @@ smtp_transport_options_block smtp_transport_option_defaults = { .size_addition = 1024, .hosts_max_try = 5, .hosts_max_try_hardlimit = 50, + .message_linelength_limit = 998, .address_retry_include_sender = TRUE, .allow_localhost = FALSE, .authenticated_sender_force = FALSE, @@ -4524,6 +4526,23 @@ DEBUG(D_transport) cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0); } +/* Check the restrictions on line length */ + +debug_printf("%s %d: max_received_linelength %u message_linelength_limit %u\n", __FUNCTION__, __LINE__, max_received_linelength, ob->message_linelength_limit); +if (max_received_linelength > ob->message_linelength_limit) + { + struct timeval now; + gettimeofday(&now, NULL); + + for (address_item * addr = addrlist; addr; addr = addr->next) + if (addr->transport_return == DEFER) + addr->transport_return = PENDING_DEFER; + + set_errno_nohost(addrlist, ERRNO_SMTPFORMAT, + US"message has lines too long for transport", FAIL, TRUE, &now); + goto END_TRANSPORT; + } + /* Set the flag requesting that these hosts be added to the waiting database if the delivery fails temporarily or if we are running with queue_smtp or a 2-stage queue run. This gets unset for certain diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h index 037105a00..607a3772d 100644 --- a/src/src/transports/smtp.h +++ b/src/src/transports/smtp.h @@ -62,6 +62,7 @@ typedef struct { int size_addition; int hosts_max_try; int hosts_max_try_hardlimit; + int message_linelength_limit; BOOL address_retry_include_sender; BOOL allow_localhost; BOOL authenticated_sender_force; diff --git a/test/confs/0588 b/test/confs/0588 new file mode 100644 index 000000000..9a88c9c52 --- /dev/null +++ b/test/confs/0588 @@ -0,0 +1,42 @@ +# Exim test configuration 0588 + +.include DIR/aux-var/std_conf_prefix + +log_selector = +received_recipients +sender_on_delivery +millisec + +# ----- Main settings ----- + +acl_smtp_rcpt = accept + +# ----- Routers ----- + +begin routers + +rx_dump: + driver = redirect + condition = ${if !eq {$received_ip_address}{127.0.0.1}} + data = :blackhole: + +smtp_try: + driver = accept + transport = send_to_server + +# ----- Transports ----- + +begin transports + +send_to_server: + driver = smtp + hosts = HOSTIPV4 + allow_localhost + port = PORT_D + hosts_try_fastopen = : + +# ----- Retry ----- + +begin retry + +* * F,5d,10s + +# End + diff --git a/test/log/0588 b/test/log/0588 new file mode 100644 index 000000000..944645114 --- /dev/null +++ b/test/log/0588 @@ -0,0 +1,15 @@ + +******** SERVER ******** +2017-07-30 18:51:05.712 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port PORT_D +2017-07-30 18:51:05.712 10HmaX-0005vi-00 <= CALLER@test.ex H=(test) [127.0.0.1] P=smtp S=sss for good@test.ex +2017-07-30 18:51:05.712 10HmaY-0005vi-00 <= CALLER@test.ex H=(test) [127.0.0.1] P=smtp S=sss for bad@test.ex +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 <= CALLER@test.ex H=the.local.host.name [ip4.ip4.ip4.ip4] P=esmtp S=sss for good@test.ex +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 => :blackhole: R=rx_dump +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 Completed +2017-07-30 18:51:05.712 10HmaX-0005vi-00 => good@test.ex F= R=smtp_try T=send_to_server H=ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4] C="250 OK id=10HmaZ-0005vi-00" +2017-07-30 18:51:05.712 10HmaX-0005vi-00 Completed +2017-07-30 18:51:05.712 10HmaY-0005vi-00 ** bad@test.ex F= R=smtp_try T=send_to_server: message has lines too long for transport +2017-07-30 18:51:05.712 10HmbA-0005vi-00 <= <> R=10HmaY-0005vi-00 U=EXIMUSER P=local S=sss for CALLER@test.ex +2017-07-30 18:51:05.712 10HmbA-0005vi-00 => :blackhole: R=rx_dump +2017-07-30 18:51:05.712 10HmbA-0005vi-00 Completed +2017-07-30 18:51:05.712 10HmaY-0005vi-00 Completed diff --git a/test/scripts/0000-Basic/0588 b/test/scripts/0000-Basic/0588 new file mode 100644 index 000000000..44328a70c --- /dev/null +++ b/test/scripts/0000-Basic/0588 @@ -0,0 +1,45 @@ +# message_linelength_limit +# +# The "write" script cmd subtracts 1 for the newline, +# and the linecount in exim doesn't count the line-ending. +write test-data-good 1x999 +++++ +**** +write test-data-bad 1x1000 +++++ +**** +# +exim -bd -DSERVER=server -oX PORT_D +**** +client 127.0.0.1 PORT_D +??? 220 +HELO test +??? 250 +MAIL FROM: +??? 250 +RCPT TO: +??? 250 +DATA +??? 354 +Subject: should be good + +<<< test-data-good +. +??? 250 +MAIL FROM: +??? 250 +RCPT TO: +??? 250 +DATA +??? 354 +Subject: should be bad + +<<< test-data-bad +. +??? 250 +QUIT +??? 221 +**** +# +sleep 1 +killdaemon diff --git a/test/stdout/0572 b/test/stdout/0572 index d66f928d4..fd77c72b7 100644 --- a/test/stdout/0572 +++ b/test/stdout/0572 @@ -64,6 +64,7 @@ interface = ip4.ip4.ip4.ip4 keepalive no_lmtp_ignore_quota max_rcpt = 100 +message_linelength_limit = 998 multi_domain port = 1224 protocol = smtp diff --git a/test/stdout/0588 b/test/stdout/0588 new file mode 100644 index 000000000..994f04e2e --- /dev/null +++ b/test/stdout/0588 @@ -0,0 +1,40 @@ +Connecting to 127.0.0.1 port 1225 ... connected +??? 220 +<<< 220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +>>> HELO test +??? 250 +<<< 250 the.local.host.name Hello test [127.0.0.1] +>>> MAIL FROM: +??? 250 +<<< 250 OK +>>> RCPT TO: +??? 250 +<<< 250 Accepted +>>> DATA +??? 354 +<<< 354 Enter message, ending with "." on a line by itself +>>> Subject: should be good +>>> +>>> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +>>> . +??? 250 +<<< 250 OK id=10HmaX-0005vi-00 +>>> MAIL FROM: +??? 250 +<<< 250 OK +>>> RCPT TO: +??? 250 +<<< 250 Accepted +>>> DATA +??? 354 +<<< 354 Enter message, ending with "." on a line by itself +>>> Subject: should be bad +>>> +>>> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +>>> . +??? 250 +<<< 250 OK id=10HmaY-0005vi-00 +>>> QUIT +??? 221 +<<< 221 the.local.host.name closing connection +End of script -- cgit v1.2.3 From 618f89bfd392ed2094889bb5d4c6d734bcc72b03 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 16 May 2020 13:13:39 +0100 Subject: Docs: chapter title Broken-by: 7adc9ca07a --- doc/doc-docbook/spec.xfpt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 5c6955e58..b4d4232e0 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -40590,8 +40590,8 @@ There is no dot-stuffing (and no dot-termination). . //////////////////////////////////////////////////////////////////////////// . //////////////////////////////////////////////////////////////////////////// -.chapter "DKIM, SPF and DMARC" "CHAPdkim" &&& - "DKIM, SPF and DMARC Support" +.chapter "DKIM, SPF, SRS and DMARC" "CHAPdkim" &&& + "DKIM, SPF, SRS and DMARC Support" .section "DKIM (DomainKeys Identified Mail)" SECDKIM .cindex "DKIM" -- cgit v1.2.3 From 3db72f4b639a64cacf152e4f7718a18581426b10 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 16 May 2020 19:30:37 +0100 Subject: Lookups: cache=no_rd option. Bug 1751 --- doc/doc-docbook/spec.xfpt | 19 +++++++++++++ doc/doc-txt/NewStuff | 2 ++ src/src/search.c | 63 ++++++++++++++++++++++++++++-------------- test/scripts/2500-dsearch/2500 | 11 ++++++++ test/stderr/0002 | 2 ++ test/stderr/0085 | 5 ++++ test/stderr/0123 | 18 ++++++++++++ test/stderr/0387 | 32 +++++++++++++++++++++ test/stderr/0403 | 2 ++ test/stderr/0414 | 4 +++ test/stderr/0437 | 5 ++++ test/stderr/0464 | 2 ++ test/stderr/0471 | 5 ++++ test/stderr/0484 | 4 +++ test/stderr/2200 | 3 ++ test/stderr/2201 | 5 ++++ test/stderr/2202 | 1 + test/stderr/2600 | 13 +++++++++ test/stderr/2610 | 21 ++++++++++++++ test/stderr/2620 | 19 +++++++++++++ test/stderr/3201 | 2 ++ test/stderr/3212 | 4 +++ test/stdout/2500 | 11 ++++++++ 23 files changed, 232 insertions(+), 21 deletions(-) (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index b4d4232e0..a1f361339 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -6668,6 +6668,18 @@ If the value of &$sender_host_address$& is 192.168.5.6, expansion of the first &%domains%& setting above generates the second setting, which therefore causes a second lookup to occur. +.new +The lookup type may optionally be followed by a comma +and a comma-separated list of options. +Each option is a &"name=value"& pair. +Whether an option is meaningful depands on the lookup type. + +All lookups support the option &"cache=no_rd"&. +If this is given then the cache that Exim manages for lookup results +is not checked before diong the lookup. +The result of the lookup is still written to the cache. +.wen + The rest of this chapter describes the different lookup types that are available. Any of them can be used in any part of the configuration where a lookup is permitted. @@ -6685,6 +6697,13 @@ lookup to succeed. The lookup type determines how the file is searched. .new .cindex "tainted data" "single-key lookups" The file string may not be tainted + +All single-key lookups support the option &"ret=key"&. +If this is given and the lookup +(either underlying implementation or cached value) +returns data, the result is replaced with a non-tainted +version of the lookup key. +.cindex "tainted data" "de-tainting" .wen .next .cindex "query-style lookup" "definition of" diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 82f1c5c18..83b696fe5 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -24,6 +24,8 @@ Version 4.95 5. Option "message_linelength_limit" on the smtp transport to enforce (by default) the RFC 998 character limit. + 6. An option to ignore the cache on a lookup. + Version 4.94 ------------ diff --git a/src/src/search.c b/src/src/search.c index 94a58897f..d1633a5e1 100644 --- a/src/src/search.c +++ b/src/src/search.c @@ -462,6 +462,7 @@ Arguments: NULL for query-style searches keystring the keystring for single-key+file lookups, or the querystring for query-style lookups + cache_rd FALSE to avoid lookup in cache layer opts type-specific options Returns: a pointer to a dynamic string containing the answer, @@ -472,7 +473,7 @@ Returns: a pointer to a dynamic string containing the answer, static uschar * internal_search_find(void * handle, const uschar * filename, uschar * keystring, - const uschar * opts) + BOOL cache_rd, const uschar * opts) { tree_node * t = (tree_node *)handle; search_cache * c = (search_cache *)(t->data.ptr); @@ -501,11 +502,13 @@ if (keystring[0] == 0) return NULL; store_pool = POOL_SEARCH; /* Look up the data for the key, unless it is already in the cache for this -file. No need to check c->item_cache for NULL, tree_search will do so. */ +file. No need to check c->item_cache for NULL, tree_search will do so. Check +whether we want to use the cache entry last so that we can always replace it. */ if ( (t = tree_search(c->item_cache, keystring)) && (!(e = t->data.ptr)->expiry || e->expiry > time(NULL)) && (!opts && !e->opts || opts && e->opts && Ustrcmp(opts, e->opts) == 0) + && cache_rd ) { /* Data was in the cache already; set the pointer from the tree node */ data = e->data.ptr; @@ -522,7 +525,8 @@ else { if (t) debug_printf_indent("cached data found but %s; ", - e->expiry && e->expiry <= time(NULL) ? "out-of-date" : "wrong opts"); + e->expiry && e->expiry <= time(NULL) ? "out-of-date" + : cache_rd ? "wrong opts" : "no_rd option set"); debug_printf_indent("%s lookup required for %s%s%s\n", filename ? US"file" : US"database", keystring, @@ -541,14 +545,19 @@ else or points to a bit of dynamic store. Cache the result of the lookup if caching is permitted. Lookups can disable caching, when they did something that changes their data. The mysql and pgsql lookups do this when an - UPDATE/INSERT query was executed. */ + UPDATE/INSERT query was executed. Lookups can also set a TTL for the + cache entry; the dnsdb lookup does. + Finally, the caller can request no caching by setting an option. */ else if (do_cache) { + DEBUG(D_lookup) debug_printf_indent("%s cache entry\n", + t ? "replacing old" : "creating new"); if (!t) /* No existing entry. Create new one. */ { int len = keylength + 1; - e = store_get(sizeof(expiring_data) + sizeof(tree_node) + len, is_tainted(keystring)); + e = store_get(sizeof(expiring_data) + sizeof(tree_node) + len, + is_tainted(keystring)); t = (tree_node *)(e+1); memcpy(t->name, keystring, len); t->data.ptr = e; @@ -621,9 +630,9 @@ search_find(void * handle, const uschar * filename, uschar * keystring, int partial, const uschar * affix, int affixlen, int starflags, int * expand_setup, const uschar * opts) { -tree_node *t = (tree_node *)handle; -BOOL set_null_wild = FALSE; -uschar *yield; +tree_node * t = (tree_node *)handle; +BOOL set_null_wild = FALSE, cache_rd = TRUE, ret_key = FALSE; +uschar * yield; DEBUG(D_lookup) { @@ -636,6 +645,23 @@ DEBUG(D_lookup) } +/* Parse global lookup options. Also, create a new options list with +the global options dropped so that the cache-modifiers are not +used in the cache key. */ + +if (opts) + { + int sep = ','; + gstring * g = NULL; + + for (uschar * ele; ele = string_nextinlist(&opts, &sep, NULL, 0); ) + if (Ustrcmp(ele, "ret=key") == 0) ret_key = TRUE; + else if (Ustrcmp(ele, "cache=no_rd") == 0) cache_rd = FALSE; + else g = string_append_listele(g, ',', ele); + + opts = string_from_gstring(g); + } + /* Arrange to put this database at the top of the LRU chain if it is a type that opens real files. */ @@ -683,7 +709,7 @@ DEBUG(D_lookup) /* First of all, try to match the key string verbatim. If matched a complete entry but could have been partial, flag to set up variables. */ -yield = internal_search_find(handle, filename, keystring, opts); +yield = internal_search_find(handle, filename, keystring, cache_rd, opts); if (f.search_find_defer) return NULL; if (yield) { if (partial >= 0) set_null_wild = TRUE; } @@ -708,7 +734,7 @@ else if (partial >= 0) Ustrncpy(keystring2, affix, affixlen); Ustrcpy(keystring2 + affixlen, keystring); DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring2); - yield = internal_search_find(handle, filename, keystring2, opts); + yield = internal_search_find(handle, filename, keystring2, cache_rd, opts); if (f.search_find_defer) return NULL; } @@ -746,7 +772,8 @@ else if (partial >= 0) } DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring3); - yield = internal_search_find(handle, filename, keystring3, opts); + yield = internal_search_find(handle, filename, keystring3, + cache_rd, opts); if (f.search_find_defer) return NULL; if (yield) { @@ -787,7 +814,7 @@ if (!yield && starflags & SEARCH_STARAT) *atat = '*'; DEBUG(D_lookup) debug_printf_indent("trying default match %s\n", atat); - yield = internal_search_find(handle, filename, atat, opts); + yield = internal_search_find(handle, filename, atat, cache_rd, opts); *atat = savechar; if (f.search_find_defer) return NULL; @@ -810,7 +837,7 @@ and the second is empty. */ if (!yield && starflags & (SEARCH_STAR|SEARCH_STARAT)) { DEBUG(D_lookup) debug_printf_indent("trying to match *\n"); - yield = internal_search_find(handle, filename, US"*", opts); + yield = internal_search_find(handle, filename, US"*", cache_rd, opts); if (yield && expand_setup && *expand_setup >= 0) { *expand_setup += 1; @@ -843,14 +870,8 @@ if (set_null_wild && expand_setup && *expand_setup >= 0) than the result. Return a de-tainted version of the key on the grounds that it have been validated by the lookup. */ -if (yield && opts) - { - int sep = ','; - uschar * ele; - while ((ele = string_nextinlist(&opts, &sep, NULL, 0))) - if (Ustrcmp(ele, "ret=key") == 0) - { yield = string_copy_taint(keystring, FALSE); break; } - } +if (yield && ret_key) + yield = string_copy_taint(keystring, FALSE); return yield; } diff --git a/test/scripts/2500-dsearch/2500 b/test/scripts/2500-dsearch/2500 index 381ef2563..8677b7166 100644 --- a/test/scripts/2500-dsearch/2500 +++ b/test/scripts/2500-dsearch/2500 @@ -1,6 +1,7 @@ # dsearch lookup # This test will fail on a case-insensitive filesystem (e.g. MacOS default) exim -be +dsearch specifics ok: ${lookup{TESTNUM.tst} dsearch{DIR/aux-fixed}{$value}{FAIL}} fail: ${lookup{TESTNUM.file_not_here} dsearch{DIR/aux-fixed}{$value}{FAIL}} fail: ${lookup{TESTNUM.tst} dsearch{DIR/dir_not_here}{$value}{FAIL}} @@ -16,6 +17,16 @@ fail,dir: ${lookup{TESTNUM.tst} dsearch,filter=dir {DIR/aux-fixed}{$value}{ ok,subdir: ${lookup{TESTNUM.dir} dsearch,filter=subdir {DIR/aux-fixed}{$value}{FAIL}} fail,subdir:${lookup{..} dsearch,filter=subdir {DIR/aux-fixed}{$value}{FAIL}} fail,subdir:${lookup{TESTNUM.tst} dsearch,filter=subdir {DIR/aux-fixed}{$value}{FAIL}} + +cachelayer tests +fail: ${lookup{test-data} dsearch {DIR/} {$value}{FAIL}} +createfile: ${run {/bin/cp DIR/aux-fixed/TESTNUM.tst DIR/test-data} {OK}{FAIL}} +fail,cached:${lookup{test-data} dsearch {DIR/} {$value}{FAIL}} +ok,no_rd ${lookup{test-data} dsearch,cache=no_rd {DIR/} {$value}{FAIL}} +delfile: ${run {/bin/rm DIR/test-data} {OK}{FAIL}} +ok,cached: ${lookup{test-data} dsearch {DIR/} {$value}{FAIL}} +fail,no_rd ${lookup{test-data} dsearch,cache=no_rd {DIR/} {$value}{FAIL}} +fail: ${lookup{test-data} dsearch {DIR/} {$value}{FAIL}} **** # 1 diff --git a/test/stderr/0002 b/test/stderr/0002 index 9d6fb5df8..bd2d8067f 100644 --- a/test/stderr/0002 +++ b/test/stderr/0002 @@ -326,6 +326,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0002.lsearch" type=lsearch key="ten-1.test.ex" opts=NULL file lookup required for ten-1.test.ex in TESTSUITE/aux-fixed/0002.lsearch +creating new cache entry lookup yielded: host in "< partial-lsearch;TESTSUITE/aux-fixed/0002.lsearch @@ -372,6 +373,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0002.lsearch" type=lsearch key="V4NET.0.0.2" opts=NULL file lookup required for V4NET.0.0.2 in TESTSUITE/aux-fixed/0002.lsearch +creating new cache entry lookup failed host in "net-lsearch;TESTSUITE/aux-fixed/0002.lsearch"? no (end of list) deny: condition test failed in ACL "connect2" diff --git a/test/stderr/0085 b/test/stderr/0085 index e27089647..7a30dfe54 100644 --- a/test/stderr/0085 +++ b/test/stderr/0085 @@ -54,6 +54,7 @@ checking local_parts type=lsearch key="smart.domain" opts=NULL file lookup required for smart.domain in TESTSUITE/aux-fixed/0085.data + creating new cache entry lookup yielded: x : y : abc@d.e.f x in "x : y : abc@d.e.f"? yes (matched "x") checking senders @@ -124,6 +125,7 @@ checking local_parts type=lsearch key="test.ex" opts=NULL file lookup required for test.ex in TESTSUITE/aux-fixed/0085.data + creating new cache entry lookup yielded: x : y : abc@d.e.f x in "x : y : abc@d.e.f"? yes (matched "x") checking senders @@ -158,6 +160,7 @@ checking require_files type=lsearch key="test.ex.files" opts=NULL file lookup required for test.ex.files in TESTSUITE/aux-fixed/0085.data + creating new cache entry lookup yielded: /etc/passwd file check: ${lookup{$domain.files}lsearch{TESTSUITE/aux-fixed/0085.data}{$value}} expanded file: /etc/passwd @@ -255,6 +258,7 @@ checking local_parts type=lsearch key="smart.domain" opts=NULL file lookup required for smart.domain in TESTSUITE/aux-fixed/0085.data + creating new cache entry lookup yielded: x : y : abc@d.e.f x in "x : y : abc@d.e.f"? yes (matched "x") checking senders @@ -323,6 +327,7 @@ checking local_parts type=lsearch key="test.ex" opts=NULL file lookup required for test.ex in TESTSUITE/aux-fixed/0085.data + creating new cache entry lookup yielded: x : y : abc@d.e.f x in "x : y : abc@d.e.f"? yes (matched "x") checking senders diff --git a/test/stderr/0123 b/test/stderr/0123 index d393252b5..347d9761b 100644 --- a/test/stderr/0123 +++ b/test/stderr/0123 @@ -27,6 +27,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="x" opts=NULL file lookup required for x in TESTSUITE/aux-fixed/0123.aliases1 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -47,6 +48,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="x" opts=NULL file lookup required for x in TESTSUITE/aux-fixed/0123.aliases2 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -69,6 +71,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="x" opts=NULL file lookup required for x in TESTSUITE/aux-fixed/0123.aliases3 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -91,6 +94,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="x" opts=NULL file lookup required for x in TESTSUITE/aux-fixed/0123.aliases4 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -113,6 +117,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="x" opts=NULL file lookup required for x in TESTSUITE/aux-fixed/0123.aliases5 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -135,6 +140,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="x" opts=NULL file lookup required for x in TESTSUITE/aux-fixed/0123.aliases6 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -165,6 +171,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="y" opts=NULL file lookup required for y in TESTSUITE/aux-fixed/0123.aliases1 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -188,6 +195,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="y" opts=NULL file lookup required for y in TESTSUITE/aux-fixed/0123.aliases2 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -211,6 +219,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="y" opts=NULL file lookup required for y in TESTSUITE/aux-fixed/0123.aliases3 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -234,6 +243,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="y" opts=NULL file lookup required for y in TESTSUITE/aux-fixed/0123.aliases4 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -257,6 +267,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="y" opts=NULL file lookup required for y in TESTSUITE/aux-fixed/0123.aliases5 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -280,6 +291,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="y" opts=NULL file lookup required for y in TESTSUITE/aux-fixed/0123.aliases6 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -310,6 +322,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="z" opts=NULL file lookup required for z in TESTSUITE/aux-fixed/0123.aliases1 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -333,6 +346,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="z" opts=NULL file lookup required for z in TESTSUITE/aux-fixed/0123.aliases2 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -356,6 +370,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="z" opts=NULL file lookup required for z in TESTSUITE/aux-fixed/0123.aliases3 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -379,6 +394,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="z" opts=NULL file lookup required for z in TESTSUITE/aux-fixed/0123.aliases4 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -402,6 +418,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="z" opts=NULL file lookup required for z in TESTSUITE/aux-fixed/0123.aliases5 + creating new cache entry lookup failed expanded: '' file is not a filter file @@ -425,6 +442,7 @@ rda_interpret (string): '${lookup{$local_part}lsearch{TESTSUITE/aux-fixed/0123.a type=lsearch key="z" opts=NULL file lookup required for z in TESTSUITE/aux-fixed/0123.aliases6 + creating new cache entry lookup failed expanded: '' file is not a filter file diff --git a/test/stderr/0387 b/test/stderr/0387 index 292f52337..0a27a5878 100644 --- a/test/stderr/0387 +++ b/test/stderr/0387 @@ -13,18 +13,21 @@ dropping to exim gid; retaining priv uid type=lsearch key="a.b.c" opts=NULL file lookup required for a.b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.a.b.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.a.b.c" opts=NULL file lookup required for *.a.b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.b.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.b.c" opts=NULL file lookup required for *.b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [*.b.c] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -37,18 +40,21 @@ dropping to exim gid; retaining priv uid type=lsearch key="x.y.c" opts=NULL file lookup required for x.y.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.x.y.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.x.y.c" opts=NULL file lookup required for *.x.y.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.y.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.y.c" opts=NULL file lookup required for *.y.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -79,6 +85,7 @@ dropping to exim gid; retaining priv uid type=lsearch key="*" opts=NULL file lookup required for * in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [*] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -109,6 +116,7 @@ dropping to exim gid; retaining priv uid type=lsearch key="*.c" opts=NULL file lookup required for *.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [*.c] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -121,12 +129,14 @@ dropping to exim gid; retaining priv uid type=lsearch key="x@y.c" opts=NULL file lookup required for x@y.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.x@y.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.x@y.c" opts=NULL file lookup required for *.x@y.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" @@ -151,6 +161,7 @@ dropping to exim gid; retaining priv uid type=lsearch key="*@y.c" opts=NULL file lookup required for *@y.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [*@y.c] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -193,12 +204,14 @@ dropping to exim gid; retaining priv uid type=lsearch key=".a.b.c" opts=NULL file lookup required for .a.b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match .b.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key=".b.c" opts=NULL file lookup required for .b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [.b.c] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -217,6 +230,7 @@ dropping to exim gid; retaining priv uid type=lsearch key="b.c" opts=NULL file lookup required for b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [b.c] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -235,12 +249,14 @@ dropping to exim gid; retaining priv uid type=lsearch key="*a.b.c" opts=NULL file lookup required for *a.b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *b.c internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*b.c" opts=NULL file lookup required for *b.c in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [*b.c] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -253,24 +269,28 @@ dropping to exim gid; retaining priv uid type=lsearch key="p.q.r" opts=NULL file lookup required for p.q.r in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.p.q.r internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.p.q.r" opts=NULL file lookup required for *.p.q.r in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.q.r internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.q.r" opts=NULL file lookup required for *.q.r in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match *.r internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="*.r" opts=NULL file lookup required for *.r in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match * internal_search_find: file="TESTSUITE/aux-fixed/0387.1" @@ -295,24 +315,28 @@ dropping to exim gid; retaining priv uid type=lsearch key=".p.q.r" opts=NULL file lookup required for .p.q.r in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match .q.r internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key=".q.r" opts=NULL file lookup required for .q.r in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match .r internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key=".r" opts=NULL file lookup required for .r in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match . internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="." opts=NULL file lookup required for . in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [.] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -325,18 +349,21 @@ dropping to exim gid; retaining priv uid type=lsearch key="x.aa.bb" opts=NULL file lookup required for x.aa.bb in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match ++x.aa.bb internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="++x.aa.bb" opts=NULL file lookup required for ++x.aa.bb in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match ++aa.bb internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="++aa.bb" opts=NULL file lookup required for ++aa.bb in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [++aa.bb] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open @@ -349,30 +376,35 @@ dropping to exim gid; retaining priv uid type=lsearch key="x.aa.zz" opts=NULL file lookup required for x.aa.zz in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match ++x.aa.zz internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="++x.aa.zz" opts=NULL file lookup required for ++x.aa.zz in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match ++aa.zz internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="++aa.zz" opts=NULL file lookup required for ++aa.zz in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match ++zz internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="++zz" opts=NULL file lookup required for ++zz in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup failed trying partial match ++ internal_search_find: file="TESTSUITE/aux-fixed/0387.1" type=lsearch key="++" opts=NULL file lookup required for ++ in TESTSUITE/aux-fixed/0387.1 + creating new cache entry lookup yielded: [++] search_open: lsearch "TESTSUITE/aux-fixed/0387.1" cached open diff --git a/test/stderr/0403 b/test/stderr/0403 index 0c22cf8f5..16bf13b5e 100644 --- a/test/stderr/0403 +++ b/test/stderr/0403 @@ -94,6 +94,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0403.data" type=lsearch key="test.ex" opts=NULL file lookup required for test.ex in TESTSUITE/aux-fixed/0403.data +creating new cache entry lookup yielded: [DOMAINDATA_test.ex] test.ex in "lsearch;TESTSUITE/aux-fixed/0403.data"? yes (matched "lsearch;TESTSUITE/aux-fixed/0403.data") checking local_parts @@ -108,6 +109,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0403.data" type=lsearch key="userx" opts=NULL file lookup required for userx in TESTSUITE/aux-fixed/0403.data +creating new cache entry lookup yielded: [LOCALPARTDATA_userx] userx in "lsearch;TESTSUITE/aux-fixed/0403.data"? yes (matched "lsearch;TESTSUITE/aux-fixed/0403.data") +++ROUTER: diff --git a/test/stderr/0414 b/test/stderr/0414 index f1edd1ae3..18a514700 100644 --- a/test/stderr/0414 +++ b/test/stderr/0414 @@ -28,6 +28,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0414.list1" type=lsearch key="b.domain" opts=NULL file lookup required for b.domain in TESTSUITE/aux-fixed/0414.list1 +creating new cache entry lookup failed b.domain in "lsearch;TESTSUITE/aux-fixed/0414.list1"? no (end of list) search_open: lsearch "TESTSUITE/aux-fixed/0414.list2" @@ -41,6 +42,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0414.list2" type=lsearch key="b.domain" opts=NULL file lookup required for b.domain in TESTSUITE/aux-fixed/0414.list2 +creating new cache entry lookup yielded: b.domain-data b.domain in "lsearch;TESTSUITE/aux-fixed/0414.list2"? yes (matched "lsearch;TESTSUITE/aux-fixed/0414.list2") data from lookup saved for cache for +B: key 'b.domain' value 'b.domain-data' @@ -87,6 +89,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0414.list1" type=lsearch key="a.domain" opts=NULL file lookup required for a.domain in TESTSUITE/aux-fixed/0414.list1 +creating new cache entry lookup yielded: a.domain-data a.domain in "lsearch;TESTSUITE/aux-fixed/0414.list1"? yes (matched "lsearch;TESTSUITE/aux-fixed/0414.list1") data from lookup saved for cache for +A: key 'a.domain' value 'a.domain-data' @@ -109,6 +112,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0414.list2" type=lsearch key="a.domain" opts=NULL file lookup required for a.domain in TESTSUITE/aux-fixed/0414.list2 +creating new cache entry lookup failed a.domain in "lsearch;TESTSUITE/aux-fixed/0414.list2"? no (end of list) a.domain in "+B"? no (end of list) diff --git a/test/stderr/0437 b/test/stderr/0437 index 0e5a5b3e9..8c2cf0a8a 100644 --- a/test/stderr/0437 +++ b/test/stderr/0437 @@ -9,6 +9,7 @@ Exim version x.yz .... type=lsearch key="spool" opts=NULL file lookup required for spool in TESTSUITE/aux-fixed/0437.ls + creating new cache entry lookup yielded: spool configuration file is TESTSUITE/test-config admin user @@ -26,6 +27,7 @@ search_tidyup called type=lsearch key="transport" opts=NULL file lookup required for transport in TESTSUITE/aux-fixed/0437.ls + creating new cache entry lookup yielded: t1 search_tidyup called search_tidyup called @@ -39,6 +41,7 @@ search_tidyup called type=lsearch key="file" opts=NULL file lookup required for file in TESTSUITE/aux-fixed/0437.ls + creating new cache entry lookup yielded: file search_tidyup called LOG: MAIN @@ -58,6 +61,7 @@ search_tidyup called type=lsearch key="transport" opts=NULL file lookup required for transport in TESTSUITE/aux-fixed/0437.ls + creating new cache entry lookup yielded: t1 search_tidyup called search_tidyup called @@ -71,6 +75,7 @@ search_tidyup called type=lsearch key="file" opts=NULL file lookup required for file in TESTSUITE/aux-fixed/0437.ls + creating new cache entry lookup yielded: file search_tidyup called LOG: MAIN diff --git a/test/stderr/0464 b/test/stderr/0464 index 6ea3f3f97..625b57122 100644 --- a/test/stderr/0464 +++ b/test/stderr/0464 @@ -31,6 +31,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0464.domains" type=lsearch key="domain1" opts=NULL file lookup required for domain1 in TESTSUITE/aux-fixed/0464.domains +creating new cache entry lookup yielded: data for domain1 domain1 in "lsearch;TESTSUITE/aux-fixed/0464.domains"? yes (matched "lsearch;TESTSUITE/aux-fixed/0464.domains") data from lookup saved for cache for +special_domains: key 'domain1' value 'data for domain1' @@ -78,6 +79,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/0464.domains" type=lsearch key="xxx.domain1" opts=NULL file lookup required for xxx.domain1 in TESTSUITE/aux-fixed/0464.domains +creating new cache entry lookup failed xxx.domain1 in "lsearch;TESTSUITE/aux-fixed/0464.domains"? no (end of list) xxx.domain1 in "+special_domains"? no (end of list) diff --git a/test/stderr/0471 b/test/stderr/0471 index f4619a3e9..14fbe5534 100644 --- a/test/stderr/0471 +++ b/test/stderr/0471 @@ -111,12 +111,14 @@ r1@test.ex in "*@*"? yes (matched "*@*") type=lsearch key="test.ex" opts=NULL file lookup required for test.ex in TESTSUITE/aux-fixed/0471.rw + creating new cache entry lookup failed trying partial match *.test.ex internal_search_find: file="TESTSUITE/aux-fixed/0471.rw" type=lsearch key="*.test.ex" opts=NULL file lookup required for *.test.ex in TESTSUITE/aux-fixed/0471.rw + creating new cache entry lookup failed address match test: subject=CALLER@myhost.test.ex pattern=*@* myhost.test.ex in "*"? yes (matched "*") @@ -132,12 +134,14 @@ CALLER@myhost.test.ex in "*@*"? yes (matched "*@*") type=lsearch key="myhost.test.ex" opts=NULL file lookup required for myhost.test.ex in TESTSUITE/aux-fixed/0471.rw + creating new cache entry lookup failed trying partial match *.myhost.test.ex internal_search_find: file="TESTSUITE/aux-fixed/0471.rw" type=lsearch key="*.myhost.test.ex" opts=NULL file lookup required for *.myhost.test.ex in TESTSUITE/aux-fixed/0471.rw + creating new cache entry lookup failed trying partial match *.test.ex internal_search_find: file="TESTSUITE/aux-fixed/0471.rw" @@ -241,6 +245,7 @@ random@test.example in "*@*"? yes (matched "*@*") type=lsearch key="test.example" opts=NULL file lookup required for test.example in TESTSUITE/aux-fixed/0471.rw + creating new cache entry lookup yielded: rwtest.example LOG: address_rewrite MAIN "random@test.example" from to: rewritten as "random@rwtest.example" by rule 1 diff --git a/test/stderr/0484 b/test/stderr/0484 index 25d7f745d..56ce2e13a 100644 --- a/test/stderr/0484 +++ b/test/stderr/0484 @@ -16,6 +16,7 @@ sender address = CALLER@myhost.test.ex type=lsearch key="list" opts=NULL file lookup required for list in TESTSUITE/aux-fixed/0484.aliases + creating new cache entry lookup yielded: userx, usery search_open: lsearch "TESTSUITE/aux-fixed/0484.aliases" cached open @@ -40,6 +41,7 @@ sender address = CALLER@myhost.test.ex type=lsearch key="root" opts=NULL file lookup required for root in TESTSUITE/aux-fixed/0484.aliases + creating new cache entry lookup yielded: userx search_open: lsearch "TESTSUITE/aux-fixed/0484.aliases" cached open @@ -76,6 +78,7 @@ sender address = CALLER@myhost.test.ex type=lsearch key="list" opts=NULL file lookup required for list in TESTSUITE/aux-fixed/0484.aliases2 + creating new cache entry lookup yielded: userx2, usery2 search_open: lsearch "TESTSUITE/aux-fixed/0484.aliases2" cached open @@ -89,6 +92,7 @@ sender address = CALLER@myhost.test.ex type=lsearch key="root" opts=NULL file lookup required for root in TESTSUITE/aux-fixed/0484.aliases2 + creating new cache entry lookup failed search_open: lsearch "TESTSUITE/aux-fixed/0484.aliases2" cached open diff --git a/test/stderr/2200 b/test/stderr/2200 index ba50dffa7..9631e9b82 100644 --- a/test/stderr/2200 +++ b/test/stderr/2200 @@ -10,6 +10,7 @@ dropping to exim gid; retaining priv uid type=dnsdb key="a=localhost.test.ex" opts=NULL database lookup required for a=localhost.test.ex dnsdb key: localhost.test.ex + creating new cache entry lookup yielded: 127.0.0.1 search_open: dnsdb "NULL" cached open @@ -35,6 +36,7 @@ search_tidyup called type=dnsdb key="a=shorthost.test.ex" opts=NULL database lookup required for a=shorthost.test.ex dnsdb key: shorthost.test.ex + creating new cache entry lookup yielded: 127.0.0.1 search_open: dnsdb "NULL" cached open @@ -45,6 +47,7 @@ search_tidyup called type=dnsdb key="a=shorthost.test.ex" opts=NULL cached data found but out-of-date; database lookup required for a=shorthost.test.ex dnsdb key: shorthost.test.ex + replacing old cache entry lookup yielded: 127.0.0.1 LOG: MAIN <= CALLER@myhost.test.ex U=CALLER P=local S=sss diff --git a/test/stderr/2201 b/test/stderr/2201 index 8f9d0b585..d25ac4575 100644 --- a/test/stderr/2201 +++ b/test/stderr/2201 @@ -37,6 +37,7 @@ database lookup required for test.ex dnsdb key: test.ex DNS lookup of test.ex (TXT) using fakens DNS lookup of test.ex (TXT) succeeded +creating new cache entry lookup yielded: A TXT record for test.ex. test.ex in "dnsdb;test.ex"? yes (matched "dnsdb;test.ex") checking local_parts @@ -101,6 +102,7 @@ DNS lookup of unknown (TXT) gave HOST_NOT_FOUND returning DNS_NOMATCH faking res_search(TXT) response length as 65535 writing neg-cache entry for unknown-TXT-xxxx, ttl 3000 +creating new cache entry lookup failed unknown in "dnsdb;unknown"? no (end of list) r1 router skipped: local_parts mismatch @@ -130,6 +132,7 @@ database lookup required for A=myhost.test.ex dnsdb key: myhost.test.ex DNS lookup of myhost.test.ex (A) using fakens DNS lookup of myhost.test.ex (A) succeeded +creating new cache entry lookup yielded: V4NET.10.10.10 CALLER@myhost.test.ex in "dnsdb;A=myhost.test.ex"? yes (matched "dnsdb;A=myhost.test.ex") calling r2 router @@ -159,6 +162,7 @@ internal_search_find: file="NULL" type=dnsdb key="a=shorthost.test.ex" opts=NULL database lookup required for a=shorthost.test.ex dnsdb key: shorthost.test.ex +creating new cache entry lookup yielded: 127.0.0.1 search_open: dnsdb "NULL" cached open @@ -178,6 +182,7 @@ internal_search_find: file="NULL" type=dnsdb key="a=shorthost.test.ex" opts=NULL cached data found but out-of-date; database lookup required for a=shorthost.test.ex dnsdb key: shorthost.test.ex +replacing old cache entry lookup yielded: 127.0.0.1 LOG: MAIN <= a@shorthost.test.ex U=CALLER P=local S=sss diff --git a/test/stderr/2202 b/test/stderr/2202 index 3752ec538..2c595a6dd 100644 --- a/test/stderr/2202 +++ b/test/stderr/2202 @@ -44,6 +44,7 @@ cioce.test.again.dns in dns_again_means_nonexist? yes (matched "*") cioce.test.again.dns is in dns_again_means_nonexist: returning DNS_NOMATCH DNS: no SOA record found for neg-TTL writing neg-cache entry for cioce.test.again.dns-MX-xxxx, ttl -1 + creating new cache entry lookup failed sender host name required, to match against *.cioce.test.again.dns looking up host name for ip4.ip4.ip4.ip4 diff --git a/test/stderr/2600 b/test/stderr/2600 index 399fa0adb..e5dd60a27 100644 --- a/test/stderr/2600 +++ b/test/stderr/2600 @@ -10,6 +10,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select name from them where id='userx';" opts=NULL file lookup required for select name from them where id='userx'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: Ayen Other search_open: sqlite "TESTSUITE/aux-fixed/sqlitedb" cached open @@ -41,6 +42,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select name from them where id='nothing';" opts=NULL file lookup required for select name from them where id='nothing'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: search_open: sqlite "TESTSUITE/aux-fixed/sqlitedb" cached open @@ -51,6 +53,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select id,name from them where id='nothing';" opts=NULL file lookup required for select id,name from them where id='nothing'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: id=nothing name="" search_open: sqlite "TESTSUITE/aux-fixed/sqlitedb" cached open @@ -61,6 +64,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select * from them where id='quote2';" opts=NULL file lookup required for select * from them where id='quote2'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: name="\"stquot" id=quote2 search_open: sqlite "TESTSUITE/aux-fixed/sqlitedb" cached open @@ -71,6 +75,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select * from them where id='newline';" opts=NULL file lookup required for select * from them where id='newline'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: name="before after" id=newline search_open: sqlite "TESTSUITE/aux-fixed/sqlitedb" @@ -82,6 +87,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select * from them where id='tab';" opts=NULL file lookup required for select * from them where id='tab'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: name="x x" id=tab search_open: sqlite "TESTSUITE/aux-fixed/sqlitedb" cached open @@ -92,6 +98,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select * from them where id='its';" opts=NULL file lookup required for select * from them where id='its'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: name=it's id=its search_open: sqlite "TESTSUITE/aux-fixed/sqlitedb" cached open @@ -102,6 +109,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select * from them where name='it''s';" opts=NULL file lookup required for select * from them where name='it''s'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: name=it's id=its search_tidyup called >>>>>>>>>>>>>>>> Exim pid=pppp (fresh-exec) terminating with rc=0 >>>>>>>>>>>>>>>> @@ -247,6 +255,7 @@ internal_search_find: file="TESTSUITE/aux-fixed/sqlitedb" type=sqlite key="select * from them where id='10.10.10.10'" opts=NULL file lookup required for select * from them where id='10.10.10.10' in TESTSUITE/aux-fixed/sqlitedb +creating new cache entry lookup yielded: name=ok id=10.10.10.10 host in "sqlite;TESTSUITE/aux-fixed/sqlitedb select * from them where id='10.10.10.10'"? yes (matched "sqlite;TESTSUITE/aux-fixed/sqlitedb select * from them where id='10.10.10.10'") host in "+relay_hosts"? yes (matched "+relay_hosts") @@ -377,6 +386,7 @@ processing address_data type=sqlite key="select name from them where id='userx'" opts=NULL file lookup required for select name from them where id='userx' in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: Ayen Other calling r1 router r1 router called for userx@myhost.test.ex @@ -419,6 +429,7 @@ appendfile transport entered type=sqlite key="select id from them where id='userx'" opts=NULL file lookup required for select id from them where id='userx' in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: userx appendfile: mode=600 notify_comsat=0 quota=0 warning=0 file=TESTSUITE/test-mail/userx format=unix @@ -481,6 +492,7 @@ dropping to exim gid; retaining priv uid type=sqlite key="select name from them where id='userx';" opts=NULL file lookup required for select name from them where id='userx'; in TESTSUITE/aux-fixed/sqlitedb + creating new cache entry lookup yielded: Ayen Other search_tidyup called >>>>>>>>>>>>>>>> Exim pid=pppp (fresh-exec) terminating with rc=0 >>>>>>>>>>>>>>>> @@ -495,6 +507,7 @@ dropping to exim gid; retaining priv uid internal_search_find: file="NULL" type=sqlite key="select name from them where id='userx';" opts=NULL database lookup required for select name from them where id='userx'; + creating new cache entry lookup yielded: Ayen Other search_tidyup called >>>>>>>>>>>>>>>> Exim pid=pppp (fresh-exec) terminating with rc=0 >>>>>>>>>>>>>>>> diff --git a/test/stderr/2610 b/test/stderr/2610 index 39071aef5..4cc8801f7 100644 --- a/test/stderr/2610 +++ b/test/stderr/2610 @@ -11,6 +11,7 @@ dropping to exim gid; retaining priv uid database lookup required for select name from them where id='ph10'; MySQL query: "select name from them where id='ph10';" opts 'NULL' MYSQL new connection: host=127.0.0.1 port=1223 socket=NULL database=test user=root + creating new cache entry lookup yielded: Philip Hazel search_open: mysql "NULL" cached open @@ -32,6 +33,7 @@ dropping to exim gid; retaining priv uid MySQL query: "select name from them where id='xxxx';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found + creating new cache entry lookup failed search_open: mysql "NULL" cached open @@ -43,6 +45,7 @@ dropping to exim gid; retaining priv uid database lookup required for select name from them where id='nothing'; MySQL query: "select name from them where id='nothing';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root + creating new cache entry lookup yielded: search_open: mysql "NULL" cached open @@ -54,6 +57,7 @@ dropping to exim gid; retaining priv uid database lookup required for select id,name from them where id='nothing'; MySQL query: "select id,name from them where id='nothing';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root + creating new cache entry lookup yielded: id=nothing name="" search_open: mysql "NULL" cached open @@ -79,6 +83,7 @@ dropping to exim gid; retaining priv uid MySQL query: "select * from them where id='quote';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found + creating new cache entry lookup failed search_open: mysql "NULL" cached open @@ -91,6 +96,7 @@ dropping to exim gid; retaining priv uid MySQL query: "select * from them where id='filter';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found + creating new cache entry lookup failed search_open: mysql "NULL" cached open @@ -102,6 +108,7 @@ dropping to exim gid; retaining priv uid database lookup required for select * from them where id='quote2'; MySQL query: "select * from them where id='quote2';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root + creating new cache entry lookup yielded: name="\"stquot" id=quote2 search_open: mysql "NULL" cached open @@ -114,6 +121,7 @@ dropping to exim gid; retaining priv uid MySQL query: "select * from them where id='nlonly';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found + creating new cache entry lookup failed search_open: mysql "NULL" cached open @@ -135,6 +143,7 @@ dropping to exim gid; retaining priv uid database lookup required for servers=127.0.0.1::1223:x; select name from them where id='ph10'; MySQL query: "servers=127.0.0.1::1223:x; select name from them where id='ph10';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root + creating new cache entry lookup yielded: Philip Hazel search_open: mysql "NULL" cached open @@ -146,6 +155,7 @@ dropping to exim gid; retaining priv uid database lookup required for servers=127.0.0.1::1223/test/root/:x; select name from them where id='ph10'; MySQL query: "servers=127.0.0.1::1223/test/root/:x; select name from them where id='ph10';" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root + creating new cache entry lookup yielded: Philip Hazel search_open: mysql "NULL" cached open @@ -157,6 +167,7 @@ dropping to exim gid; retaining priv uid database lookup required for servers=ip4.ip4.ip4.ip4::1223/test/root/:127.0.0.1::1223; select name from them where id='ph10'; MySQL query: "servers=ip4.ip4.ip4.ip4::1223/test/root/:127.0.0.1::1223; select name from them where id='ph10';" opts 'NULL' MYSQL new connection: host=ip4.ip4.ip4.ip4 port=1223 socket=NULL database=test user=root + creating new cache entry lookup yielded: Philip Hazel search_open: mysql "NULL" cached open @@ -168,6 +179,7 @@ dropping to exim gid; retaining priv uid database lookup required for servers=localhost(TESTSUITE/mysql/sock)/test/root/; select name from them where id='ph10'; MySQL query: "servers=localhost(TESTSUITE/mysql/sock)/test/root/; select name from them where id='ph10';" opts 'NULL' MYSQL new connection: host=localhost port=0 socket=TESTSUITE/mysql/sock database=test user=root + creating new cache entry lookup yielded: Philip Hazel search_open: mysql "NULL" cached open @@ -179,6 +191,7 @@ dropping to exim gid; retaining priv uid database lookup required for SELECT name FROM them WHERE id IN ('ph10', 'aaaa'); MySQL query: "SELECT name FROM them WHERE id IN ('ph10', 'aaaa');" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root + creating new cache entry lookup yielded: Philip Hazel Aristotle search_open: mysql "NULL" @@ -191,6 +204,7 @@ dropping to exim gid; retaining priv uid database lookup required for SELECT * FROM them WHERE id IN ('ph10', 'aaaa'); MySQL query: "SELECT * FROM them WHERE id IN ('ph10', 'aaaa');" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root + creating new cache entry lookup yielded: name="Philip Hazel" id=ph10 name=Aristotle id=aaaa search_open: mysql "NULL" @@ -254,6 +268,7 @@ processing "warn" (TESTSUITE/test-config 25) MySQL query: "select name from them where id = 'c'" opts 'NULL' MYSQL new connection: host=127.0.0.1 port=1223 socket=NULL database=test user=root MYSQL: no data found + creating new cache entry lookup failed check set acl_m0 = ok: ${lookup mysql {select name from them where id = '$local_part'}} = ok: @@ -268,6 +283,7 @@ check set acl_m0 = ok: ${lookup mysql {select name from the MySQL query: "select name from them where id = 'c'" opts 'servers=127.0.0.1::1223/test/root/' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found + replacing old cache entry lookup failed check set acl_m0 = ok: ${lookup mysql,servers=127.0.0.1::1223/test/root/ {select name from them where id = '$local_part'}} = ok: @@ -282,6 +298,7 @@ check set acl_m0 = ok: ${lookup mysql,servers=127.0.0.1::1223/test/root/ MySQL query: "select name from them where id = 'c'" opts 'servers=127.0.0.1::1223' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found + replacing old cache entry lookup failed check set acl_m0 = ok: ${lookup mysql,servers=127.0.0.1::1223 {select name from them where id = '$local_part'}} = ok: @@ -312,6 +329,7 @@ database lookup required for select * from them where id='c' MySQL query: "select * from them where id='c'" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found +creating new cache entry lookup failed host in "net-mysql;select * from them where id='c'"? no (end of list) warn: condition test failed in ACL "check_recipient" @@ -367,6 +385,7 @@ database lookup required for select * from them where id='10.0.0.0' MySQL query: "select * from them where id='10.0.0.0'" opts 'NULL' MYSQL using cached connection for 127.0.0.1:1223/test/root MYSQL: no data found +creating new cache entry lookup failed host in "net-mysql;select * from them where id='10.0.0.0'"? no (end of list) host in "+relay_hosts"? no (end of list) @@ -480,6 +499,7 @@ processing address_data database lookup required for select name from them where id='ph10' MySQL query: "select name from them where id='ph10'" opts 'NULL' MYSQL new connection: host=127.0.0.1 port=1223 socket=NULL database=test user=root + creating new cache entry lookup yielded: Philip Hazel calling r1 router r1 router called for ph10@myhost.test.ex @@ -524,6 +544,7 @@ appendfile transport entered database lookup required for select id from them where id='ph10' MySQL query: "select id from them where id='ph10'" opts 'NULL' MYSQL new connection: host=127.0.0.1 port=1223 socket=NULL database=test user=root + creating new cache entry lookup yielded: ph10 appendfile: mode=600 notify_comsat=0 quota=0 warning=0 file=TESTSUITE/test-mail/ph10 format=unix diff --git a/test/stderr/2620 b/test/stderr/2620 index 8c610d87e..eee26190d 100644 --- a/test/stderr/2620 +++ b/test/stderr/2620 @@ -15,6 +15,7 @@ dropping to exim gid; retaining priv uid database lookup required for select name from them where id='ph10'; PostgreSQL query: "select name from them where id='ph10';" opts 'NULL' PGSQL new connection: host=localhost port=1223 database=test user=CALLER + creating new cache entry lookup yielded: Philip Hazel search_open: pgsql "NULL" cached open @@ -36,6 +37,7 @@ dropping to exim gid; retaining priv uid PostgreSQL query: "select name from them where id='xxxx';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER PGSQL: no data found + creating new cache entry lookup failed search_open: pgsql "NULL" cached open @@ -47,6 +49,7 @@ dropping to exim gid; retaining priv uid database lookup required for select name from them where id='nothing'; PostgreSQL query: "select name from them where id='nothing';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: search_open: pgsql "NULL" cached open @@ -58,6 +61,7 @@ dropping to exim gid; retaining priv uid database lookup required for select id,name from them where id='nothing'; PostgreSQL query: "select id,name from them where id='nothing';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: id=nothing name="" search_open: pgsql "NULL" cached open @@ -82,6 +86,7 @@ dropping to exim gid; retaining priv uid database lookup required for select * from them where id='quote2'; PostgreSQL query: "select * from them where id='quote2';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: name="\"stquot" id=quote2 search_open: pgsql "NULL" cached open @@ -93,6 +98,7 @@ dropping to exim gid; retaining priv uid database lookup required for select * from them where id='newline'; PostgreSQL query: "select * from them where id='newline';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: name="before after" id=newline search_open: pgsql "NULL" @@ -105,6 +111,7 @@ dropping to exim gid; retaining priv uid database lookup required for select * from them where id='tab'; PostgreSQL query: "select * from them where id='tab';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: name="x x" id=tab search_open: pgsql "NULL" cached open @@ -116,6 +123,7 @@ dropping to exim gid; retaining priv uid database lookup required for select * from them where name='''stquot'; PostgreSQL query: "select * from them where name='''stquot';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: name='stquot id=quote1 search_open: pgsql "NULL" cached open @@ -137,6 +145,7 @@ dropping to exim gid; retaining priv uid database lookup required for servers=localhost::1223:x; select name from them where id='ph10'; PostgreSQL query: "servers=localhost::1223:x; select name from them where id='ph10';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: Philip Hazel search_open: pgsql "NULL" cached open @@ -148,6 +157,7 @@ dropping to exim gid; retaining priv uid database lookup required for servers=localhost::1223/test/CALLER/:x; select name from them where id='ph10'; PostgreSQL query: "servers=localhost::1223/test/CALLER/:x; select name from them where id='ph10';" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: Philip Hazel search_open: pgsql "NULL" cached open @@ -159,6 +169,7 @@ dropping to exim gid; retaining priv uid database lookup required for servers=(TESTSUITE/pgsql/.s.PGSQL.1223)/test/CALLER/:x; select name from them where id='ph10'; PostgreSQL query: "servers=(TESTSUITE/pgsql/.s.PGSQL.1223)/test/CALLER/:x; select name from them where id='ph10';" opts 'NULL' PGSQL new connection: socket=TESTSUITE/pgsql/.s.PGSQL.1223 database=test user=CALLER + creating new cache entry lookup yielded: Philip Hazel search_open: pgsql "NULL" cached open @@ -170,6 +181,7 @@ dropping to exim gid; retaining priv uid database lookup required for SELECT name FROM them WHERE id IN ('ph10', 'aaaa'); PostgreSQL query: "SELECT name FROM them WHERE id IN ('ph10', 'aaaa');" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: Philip Hazel Aristotle search_open: pgsql "NULL" @@ -182,6 +194,7 @@ dropping to exim gid; retaining priv uid database lookup required for SELECT * FROM them WHERE id IN ('ph10', 'aaaa'); PostgreSQL query: "SELECT * FROM them WHERE id IN ('ph10', 'aaaa');" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER + creating new cache entry lookup yielded: name="Philip Hazel" id=ph10 name=Aristotle id=aaaa search_open: pgsql "NULL" @@ -244,6 +257,7 @@ processing "warn" (TESTSUITE/test-config 27) PostgreSQL query: "select name from them where id = 'c'" opts 'NULL' PGSQL new connection: host=localhost port=1223 database=test user=CALLER PGSQL: no data found + creating new cache entry lookup failed check set acl_m0 = ok: ${lookup pgsql {select name from them where id = '$local_part'}} = ok: @@ -274,6 +288,7 @@ database lookup required for select * from them where id='c' PostgreSQL query: "select * from them where id='c'" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER PGSQL: no data found +creating new cache entry lookup failed host in "net-pgsql;select * from them where id='c'"? no (end of list) warn: condition test failed in ACL "check_recipient" @@ -329,6 +344,7 @@ database lookup required for select * from them where id='10.0.0.0' PostgreSQL query: "select * from them where id='10.0.0.0'" opts 'NULL' PGSQL using cached connection for localhost:1223/test/CALLER PGSQL: no data found +creating new cache entry lookup failed host in "net-pgsql;select * from them where id='10.0.0.0'"? no (end of list) host in "+relay_hosts"? no (end of list) @@ -543,6 +559,7 @@ processing address_data database lookup required for select name from them where id='ph10' PostgreSQL query: "select name from them where id='ph10'" opts 'NULL' PGSQL new connection: host=localhost port=1223 database=test user=CALLER + creating new cache entry lookup yielded: Philip Hazel calling r1 router r1 router called for CALLER@myhost.test.ex @@ -587,6 +604,7 @@ appendfile transport entered database lookup required for select id from them where id='ph10' PostgreSQL query: "select id from them where id='ph10'" opts 'NULL' PGSQL new connection: host=localhost port=1223 database=test user=CALLER + creating new cache entry lookup yielded: ph10 appendfile: mode=600 notify_comsat=0 quota=0 warning=0 file=TESTSUITE/test-mail/ph10 format=unix @@ -651,6 +669,7 @@ dropping to exim gid; retaining priv uid database lookup required for select name from them where id='ph10'; PostgreSQL query: "select name from them where id='ph10';" opts 'NULL' PGSQL new connection: socket=TESTSUITE/pgsql/.s.PGSQL.1223 database=test user=CALLER + creating new cache entry lookup yielded: Philip Hazel search_tidyup called close PGSQL connection: (TESTSUITE/pgsql/.s.PGSQL.1223)/test/CALLER diff --git a/test/stderr/3201 b/test/stderr/3201 index 32bf69ca6..601a9ec85 100644 --- a/test/stderr/3201 +++ b/test/stderr/3201 @@ -31,6 +31,7 @@ internal_search_find: file="NULL" type=testdb key="fail" opts=NULL database lookup required for fail testdb lookup forced FAIL +creating new cache entry lookup failed host in "testdb;fail"? no (end of list) deny: condition test failed in ACL "connect1" @@ -73,6 +74,7 @@ internal_search_find: file="NULL" type=testdb key="fail" opts=NULL database lookup required for fail testdb lookup forced FAIL +creating new cache entry lookup failed host in "net-testdb;fail"? no (end of list) deny: condition test failed in ACL "connect2" diff --git a/test/stderr/3212 b/test/stderr/3212 index 1c944c24c..7b7719801 100644 --- a/test/stderr/3212 +++ b/test/stderr/3212 @@ -16,6 +16,7 @@ sender address = CALLER@myhost.test.ex type=lsearch key="list" opts=NULL file lookup required for list in TESTSUITE/aux-fixed/3212.aliases + creating new cache entry lookup yielded: userx, usery search_open: lsearch "TESTSUITE/aux-fixed/3212.aliases" cached open @@ -40,6 +41,7 @@ sender address = CALLER@myhost.test.ex type=lsearch key="root" opts=NULL file lookup required for root in TESTSUITE/aux-fixed/3212.aliases + creating new cache entry lookup yielded: userx search_open: lsearch "TESTSUITE/aux-fixed/3212.aliases" cached open @@ -74,6 +76,7 @@ sender address = CALLER@myhost.test.ex internal_search_find: file="NULL" type=testdb key="something" opts=NULL database lookup required for something + creating new cache entry lookup yielded: something search_open: testdb "NULL" cached open @@ -108,6 +111,7 @@ sender address = CALLER@myhost.test.ex internal_search_find: file="NULL" type=testdb key="something" opts=NULL database lookup required for something + creating new cache entry lookup yielded: something search_open: lsearch "TESTSUITE/aux-fixed/3212.aliases" cached open diff --git a/test/stdout/2500 b/test/stdout/2500 index 7e413c0a3..c6014d388 100644 --- a/test/stdout/2500 +++ b/test/stdout/2500 @@ -1,3 +1,4 @@ +> dsearch specifics > ok: 2500.tst > fail: FAIL > Failed: failed to open TESTSUITE/dir_not_here for directory search: No such file or directory @@ -14,3 +15,13 @@ > fail,subdir:FAIL > fail,subdir:FAIL > +> cachelayer tests +> fail: FAIL +> createfile: OK +> fail,cached:FAIL +> ok,no_rd test-data +> delfile: OK +> ok,cached: test-data +> fail,no_rd FAIL +> fail: FAIL +> -- cgit v1.2.3 From 5fbb51949cbd9765421c5832c2d391f80f8183e9 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 25 May 2020 22:57:57 +0100 Subject: Quota checking during reception. Bug 163 --- doc/doc-docbook/spec.xfpt | 51 ++++++ doc/doc-txt/NewStuff | 3 + src/src/acl.c | 85 ++++++++- src/src/child.c | 2 +- src/src/deliver.c | 2 +- src/src/exim.c | 21 +++ src/src/functions.h | 4 + src/src/globals.h | 1 + src/src/macros.h | 1 + src/src/smtp_in.c | 3 +- src/src/transports/appendfile.c | 12 ++ src/src/verify.c | 377 +++++++++++++++++++++++++++++++++------- test/confs/0589 | 60 +++++++ test/log/0589 | 17 ++ test/rejectlog/0589 | 6 + test/scripts/0000-Basic/0589 | 44 +++++ test/stdout/0589 | 61 +++++++ 17 files changed, 680 insertions(+), 70 deletions(-) create mode 100644 test/confs/0589 create mode 100644 test/log/0589 create mode 100644 test/rejectlog/0589 create mode 100644 test/scripts/0000-Basic/0589 create mode 100644 test/stdout/0589 (limited to 'doc/doc-docbook') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index a1f361339..68f3b5b5a 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -3906,6 +3906,13 @@ together with the file descriptor number of an open pipe. Closure of the pipe signals the final completion of the sequence of processes that are passing messages through the same SMTP connection. +.new +.vitem &%-MCq%&&~<&'recipient&~address'&>&~<&'size'&> +.oindex "&%-MCq%&" +This option is not intended for use by external callers. It is used internally +by Exim to implement quota checking for local users. +.wen + .vitem &%-MCS%& .oindex "&%-MCS%&" This option is not intended for use by external callers. It is used internally @@ -32740,6 +32747,15 @@ The &%success_on_redirect%& option causes verification always to succeed immediately after a successful redirection. By default, if a redirection generates just one address, that address is also verified. See further discussion in section &<>&. +.new +.next +If the &%quota%& option is specified for recipient verify, +successful routing to an appendfile transport is followed by a call into +the transport to evaluate the quota status for the recipient. +No actual delivery is done, but verification will succeed if the quota +is sufficient for the message (if the sender gave a message size) or +not already exceeded (otherwise). +.wen .endlist .cindex "verifying address" "differentiating failures" @@ -32773,6 +32789,9 @@ connection, HELO, or MAIL). &%recipient%&: The RCPT command in a callout was rejected. .next &%postmaster%&: The postmaster check in a callout was rejected. +.new +.next +&%quota%&: The quota check for a local recipient did non pass. .endlist The main use of these variables is expected to be to distinguish between @@ -33102,6 +33121,38 @@ behaviour will be the same. +.new +.section "Quota caching" "SECTquotacache" +.cindex "hints database" "quota cache" +.cindex "quota" "cache, description of" +.cindex "caching" "quota" +Exim caches the results of quota verification +in order to reduce the amount of resources used. +The &"callout"& hints database is used. + +The default cache periods are five minutes for a positive (good) result +and one hour for a negative result. +To change the periods the &%quota%& option can be followed by an equals sign +and a number of optional paramemters, separated by commas. +For example: +.code +verify = recipient/quota=cachepos=1h,cacheneg=1d +.endd +Possible parameters are: +.vlist +.vitem &*cachepos&~=&~*&<&'time&~interval'&> +.cindex "quota cache" "positive entry expiry, specifying" +Set the lifetime for a positive cache entry. +A value of zero seconds is legitimate. + +.vitem &*cacheneg&~=&~*&<&'time&~interval'&> +.cindex "quota cache" "negative entry expiry, specifying" +As above, for a negative entry. + +.vitem &*no_cache*& +Set both positive and negative lifetimes to zero. +.wen + .section "Sender address verification reporting" "SECTsenaddver" .cindex "verifying" "suppressing error details" See section &<>& for a general discussion of diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 83b696fe5..9797e97db 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -26,6 +26,9 @@ Version 4.95 6. An option to ignore the cache on a lookup. + 7. Quota checking during reception (i.e. at SMTP time) for appendfile- + transport-managed quotas. + Version 4.94 ------------ diff --git a/src/src/acl.c b/src/src/acl.c index 24716f0d1..57a07296f 100644 --- a/src/src/acl.c +++ b/src/src/acl.c @@ -15,6 +15,12 @@ #define CALLOUT_TIMEOUT_DEFAULT 30 +/* Default quota cache TTLs */ + +#define QUOTA_POS_DEFAULT (5*60) +#define QUOTA_NEG_DEFAULT (60*60) + + /* ACL verb codes - keep in step with the table of verbs that follows */ enum { ACL_ACCEPT, ACL_DEFER, ACL_DENY, ACL_DISCARD, ACL_DROP, ACL_REQUIRE, @@ -1556,6 +1562,20 @@ static callout_opt_t callout_opt_list[] = { +static int +v_period(const uschar * s, const uschar * arg, uschar ** log_msgptr) +{ +int period; +if ((period = readconf_readtime(s, 0, FALSE)) < 0) + { + *log_msgptr = string_sprintf("bad time value in ACL condition " + "\"verify %s\"", arg); + } +return period; +} + + + /* This function implements the "verify" condition. It is called when encountered in any ACL, because some tests are almost always permitted. Some just don't make sense, and always fail (for example, an attempt to test a host @@ -1590,6 +1610,8 @@ BOOL defer_ok = FALSE; BOOL callout_defer_ok = FALSE; BOOL no_details = FALSE; BOOL success_on_redirect = FALSE; +BOOL quota = FALSE; +int quota_pos_cache = QUOTA_POS_DEFAULT, quota_neg_cache = QUOTA_NEG_DEFAULT; address_item *sender_vaddr = NULL; uschar *verify_sender_address = NULL; uschar *pm_mailfrom = NULL; @@ -1823,12 +1845,8 @@ while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size))) } while (isspace(*opt)) opt++; } - if (op->timeval && (period = readconf_readtime(opt, 0, FALSE)) < 0) - { - *log_msgptr = string_sprintf("bad time value in ACL condition " - "\"verify %s\"", arg); + if (op->timeval && (period = v_period(opt, arg, log_msgptr)) < 0) return ERROR; - } switch(op->value) { @@ -1861,6 +1879,38 @@ while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size))) } } + /* The quota option has sub-options, comma-separated */ + + else if (strncmpic(ss, US"quota", 5) == 0) + { + quota = TRUE; + if (*(ss += 5)) + { + while (isspace(*ss)) ss++; + if (*ss++ == '=') + { + const uschar * sublist = ss; + int optsep = ','; + int period; + + while (isspace(*sublist)) sublist++; + for (uschar * opt; opt = string_nextinlist(&sublist, &optsep, NULL, 0); ) + if (Ustrncmp(opt, "cachepos=", 9) == 0) + if ((period = v_period(opt += 9, arg, log_msgptr)) < 0) + return ERROR; + else + quota_pos_cache = period; + else if (Ustrncmp(opt, "cacheneg=", 9) == 0) + if ((period = v_period(opt += 9, arg, log_msgptr)) < 0) + return ERROR; + else + quota_neg_cache = period; + else if (Ustrcmp(opt, "no_cache") == 0) + quota_pos_cache = quota_neg_cache = 0; + } + } + } + /* Option not recognized */ else @@ -1879,6 +1929,31 @@ if ((verify_options & (vopt_callout_recipsender|vopt_callout_recippmaster)) == return ERROR; } +/* Handle quota verification */ +if (quota) + { + if (vp->value != VERIFY_RCPT) + { + *log_msgptr = US"can only verify quota of recipient"; + return ERROR; + } + + if ((rc = verify_quota_call(addr->address, + quota_pos_cache, quota_neg_cache, log_msgptr)) != OK) + { + *basic_errno = errno; + if (smtp_return_error_details) + { + if (!*user_msgptr && *log_msgptr) + *user_msgptr = string_sprintf("Rejected after %s: %s", + smtp_names[smtp_connection_had[smtp_ch_index-1]], *log_msgptr); + if (rc == DEFER) f.acl_temp_details = TRUE; + } + } + + return rc; + } + /* Handle sender-in-header verification. Default the user message to the log message if giving out verification details. */ diff --git a/src/src/child.c b/src/src/child.c index b36a96fe1..1407b3718 100644 --- a/src/src/child.c +++ b/src/src/child.c @@ -27,7 +27,7 @@ Arguments: Returns: nothing */ -static void +void force_fd(int oldfd, int newfd) { if (oldfd == newfd) return; diff --git a/src/src/deliver.c b/src/src/deliver.c index acb87bca6..dd922c728 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -2125,7 +2125,7 @@ Arguments: Returns: nothing */ -static void +void deliver_local(address_item *addr, BOOL shadowing) { BOOL use_initgroups; diff --git a/src/src/exim.c b/src/src/exim.c index 905ef0a70..4e7aae69a 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -1618,6 +1618,7 @@ BOOL removed_privilege = FALSE; BOOL usage_wanted = FALSE; BOOL verify_address_mode = FALSE; BOOL verify_as_sender = FALSE; +BOOL rcpt_verify_quota = FALSE; BOOL version_printed = FALSE; uschar *alias_arg = NULL; uschar *called_as = US""; @@ -2790,6 +2791,13 @@ on the second character (the one after '-'), to save some effort. */ else badarg = TRUE; break; + /* -MCq: do a quota check on the given recipient for the given size + of message. Separate from -MC. */ + case 'q': rcpt_verify_quota = TRUE; + if (++i < argc) message_size = Uatoi(argv[i]); + else badarg = TRUE; + break; + /* -MCS: set the smtp_use_size flag; this is useful only when it precedes -MC (see above) */ @@ -4296,6 +4304,7 @@ if ( !unprivileged /* originally had root AND */ || msg_action != MSG_DELIVER /* not delivering */ ) /* and */ && (!checking || !f.address_test_mode) /* not address checking */ + && !rcpt_verify_quota /* and not quota checking */ ) ) ) exim_setugid(exim_uid, exim_gid, TRUE, US"privilege not needed"); @@ -4414,6 +4423,18 @@ needed in transports so we lost the optimisation. */ #endif } +/* Handle a request to check quota */ +if (rcpt_verify_quota) + if (real_uid != root_uid && real_uid != exim_uid) + exim_fail("exim: Permission denied\n"); + else if (recipients_arg >= argc) + exim_fail("exim: missing recipient for quota check\n"); + else + { + verify_quota(argv[recipients_arg]); + exim_exit(EXIT_SUCCESS); + } + /* Handle the -brt option. This is for checking out retry configurations. The next three arguments are a domain name or a complete address, and optionally two error numbers. All it does is to call the function that diff --git a/src/src/functions.h b/src/src/functions.h index 2a4597202..486a91595 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -189,6 +189,7 @@ extern void debug_print_socket(int); extern void decode_bits(unsigned int *, size_t, int *, uschar *, bit_table *, int, uschar *, int); extern void delete_pid_file(void); +extern void deliver_local(address_item *, BOOL); extern address_item *deliver_make_addr(uschar *, BOOL); extern void delivery_log(int, address_item *, int, uschar *); extern int deliver_message(uschar *, BOOL, BOOL); @@ -260,6 +261,7 @@ extern BOOL filter_runtest(int, uschar *, BOOL, BOOL); extern BOOL filter_system_interpret(address_item **, uschar **); extern uschar * fn_hdrs_added(void); +extern void force_fd(int, int); extern void header_add(int, const char *, ...); extern header_line *header_add_at_position_internal(BOOL, uschar *, BOOL, int, const char *, ...); @@ -620,6 +622,8 @@ extern int verify_check_this_host(const uschar **, unsigned int *, const uschar*, const uschar *, const uschar **); extern address_item *verify_checked_sender(uschar *); extern void verify_get_ident(int); +extern void verify_quota(uschar *); +extern int verify_quota_call(const uschar *, int, int, uschar **); extern BOOL verify_sender(int *, uschar **); extern BOOL verify_sender_preliminary(int *, uschar **); extern void version_init(void); diff --git a/src/src/globals.h b/src/src/globals.h index 1bdf338a2..3dfbbc2b1 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -943,6 +943,7 @@ extern int smtp_load_reserve; /* Only from reserved if load > this */ extern int smtp_mailcmd_count; /* Count of MAIL commands */ extern int smtp_max_synprot_errors;/* Max syntax/protocol errors */ extern int smtp_max_unknown_commands; /* As it says */ +extern uschar *smtp_names[]; /* decode for command codes */ extern uschar *smtp_notquit_reason; /* Global for disconnect reason */ extern FILE *smtp_out; /* Incoming SMTP output file */ extern uschar *smtp_ratelimit_hosts; /* Rate limit these hosts */ diff --git a/src/src/macros.h b/src/src/macros.h index a507bbf83..f6012447d 100644 --- a/src/src/macros.h +++ b/src/src/macros.h @@ -736,6 +736,7 @@ enum { v_none, v_sender, v_recipient, v_expn }; #define vopt_callout_recippmaster 0x0100 /* use postmaster to verify recip */ #define vopt_callout_hold 0x0200 /* lazy close connection */ #define vopt_success_on_redirect 0x0400 +#define vopt_quota 0x0800 /* quota check, to local/appendfile */ /* Values for fields in callout cache records */ diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index 2b4323bec..ce27c5f3e 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -227,7 +227,7 @@ static smtp_cmd_list *cmd_list_end = /* This list of names is used for performing the smtp_no_mail logging action. It must be kept in step with the SCH_xxx enumerations. */ -static uschar *smtp_names[] = +uschar * smtp_names[] = { US"NONE", US"AUTH", US"DATA", US"BDAT", US"EHLO", US"ETRN", US"EXPN", US"HELO", US"HELP", US"MAIL", US"NOOP", US"QUIT", US"RCPT", US"RSET", @@ -2058,6 +2058,7 @@ sending_ip_address = NULL; return_path = sender_address = NULL; deliver_localpart_data = deliver_domain_data = recipient_data = sender_data = NULL; /* Can be set by ACL */ +recipient_verify_failure = NULL; deliver_localpart_parent = deliver_localpart_orig = NULL; deliver_domain_parent = deliver_domain_orig = NULL; callout_address = NULL; diff --git a/src/src/transports/appendfile.c b/src/src/transports/appendfile.c index f07f3bacb..a621de074 100644 --- a/src/src/transports/appendfile.c +++ b/src/src/transports/appendfile.c @@ -2740,6 +2740,18 @@ if (!disable_quota && ob->quota_value > 0) } +if (verify_mode) + { + addr->basic_errno = errno; + addr->message = US"Over quota"; + addr->transport_return = yield; + DEBUG(D_transport) + debug_printf("appendfile (verify) yields %d with errno=%d more_errno=%d\n", + yield, addr->basic_errno, addr->more_errno); + + goto RETURN; + } + /* If we are writing in MBX format, what we actually do is to write the message to a temporary file, and then copy it to the real file once we know its size. This is the most straightforward way of getting the correct length in the diff --git a/src/src/verify.c b/src/src/verify.c index 7f1752120..3a40cea26 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -568,6 +568,7 @@ if (!addr->transport) { HDEBUG(D_verify) debug_printf("cannot callout via null transport\n"); } + else if (Ustrcmp(addr->transport->driver_name, "smtp") != 0) log_write(0, LOG_MAIN|LOG_PANIC|LOG_CONFIG_FOR, "callout transport '%s': %s is non-smtp", addr->transport->name, addr->transport->driver_name); @@ -1846,6 +1847,8 @@ while (addr_new) if (rc == OK) { + BOOL local_verify = FALSE; + if (routed) *routed = TRUE; if (callout > 0) { @@ -1872,72 +1875,76 @@ while (addr_new) transport's options, so as to mimic what would happen if we were really sending a message to this address. */ - if ((tp = addr->transport) && !tp->info->local) - { - (void)(tp->setup)(tp, addr, &tf, 0, 0, NULL); + if ((tp = addr->transport)) + if (!tp->info->local) + { + (void)(tp->setup)(tp, addr, &tf, 0, 0, NULL); - /* If the transport has hosts and the router does not, or if the - transport is configured to override the router's hosts, we must build a - host list of the transport's hosts, and find the IP addresses */ + /* If the transport has hosts and the router does not, or if the + transport is configured to override the router's hosts, we must build a + host list of the transport's hosts, and find the IP addresses */ - if (tf.hosts && (!host_list || tf.hosts_override)) - { - uschar *s; - const uschar *save_deliver_domain = deliver_domain; - uschar *save_deliver_localpart = deliver_localpart; - - host_list = NULL; /* Ignore the router's hosts */ - - deliver_domain = addr->domain; - deliver_localpart = addr->local_part; - s = expand_string(tf.hosts); - deliver_domain = save_deliver_domain; - deliver_localpart = save_deliver_localpart; - - if (!s) - { - log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand list of hosts " - "\"%s\" in %s transport for callout: %s", tf.hosts, - tp->name, expand_string_message); - } - else - { - int flags; - host_build_hostlist(&host_list, s, tf.hosts_randomize); - - /* Just ignore failures to find a host address. If we don't manage - to find any addresses, the callout will defer. Note that more than - one address may be found for a single host, which will result in - additional host items being inserted into the chain. Hence we must - save the next host first. */ - - flags = HOST_FIND_BY_A | HOST_FIND_BY_AAAA; - if (tf.qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE; - if (tf.search_parents) flags |= HOST_FIND_SEARCH_PARENTS; - - for (host_item * host = host_list, * nexthost; host; host = nexthost) - { - nexthost = host->next; - if (tf.gethostbyname || - string_is_ip_address(host->name, NULL) != 0) - (void)host_find_byname(host, NULL, flags, NULL, TRUE); - else + if (tf.hosts && (!host_list || tf.hosts_override)) + { + uschar *s; + const uschar *save_deliver_domain = deliver_domain; + uschar *save_deliver_localpart = deliver_localpart; + + host_list = NULL; /* Ignore the router's hosts */ + + deliver_domain = addr->domain; + deliver_localpart = addr->local_part; + s = expand_string(tf.hosts); + deliver_domain = save_deliver_domain; + deliver_localpart = save_deliver_localpart; + + if (!s) + { + log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand list of hosts " + "\"%s\" in %s transport for callout: %s", tf.hosts, + tp->name, expand_string_message); + } + else + { + int flags; + host_build_hostlist(&host_list, s, tf.hosts_randomize); + + /* Just ignore failures to find a host address. If we don't manage + to find any addresses, the callout will defer. Note that more than + one address may be found for a single host, which will result in + additional host items being inserted into the chain. Hence we must + save the next host first. */ + + flags = HOST_FIND_BY_A | HOST_FIND_BY_AAAA; + if (tf.qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE; + if (tf.search_parents) flags |= HOST_FIND_SEARCH_PARENTS; + + for (host_item * host = host_list, * nexthost; host; host = nexthost) { - const dnssec_domains * dsp = NULL; - if (Ustrcmp(tp->driver_name, "smtp") == 0) + nexthost = host->next; + if (tf.gethostbyname || + string_is_ip_address(host->name, NULL) != 0) + (void)host_find_byname(host, NULL, flags, NULL, TRUE); + else { - smtp_transport_options_block * ob = - (smtp_transport_options_block *) tp->options_block; - dsp = &ob->dnssec; - } + const dnssec_domains * dsp = NULL; + if (Ustrcmp(tp->driver_name, "smtp") == 0) + { + smtp_transport_options_block * ob = + (smtp_transport_options_block *) tp->options_block; + dsp = &ob->dnssec; + } - (void) host_find_bydns(host, NULL, flags, NULL, NULL, NULL, - dsp, NULL, NULL); + (void) host_find_bydns(host, NULL, flags, NULL, NULL, NULL, + dsp, NULL, NULL); + } } - } - } - } - } + } + } + } + else if ( options & vopt_quota + && Ustrcmp(tp->driver_name, "appendfile") == 0) + local_verify = TRUE; /* Can only do a callout if we have at least one host! If the callout fails, it will have set ${sender,recipient}_verify_failure. */ @@ -1963,11 +1970,17 @@ while (addr_new) #endif } } + else if (local_verify) + { + HDEBUG(D_verify) debug_printf("Attempting quota verification\n"); + + deliver_set_expansions(addr); + deliver_local(addr, TRUE); + rc = addr->transport_return; + } else - { HDEBUG(D_verify) debug_printf("Cannot do callout: neither router nor " "transport provided a host list, or transport is not smtp\n"); - } } } @@ -3919,6 +3932,246 @@ while ((domain = string_nextinlist(&list, &sep, NULL, 0))) return FAIL; } + + +/**************************************************** + Verify a local user account for quota sufficiency +****************************************************/ + +/* The real work, done via a re-exec for privs, calls +down to the transport for the quota check. + +Route and transport (in recipient-verify mode) the +given recipient. + +A routing result indicating any transport type other than appendfile +results in a fail. + +Return, on stdout, a result string containing: +- highlevel result code (OK, DEFER, FAIL) +- errno +- where string +- message string +*/ + +void +verify_quota(uschar * address) +{ +address_item vaddr = {.address = address}; +BOOL routed; +uschar * msg = US"\0"; +int rc, len = 1; + +if ((rc = verify_address(&vaddr, NULL, vopt_is_recipient | vopt_quota, + 1, 0, 0, NULL, NULL, &routed)) != OK) + { + uschar * where = recipient_verify_failure; + msg = acl_verify_message ? acl_verify_message : vaddr.message; + if (!msg) msg = US""; + if (rc == DEFER && vaddr.basic_errno == ERRNO_EXIMQUOTA) + { + rc = FAIL; /* DEFER -> FAIL */ + where = US"quota"; + vaddr.basic_errno = 0; + } + else if (!where) where = US""; + + len = 5 + Ustrlen(msg) + 1 + Ustrlen(where); + msg = string_sprintf("%c%c%c%c%c%s%c%s", (uschar)rc, + (vaddr.basic_errno >> 24) && 0xff, (vaddr.basic_errno >> 16) && 0xff, + (vaddr.basic_errno >> 8) && 0xff, vaddr.basic_errno && 0xff, + where, '\0', msg); + } + +DEBUG(D_verify) debug_printf_indent("verify_quota: len %d\n", len); +write(1, msg, len); +return; +} + + +/******************************************************************************/ + +/* Quota cache lookup. We use the callout hints db also for the quota cache. +Return TRUE if a nonexpired record was found, having filled in the yield +argument. +*/ + +static BOOL +cached_quota_lookup(const uschar * rcpt, int * yield, + int pos_cache, int neg_cache) +{ +open_db dbblock, *dbm_file = NULL; +dbdata_callout_cache_address * cache_address_record; + +if (!pos_cache && !neg_cache) + return FALSE; +if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE, TRUE))) + { + HDEBUG(D_verify) debug_printf_indent("quota cache: not available\n"); + return FALSE; + } +if (!(cache_address_record = (dbdata_callout_cache_address *) + get_callout_cache_record(dbm_file, rcpt, US"address", + pos_cache, neg_cache))) + { + dbfn_close(dbm_file); + return FALSE; + } +if (cache_address_record->result == ccache_accept) + *yield = OK; +dbfn_close(dbm_file); +return TRUE; +} + +/* Quota cache write */ + +static void +cache_quota_write(const uschar * rcpt, int yield, int pos_cache, int neg_cache) +{ +open_db dbblock, *dbm_file = NULL; +dbdata_callout_cache_address cache_address_record; + +if (!pos_cache && !neg_cache) + return; +if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE, TRUE))) + { + HDEBUG(D_verify) debug_printf_indent("quota cache: not available\n"); + return; + } + +cache_address_record.result = yield == OK ? ccache_accept : ccache_reject; + +(void)dbfn_write(dbm_file, rcpt, &cache_address_record, + (int)sizeof(dbdata_callout_cache_address)); +HDEBUG(D_verify) debug_printf_indent("wrote %s quota cache record for %s\n", + yield == OK ? "positive" : "negative", rcpt); + +dbfn_close(dbm_file); +return; +} + + +/* To evaluate a local user's quota, starting in ACL, we need to +fork & exec to regain privileges, to that we can change to the user's +identity for access to their files. + +Arguments: + rcpt Recipient account + pos_cache Number of seconds to cache a positive result (delivery + to be accepted). Zero to disable caching. + neg_cache Number of seconds to cache a negative result. Zero to disable. + msg Pointer to result string pointer + +Return: OK/DEFER/FAIL code +*/ + +int +verify_quota_call(const uschar * rcpt, int pos_cache, int neg_cache, + uschar ** msg) +{ +int pfd[2], pid, save_errno, yield = FAIL; +void (*oldsignal)(int); +const uschar * where = US"socketpair"; + +*msg = NULL; + +if (cached_quota_lookup(rcpt, &yield, pos_cache, neg_cache)) + { + HDEBUG(D_verify) debug_printf_indent("quota cache: address record is %d\n", + yield == OK ? "positive" : "negative"); + if (yield != OK) + { + recipient_verify_failure = US"quota"; + acl_verify_message = *msg = + US"Previous (cached) quota verification failure"; + } + return yield; + } + +if (pipe(pfd) != 0) + goto fail; + +where = US"fork"; +oldsignal = signal(SIGCHLD, SIG_DFL); +if ((pid = exim_fork(US"quota-verify")) < 0) + { + save_errno = errno; + close(pfd[pipe_write]); + close(pfd[pipe_read]); + errno = save_errno; + goto fail; + } + +if (pid == 0) /* child */ + { + close(pfd[pipe_read]); + force_fd(pfd[pipe_write], 1); /* stdout to pipe */ + close(pfd[pipe_write]); + dup2(1, 0); + if (debug_fd > 0) force_fd(debug_fd, 2); + + child_exec_exim(CEE_EXEC_EXIT, FALSE, NULL, FALSE, 3, + US"-MCq", string_sprintf("%d", message_size), rcpt); + /*NOTREACHED*/ + } + +save_errno = errno; +close(pfd[pipe_write]); + +if (pid < 0) + { + DEBUG(D_verify) debug_printf_indent(" fork: %s\n", strerror(save_errno)); + } +else + { + uschar buf[128]; + int n = read(pfd[pipe_read], buf, sizeof(buf)); + int status; + + waitpid(pid, &status, 0); + if (status == 0) + { + uschar * s; + + if (n > 0) yield = buf[0]; + if (n > 4) + save_errno = (buf[1] << 24) | (buf[2] << 16) | (buf[3] << 8) | buf[4]; + if ((recipient_verify_failure = n > 5 + ? string_copyn_taint(buf+5, n-5, FALSE) : NULL)) + { + int m; + s = buf + 5 + Ustrlen(recipient_verify_failure) + 1; + m = n - (s - buf); + acl_verify_message = *msg = + m > 0 ? string_copyn_taint(s, m, FALSE) : NULL; + } + + DEBUG(D_verify) debug_printf_indent("verify call response:" + " len %d yield %s errno '%s' where '%s' msg '%s'\n", + n, rc_names[yield], strerror(save_errno), recipient_verify_failure, *msg); + + if ( yield == OK + || save_errno == 0 && Ustrcmp(recipient_verify_failure, "quota") == 0) + cache_quota_write(rcpt, yield, pos_cache, neg_cache); + else DEBUG(D_verify) + debug_printf_indent("result not cacheable\n"); + } + else + { + DEBUG(D_verify) + debug_printf_indent("verify call response: waitpid status 0x%04x\n", status); + } + } + +close(pfd[pipe_read]); +errno = save_errno; + +fail: + +return yield; +} + + /* vi: aw ai sw=2 */ /* End of verify.c */ diff --git a/test/confs/0589 b/test/confs/0589 new file mode 100644 index 000000000..b0f731143 --- /dev/null +++ b/test/confs/0589 @@ -0,0 +1,60 @@ +# Exim test configuration 0589 + +.include DIR/aux-var/std_conf_prefix + +primary_hostname = myhost.test.ex + +# ----- Main settings ----- + +acl_smtp_rcpt = chk_rcpt +log_selector = +millisec + +# --- ACL --- + +begin acl + +chk_rcpt: + deny !verify = recipient/quota=cachepos=20s,cacheneg=100s + logwrite = where='$recipient_verify_failure' msg='$acl_verify_message' + accept + +loggit: + accept logwrite = $acl_arg1 was called for $local_part@$domain + message = / + +# ----- Routers ----- + +begin routers + +localuser: + driver = accept + local_parts = !route_fail + transport = appendfile + # set address_data purely to get logging of use of the router + address_data = ${acl {loggit}{router }} + +failrouter: + driver = accept + condition = ${if eq {${acl {loggit}{router }}}{} {no}{no}} + transport = appendfile + +# ----- Transports ----- + +begin transports + +appendfile: + driver = appendfile + file = DIR/test-mail/themailfile + quota = 1k + user = CALLER + # set homedir purely to get logging of use of the transport + home_directory = ${acl {loggit}{transport}} + +# ----- Retry ----- + +begin retry + +* * F,5d,1d + + +# End diff --git a/test/log/0589 b/test/log/0589 new file mode 100644 index 000000000..dcbd6565a --- /dev/null +++ b/test/log/0589 @@ -0,0 +1,17 @@ + +******** SERVER ******** +2017-07-30 18:51:05.712 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port PORT_D +2017-07-30 18:51:05.712 router was called for quota_good@test.ex +2017-07-30 18:51:05.712 transport was called for quota_good@test.ex +2017-07-30 18:51:05.712 router was called for quota_fail@test.ex +2017-07-30 18:51:05.712 transport was called for quota_fail@test.ex +2017-07-30 18:51:05.712 where='quota' msg='Over quota' +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Over quota +2017-07-30 18:51:05.712 router was called for route_fail@test.ex +2017-07-30 18:51:05.712 where='route' msg='Unrouteable address' +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Unrouteable address +2017-07-30 18:51:05.712 where='quota' msg='Previous (cached) quota verification failure' +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Previous (cached) quota verification failure +2017-07-30 18:51:05.712 router was called for route_fail@test.ex +2017-07-30 18:51:05.712 where='route' msg='Unrouteable address' +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Unrouteable address diff --git a/test/rejectlog/0589 b/test/rejectlog/0589 new file mode 100644 index 000000000..432b6780f --- /dev/null +++ b/test/rejectlog/0589 @@ -0,0 +1,6 @@ + +******** SERVER ******** +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Over quota +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Unrouteable address +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Previous (cached) quota verification failure +2017-07-30 18:51:05.712 H=(test) [127.0.0.1] F= rejected RCPT : Unrouteable address diff --git a/test/scripts/0000-Basic/0589 b/test/scripts/0000-Basic/0589 new file mode 100644 index 000000000..8cea367eb --- /dev/null +++ b/test/scripts/0000-Basic/0589 @@ -0,0 +1,44 @@ +# verify quota at smtp time +# +exim -bd -DSERVER=server -oX PORT_D +**** +client 127.0.0.1 PORT_D +??? 220 +EHLO test +??? 250- +??? 250-SIZE +??? 250-8BITMIME +??? 250-PIPELINING +??? 250 HELP +MAIL FROM:\r\nRCPT TO: +??? 250 +??? 250 +RSET +??? 250 +MAIL FROM: SIZE=2048\r\nRCPT TO: +??? 250 +??? 550 +RSET +??? 250 +MAIL FROM:\r\nRCPT TO: +??? 250 +??? 550 +RSET +??? 250 +MAIL FROM:\r\nRCPT TO: +??? 250 +??? 250 +RSET +??? 250 +MAIL FROM: SIZE=512\r\nRCPT TO: +??? 250 +??? 550 +RSET +??? 250 +MAIL FROM:\r\nRCPT TO: +??? 250 +??? 550 +QUIT +**** +# +killdaemon diff --git a/test/stdout/0589 b/test/stdout/0589 new file mode 100644 index 000000000..9f70b7343 --- /dev/null +++ b/test/stdout/0589 @@ -0,0 +1,61 @@ +Connecting to 127.0.0.1 port 1225 ... connected +??? 220 +<<< 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +>>> EHLO test +??? 250- +<<< 250-myhost.test.ex Hello test [127.0.0.1] +??? 250-SIZE +<<< 250-SIZE 52428800 +??? 250-8BITMIME +<<< 250-8BITMIME +??? 250-PIPELINING +<<< 250-PIPELINING +??? 250 HELP +<<< 250 HELP +>>> MAIL FROM:\r\nRCPT TO: +??? 250 +<<< 250 OK +??? 250 +<<< 250 Accepted +>>> RSET +??? 250 +<<< 250 Reset OK +>>> MAIL FROM: SIZE=ssss\r\nRCPT TO: +??? 250 +<<< 250 OK +??? 550 +<<< 550 Administrative prohibition +>>> RSET +??? 250 +<<< 250 Reset OK +>>> MAIL FROM:\r\nRCPT TO: +??? 250 +<<< 250 OK +??? 550 +<<< 550 Administrative prohibition +>>> RSET +??? 250 +<<< 250 Reset OK +>>> MAIL FROM:\r\nRCPT TO: +??? 250 +<<< 250 OK +??? 250 +<<< 250 Accepted +>>> RSET +??? 250 +<<< 250 Reset OK +>>> MAIL FROM: SIZE=ssss\r\nRCPT TO: +??? 250 +<<< 250 OK +??? 550 +<<< 550 Administrative prohibition +>>> RSET +??? 250 +<<< 250 Reset OK +>>> MAIL FROM:\r\nRCPT TO: +??? 250 +<<< 250 OK +??? 550 +<<< 550 Administrative prohibition +>>> QUIT +End of script -- cgit v1.2.3