diff options
author | Jeremy Harris <jgh146exb@wizmail.org> | 2022-06-26 12:10:03 +0100 |
---|---|---|
committer | Jeremy Harris <jgh146exb@wizmail.org> | 2022-06-26 12:21:08 +0100 |
commit | 9f1a75f126ae217a3a3568b106c9133b3c5c413a (patch) | |
tree | 9a7ac8fe4c827c981b49946260ea0d1aa1e5524d /src | |
parent | 57318bfd2312ea561661906ca8001bd74a01b312 (diff) | |
parent | 89318c714454e11217505d2163d807d5d827f50a (diff) |
Merge branch '4.next'
Diffstat (limited to 'src')
35 files changed, 1294 insertions, 1040 deletions
diff --git a/src/OS/Makefile-Base b/src/OS/Makefile-Base index 78f5516a7..0c64d45d4 100644 --- a/src/OS/Makefile-Base +++ b/src/OS/Makefile-Base @@ -1,4 +1,5 @@ # This file is the basis of the main makefile for Exim and friends. The +# # makefile at the top level arranges to build the main makefile by calling # scripts/Configure-Makefile from within the build directory. This # concatenates the configuration settings from Local/Makefile and other, @@ -485,9 +486,9 @@ OBJ_LOOKUPS = lookups/lf_quote.o lookups/lf_check_file.o lookups/lf_sqlperform.o OBJ_EXIM = acl.o base64.o child.o crypt16.o daemon.o dbfn.o debug.o deliver.o \ directory.o dns.o drtables.o enq.o exim.o expand.o filter.o \ filtertest.o globals.o dkim.o dkim_transport.o dnsbl.o hash.o \ - header.o host.o ip.o log.o lss.o match.o md5.o moan.o \ + header.o host.o host_address.o ip.o log.o lss.o match.o md5.o moan.o \ os.o parse.o priv.o queue.o \ - rda.o readconf.o receive.o retry.o rewrite.o rfc2047.o \ + rda.o readconf.o receive.o retry.o rewrite.o rfc2047.o regex_cache.o \ route.o search.o sieve.o smtp_in.o smtp_out.o spool_in.o spool_out.o \ std-crypto.o store.o string.o tls.o tod.o transport.o tree.o verify.o \ environment.o macro.o \ @@ -601,10 +602,11 @@ MONBIN = em_StripChart.o $(EXIMON_TEXTPOP) em_globals.o em_init.o \ # The complete modules list also includes some specially compiled versions of # code from the main Exim source tree. -OBJ_MONBIN = util-spool_in.o \ +OBJ_MONBIN = util-host_address.o \ + util-queue.o \ + util-spool_in.o \ util-store.o \ util-string.o \ - util-queue.o \ util-tod.o \ util-tree.o \ $(MONBIN) @@ -721,6 +723,14 @@ exim_dbmbuild.o: $(HDRS) exim_dbmbuild.c # Utilities use special versions of some modules - typically with debugging # calls cut out. +util-host_address.o: $(HDRS) host_address.c + @echo "$(CC) -DCOMPILE_UTILITY host_address.c" + $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-host_address.o host_address.c + +util-md5.o: $(HDRS) md5.c + @echo "$(CC) -DCOMPILE_UTILITY queue.c" + $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-md5.o md5.c + util-spool_in.o: $(HDRS) spool_in.c @echo "$(CC) -DCOMPILE_UTILITY spool_in.c" $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-spool_in.o spool_in.c @@ -733,10 +743,6 @@ util-string.o: $(HDRS) string.c @echo "$(CC) -DCOMPILE_UTILITY string.c" $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-string.o string.c -util-md5.o: $(HDRS) md5.c - @echo "$(CC) -DCOMPILE_UTILITY queue.c" - $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-md5.o md5.c - util-queue.o: $(HDRS) queue.c @echo "$(CC) -DCOMPILE_UTILITY queue.c" $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-queue.o queue.c @@ -787,6 +793,7 @@ globals.o: $(HDRS) globals.c hash.o: $(HDRS) hash.c header.o: $(HDRS) header.c host.o: $(HDRS) host.c +host_address.o: $(HDRS) host_address.c ip.o: $(HDRS) ip.c log.o: $(HDRS) log.c lss.o: $(HDRS) lss.c @@ -802,6 +809,7 @@ readconf.o: $(HDRS) readconf.c receive.o: $(HDRS) receive.c retry.o: $(HDRS) retry.c rewrite.o: $(HDRS) rewrite.c +regex_cache.o: $(HDRS) regex_cache.c rfc2047.o: $(HDRS) rfc2047.c route.o: $(HDRS) route.c search.o: $(HDRS) search.c @@ -962,13 +970,13 @@ test_dbfn: config.h dbfn.c dummies.o sa-globals.o sa-os.o store.o \ rm -f dbfn.o store.o test_host: config.h child.c host.c dns.c dummies.c sa-globals.o os.o \ - store.o string.o tod.o tree.o + host_address.o store.o string.o tod.o tree.o $(CC) -c $(CFLAGS) $(INCLUDE) -DSTAND_ALONE -DTEST_HOST child.c $(CC) -c $(CFLAGS) $(INCLUDE) -DSTAND_ALONE -DTEST_HOST host.c $(CC) -c $(CFLAGS) $(INCLUDE) -DSTAND_ALONE -DTEST_HOST dns.c $(CC) -c $(CFLAGS) $(INCLUDE) -DSTAND_ALONE -DTEST_HOST dummies.c $(LNCC) -o test_host $(LFLAGS) \ - host.o child.o dns.o dummies.o sa-globals.o os.o store.o string.o \ + host.o host_address.o child.o dns.o dummies.o sa-globals.o os.o store.o string.o \ tod.o tree.o $(LIBS) $(LIBRESOLV) rm -f child.o dummies.o host.o dns.o diff --git a/src/OS/Makefile-FreeBSD b/src/OS/Makefile-FreeBSD index 02253d9c6..4793a438b 100644 --- a/src/OS/Makefile-FreeBSD +++ b/src/OS/Makefile-FreeBSD @@ -13,7 +13,7 @@ PERL_COMMAND=/usr/local/bin/perl HAVE_SA_LEN=YES # crypt() is in a separate library -LIBS=-lcrypt -lm -lutil +LIBS=-lcrypt -lm -lutil -lexecinfo # Dynamically loaded modules need to be built with -fPIC CFLAGS_DYNAMIC=-shared -rdynamic -fPIC diff --git a/src/OS/Makefile-OpenBSD b/src/OS/Makefile-OpenBSD index 697632682..7c451e2cc 100644 --- a/src/OS/Makefile-OpenBSD +++ b/src/OS/Makefile-OpenBSD @@ -23,6 +23,7 @@ EXIWHAT_EGREP_ARG='/exim( |$$)' EXIWHAT_KILL_SIGNAL=-USR1 HAVE_IPV6=YES +CFLAGS += -DNO_EXECINFO # OpenBSD ships with a too-old Berkeley DB. NDBM is the default if we don't specify one. #USE_DB=yes diff --git a/src/OS/Makefile-SunOS5 b/src/OS/Makefile-SunOS5 index 3b436f4bf..d07adcff4 100644 --- a/src/OS/Makefile-SunOS5 +++ b/src/OS/Makefile-SunOS5 @@ -20,5 +20,6 @@ X11LIB=$(X11)/lib OS_C_INCLUDES=setenv.c CFLAGS += -D_XOPEN_SOURCE -D_XOPEN_SOURCE_EXTENDED=1 -D__EXTENSIONS__ +CFLAGS += -DNO_EXECINFO # End diff --git a/src/exim_monitor/em_main.c b/src/exim_monitor/em_main.c index 5714b999c..86caf71eb 100644 --- a/src/exim_monitor/em_main.c +++ b/src/exim_monitor/em_main.c @@ -180,70 +180,6 @@ va_end(ap); /************************************************* -* Extract port from address string * -*************************************************/ - -/* In the spool file, a host plus port is given as an IP address followed by a -dot and a port number. This function decodes this. It is needed by the -spool-reading function, and copied here to avoid having to include the whole -host.c module. One day the interaction between exim and eximon with regard to -included code MUST be tidied up! - -Argument: - address points to the string; if there is a port, the '.' in the string - is overwritten with zero to terminate the address - -Returns: 0 if there is no port, else the port number. -*/ - -int -host_address_extract_port(uschar * address) -{ -int port = 0; -uschar *endptr; - -/* Handle the "bracketed with colon on the end" format */ - -if (*address == '[') - { - uschar *rb = address + 1; - while (*rb != 0 && *rb != ']') rb++; - if (*rb++ == 0) return 0; /* Missing ]; leave invalid address */ - if (*rb == ':') - { - port = Ustrtol(rb + 1, &endptr, 10); - if (*endptr != 0) return 0; /* Invalid port; leave invalid address */ - } - else if (*rb != 0) return 0; /* Bad syntax; leave invalid address */ - memmove(address, address + 1, rb - address - 2); - rb[-2] = 0; - } - -/* Handle the "dot on the end" format */ - -else - { - int skip = -3; /* Skip 3 dots in IPv4 addresses */ - address--; - while (*(++address) != 0) - { - int ch = *address; - if (ch == ':') skip = 0; /* Skip 0 dots in IPv6 addresses */ - else if (ch == '.' && skip++ >= 0) break; - } - if (*address == 0) return 0; - port = Ustrtol(address + 1, &endptr, 10); - if (*endptr != 0) return 0; /* Invalid port; leave invalid address */ - *address = 0; - } - -return port; -} - - - - -/************************************************* * SIGCHLD handler * *************************************************/ diff --git a/src/scripts/MakeLinks b/src/scripts/MakeLinks index 9cdb931f3..471b3a369 100755 --- a/src/scripts/MakeLinks +++ b/src/scripts/MakeLinks @@ -102,9 +102,9 @@ for f in blob.h dbfunctions.h exim.h functions.h globals.h \ acl.c buildconfig.c base64.c child.c crypt16.c daemon.c dbfn.c debug.c \ deliver.c directory.c dns.c dnsbl.c drtables.c dummies.c enq.c exim.c \ exim_dbmbuild.c exim_dbutil.c exim_lock.c expand.c filter.c filtertest.c \ - globals.c hash.c header.c host.c ip.c log.c lss.c match.c md5.c moan.c \ + globals.c hash.c header.c host.c host_address.c ip.c log.c lss.c match.c md5.c moan.c \ parse.c perl.c priv.c queue.c rda.c readconf.c receive.c retry.c rewrite.c \ - rfc2047.c route.c search.c setenv.c environment.c \ + regex_cache.c rfc2047.c route.c search.c setenv.c environment.c \ sieve.c smtp_in.c smtp_out.c spool_in.c spool_out.c std-crypto.c store.c \ string.c tls.c tlscert-gnu.c tlscert-openssl.c tls-cipher-stdname.c \ tls-gnu.c tls-openssl.c \ diff --git a/src/src/acl.c b/src/src/acl.c index fb78a7b5f..0078aca7d 100644 --- a/src/src/acl.c +++ b/src/src/acl.c @@ -3125,8 +3125,9 @@ int sep = -'/'; for (; cb; cb = cb->next) { - const uschar *arg; + const uschar * arg; int control_type; + BOOL textonly = FALSE; /* The message and log_message items set up messages to be used in case of rejection. They are expanded later. */ @@ -3160,7 +3161,8 @@ for (; cb; cb = cb->next) if (!conditions[cb->type].expand_at_top) arg = cb->arg; - else if (!(arg = expand_string(cb->arg))) + + else if (!(arg = expand_string_2(cb->arg, &textonly))) { if (f.expand_string_forcedfail) continue; *log_msgptr = string_sprintf("failed to expand ACL string \"%s\": %s", @@ -3217,8 +3219,8 @@ for (; cb; cb = cb->next) switch(cb->type) { case ACLC_ADD_HEADER: - setup_header(arg); - break; + setup_header(arg); + break; /* A nested ACL that returns "discard" makes sense only for an "accept" or "discard" verb. */ @@ -3232,12 +3234,12 @@ for (; cb; cb = cb->next) verbs[verb]); return ERROR; } - break; + break; case ACLC_AUTHENTICATED: rc = sender_host_authenticated ? match_isinlist(sender_host_authenticated, &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL) : FAIL; - break; + break; #ifdef EXPERIMENTAL_BRIGHTMAIL case ACLC_BMI_OPTIN: @@ -3254,21 +3256,21 @@ for (; cb; cb = cb->next) /* The true/false parsing here should be kept in sync with that used in expand.c when dealing with ECOND_BOOL so that we don't have too many different definitions of what can be a boolean. */ - if (*arg == '-' - ? Ustrspn(arg+1, "0123456789") == Ustrlen(arg+1) /* Negative number */ - : Ustrspn(arg, "0123456789") == Ustrlen(arg)) /* Digits, or empty */ - rc = (Uatoi(arg) == 0)? FAIL : OK; - else - rc = (strcmpic(arg, US"no") == 0 || - strcmpic(arg, US"false") == 0)? FAIL : - (strcmpic(arg, US"yes") == 0 || - strcmpic(arg, US"true") == 0)? OK : DEFER; - if (rc == DEFER) - *log_msgptr = string_sprintf("invalid \"condition\" value \"%s\"", arg); - break; + if (*arg == '-' + ? Ustrspn(arg+1, "0123456789") == Ustrlen(arg+1) /* Negative number */ + : Ustrspn(arg, "0123456789") == Ustrlen(arg)) /* Digits, or empty */ + rc = (Uatoi(arg) == 0)? FAIL : OK; + else + rc = (strcmpic(arg, US"no") == 0 || + strcmpic(arg, US"false") == 0)? FAIL : + (strcmpic(arg, US"yes") == 0 || + strcmpic(arg, US"true") == 0)? OK : DEFER; + if (rc == DEFER) + *log_msgptr = string_sprintf("invalid \"condition\" value \"%s\"", arg); + break; case ACLC_CONTINUE: /* Always succeeds */ - break; + break; case ACLC_CONTROL: { @@ -3647,14 +3649,14 @@ for (; cb; cb = cb->next) while ((ss = string_nextinlist(&list, &sep, NULL, 0))) if (strcmpic(ss, US"defer_ok") == 0 && rc == DEFER) rc = FAIL; /* FAIL so that the message is passed to the next ACL */ + break; } - break; #endif #ifdef WITH_CONTENT_SCAN case ACLC_DECODE: - rc = mime_decode(&arg); - break; + rc = mime_decode(&arg); + break; #endif case ACLC_DELAY: @@ -3719,44 +3721,44 @@ for (; cb; cb = cb->next) #endif } } + break; } - break; #ifndef DISABLE_DKIM case ACLC_DKIM_SIGNER: - if (dkim_cur_signer) - rc = match_isinlist(dkim_cur_signer, + if (dkim_cur_signer) + rc = match_isinlist(dkim_cur_signer, &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL); - else - rc = FAIL; - break; + else + rc = FAIL; + break; case ACLC_DKIM_STATUS: - rc = match_isinlist(dkim_verify_status, - &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL); - break; + rc = match_isinlist(dkim_verify_status, + &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL); + break; #endif #ifdef SUPPORT_DMARC case ACLC_DMARC_STATUS: - if (!f.dmarc_has_been_checked) - dmarc_process(); - f.dmarc_has_been_checked = TRUE; - /* used long way of dmarc_exim_expand_query() in case we need more - * view into the process in the future. */ - rc = match_isinlist(dmarc_exim_expand_query(DMARC_VERIFY_STATUS), - &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL); - break; + if (!f.dmarc_has_been_checked) + dmarc_process(); + f.dmarc_has_been_checked = TRUE; + /* used long way of dmarc_exim_expand_query() in case we need more + * view into the process in the future. */ + rc = match_isinlist(dmarc_exim_expand_query(DMARC_VERIFY_STATUS), + &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL); + break; #endif case ACLC_DNSLISTS: - rc = verify_check_dnsbl(where, &arg, log_msgptr); - break; + rc = verify_check_dnsbl(where, &arg, log_msgptr); + break; case ACLC_DOMAINS: - rc = match_isinlist(addr->domain, &arg, 0, &domainlist_anchor, - addr->domain_cache, MCL_DOMAIN, TRUE, CUSS &deliver_domain_data); - break; + rc = match_isinlist(addr->domain, &arg, 0, &domainlist_anchor, + addr->domain_cache, MCL_DOMAIN, TRUE, CUSS &deliver_domain_data); + break; /* The value in tls_cipher is the full cipher name, for example, TLSv1:DES-CBC3-SHA:168, whereas the values to test for are just the @@ -3765,19 +3767,20 @@ for (; cb; cb = cb->next) writing is poorly documented. */ case ACLC_ENCRYPTED: - if (tls_in.cipher == NULL) rc = FAIL; else - { - uschar *endcipher = NULL; - uschar *cipher = Ustrchr(tls_in.cipher, ':'); - if (!cipher) cipher = tls_in.cipher; else - { - endcipher = Ustrchr(++cipher, ':'); - if (endcipher) *endcipher = 0; - } - rc = match_isinlist(cipher, &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL); - if (endcipher) *endcipher = ':'; - } - break; + if (!tls_in.cipher) rc = FAIL; + else + { + uschar *endcipher = NULL; + uschar *cipher = Ustrchr(tls_in.cipher, ':'); + if (!cipher) cipher = tls_in.cipher; else + { + endcipher = Ustrchr(++cipher, ':'); + if (endcipher) *endcipher = 0; + } + rc = match_isinlist(cipher, &arg, 0, NULL, NULL, MCL_STRING, TRUE, NULL); + if (endcipher) *endcipher = ':'; + } + break; /* Use verify_check_this_host() instead of verify_check_host() so that we can pass over &host_data to catch any looked up data. Once it has been @@ -3787,17 +3790,17 @@ for (; cb; cb = cb->next) message in the same SMTP connection. */ case ACLC_HOSTS: - rc = verify_check_this_host(&arg, sender_host_cache, NULL, - sender_host_address ? sender_host_address : US"", CUSS &host_data); - if (rc == DEFER) *log_msgptr = search_error_message; - if (host_data) host_data = string_copy_perm(host_data, TRUE); - break; + rc = verify_check_this_host(&arg, sender_host_cache, NULL, + sender_host_address ? sender_host_address : US"", CUSS &host_data); + if (rc == DEFER) *log_msgptr = search_error_message; + if (host_data) host_data = string_copy_perm(host_data, TRUE); + break; case ACLC_LOCAL_PARTS: - rc = match_isinlist(addr->cc_local_part, &arg, 0, - &localpartlist_anchor, addr->localpart_cache, MCL_LOCALPART, TRUE, - CUSS &deliver_localpart_data); - break; + rc = match_isinlist(addr->cc_local_part, &arg, 0, + &localpartlist_anchor, addr->localpart_cache, MCL_LOCALPART, TRUE, + CUSS &deliver_localpart_data); + break; case ACLC_LOG_REJECT_TARGET: { @@ -3818,8 +3821,8 @@ for (; cb; cb = cb->next) } } log_reject_target = logbits; + break; } - break; case ACLC_LOGWRITE: { @@ -3850,8 +3853,8 @@ for (; cb; cb = cb->next) if (logbits == 0) logbits = LOG_MAIN; log_write(0, logbits, "%s", string_printing(s)); + break; } - break; #ifdef WITH_CONTENT_SCAN case ACLC_MALWARE: /* Run the malware backend. */ @@ -3874,55 +3877,55 @@ for (; cb; cb = cb->next) return ERROR; } - rc = malware(ss, timeout); + rc = malware(ss, textonly, timeout); if (rc == DEFER && defer_ok) rc = FAIL; /* FAIL so that the message is passed to the next ACL */ + break; } - break; case ACLC_MIME_REGEX: - rc = mime_regex(&arg); - break; + rc = mime_regex(&arg, textonly); + break; #endif case ACLC_QUEUE: - if (is_tainted(arg)) - { - *log_msgptr = string_sprintf("Tainted name '%s' for queue not permitted", - arg); - return ERROR; - } - if (Ustrchr(arg, '/')) - { - *log_msgptr = string_sprintf( - "Directory separator not permitted in queue name: '%s'", arg); - return ERROR; - } - queue_name = string_copy_perm(arg, FALSE); - break; + if (is_tainted(arg)) + { + *log_msgptr = string_sprintf("Tainted name '%s' for queue not permitted", + arg); + return ERROR; + } + if (Ustrchr(arg, '/')) + { + *log_msgptr = string_sprintf( + "Directory separator not permitted in queue name: '%s'", arg); + return ERROR; + } + queue_name = string_copy_perm(arg, FALSE); + break; case ACLC_RATELIMIT: - rc = acl_ratelimit(arg, where, log_msgptr); - break; + rc = acl_ratelimit(arg, where, log_msgptr); + break; case ACLC_RECIPIENTS: - rc = match_address_list(CUS addr->address, TRUE, TRUE, &arg, NULL, -1, 0, - CUSS &recipient_data); - break; + rc = match_address_list(CUS addr->address, TRUE, TRUE, &arg, NULL, -1, 0, + CUSS &recipient_data); + break; #ifdef WITH_CONTENT_SCAN case ACLC_REGEX: - rc = regex(&arg); - break; + rc = regex(&arg, textonly); + break; #endif case ACLC_REMOVE_HEADER: - setup_remove_header(arg); - break; + setup_remove_header(arg); + break; case ACLC_SEEN: - rc = acl_seen(arg, where, log_msgptr); - break; + rc = acl_seen(arg, where, log_msgptr); + break; case ACLC_SENDER_DOMAINS: { @@ -3931,13 +3934,13 @@ for (; cb; cb = cb->next) sdomain = sdomain ? sdomain + 1 : US""; rc = match_isinlist(sdomain, &arg, 0, &domainlist_anchor, sender_domain_cache, MCL_DOMAIN, TRUE, NULL); + break; } - break; case ACLC_SENDERS: - rc = match_address_list(CUS sender_address, TRUE, TRUE, &arg, - sender_address_cache, -1, 0, CUSS &sender_data); - break; + rc = match_address_list(CUS sender_address, TRUE, TRUE, &arg, + sender_address_cache, -1, 0, CUSS &sender_data); + break; /* Connection variables must persist forever; message variables not */ @@ -3959,8 +3962,8 @@ for (; cb; cb = cb->next) #endif acl_var_create(cb->u.varname)->data.ptr = string_copy(arg); store_pool = old_pool; + break; } - break; #ifdef WITH_CONTENT_SCAN case ACLC_SPAM: @@ -3974,22 +3977,23 @@ for (; cb; cb = cb->next) while ((ss = string_nextinlist(&list, &sep, NULL, 0))) if (strcmpic(ss, US"defer_ok") == 0 && rc == DEFER) rc = FAIL; /* FAIL so that the message is passed to the next ACL */ + break; } - break; #endif #ifdef SUPPORT_SPF case ACLC_SPF: rc = spf_process(&arg, sender_address, SPF_PROCESS_NORMAL); - break; + break; + case ACLC_SPF_GUESS: rc = spf_process(&arg, sender_address, SPF_PROCESS_GUESS); - break; + break; #endif case ACLC_UDPSEND: - rc = acl_udpsend(arg, log_msgptr); - break; + rc = acl_udpsend(arg, log_msgptr); + break; /* If the verb is WARN, discard any user message from verification, because such messages are SMTP responses, not header additions. The latter come @@ -3998,16 +4002,16 @@ for (; cb; cb = cb->next) (until something changes it). */ case ACLC_VERIFY: - rc = acl_verify(where, addr, arg, user_msgptr, log_msgptr, basic_errno); - if (*user_msgptr) - acl_verify_message = *user_msgptr; - if (verb == ACL_WARN) *user_msgptr = NULL; - break; + rc = acl_verify(where, addr, arg, user_msgptr, log_msgptr, basic_errno); + if (*user_msgptr) + acl_verify_message = *user_msgptr; + if (verb == ACL_WARN) *user_msgptr = NULL; + break; default: - log_write(0, LOG_MAIN|LOG_PANIC_DIE, "internal ACL error: unknown " - "condition %d", cb->type); - break; + log_write(0, LOG_MAIN|LOG_PANIC_DIE, "internal ACL error: unknown " + "condition %d", cb->type); + break; } /* If a condition was negated, invert OK/FAIL. */ diff --git a/src/src/daemon.c b/src/src/daemon.c index 8e8a515e4..54725e07d 100644 --- a/src/src/daemon.c +++ b/src/src/daemon.c @@ -1132,13 +1132,43 @@ exim_exit(EXIT_SUCCESS); * Listener socket for local work prompts * *************************************************/ +ssize_t +daemon_client_sockname(struct sockaddr_un * sup, uschar ** sname) +{ +#ifdef EXIM_HAVE_ABSTRACT_UNIX_SOCKETS +sup->sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ +return offsetof(struct sockaddr_un, sun_path) + 1 + + snprintf(sup->sun_path+1, sizeof(sup->sun_path)-1, "exim_%d", getpid()); +#else +*sname = string_sprintf("%s/p_%d", spool_directory, getpid()); +return offsetof(struct sockaddr_un, sun_path) + + snprintf(sup->sun_path, sizeof(sup->sun_path), "%s", sname); +#endif +} + +ssize_t +daemon_notifier_sockname(struct sockaddr_un * sup) +{ +#ifdef EXIM_HAVE_ABSTRACT_UNIX_SOCKETS +sup->sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ +return offsetof(struct sockaddr_un, sun_path) + 1 + + snprintf(sup->sun_path+1, sizeof(sup->sun_path)-1, "%s", + expand_string(notifier_socket)); +#else +return offsetof(struct sockaddr_un, sun_path) + + snprintf(sup->sun_path, sizeof(sup->sun_path), "%s", + expand_string(notifier_socket)); +#endif +} + + static void daemon_notifier_socket(void) { int fd; const uschar * where; struct sockaddr_un sa_un = {.sun_family = AF_UNIX}; -int len; +ssize_t len; if (!notifier_socket || !*notifier_socket) { @@ -1163,20 +1193,15 @@ if ((fd = socket(PF_UNIX, SOCK_DGRAM, 0)) < 0) (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); #endif +len = daemon_notifier_sockname(&sa_un); + #ifdef EXIM_HAVE_ABSTRACT_UNIX_SOCKETS -sa_un.sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ -len = offsetof(struct sockaddr_un, sun_path) + 1 - + snprintf(sa_un.sun_path+1, sizeof(sa_un.sun_path)-1, "%s", - expand_string(notifier_socket)); DEBUG(D_any) debug_printf(" @%s\n", sa_un.sun_path+1); #else /* filesystem-visible and persistent; will neeed removal */ -len = offsetof(struct sockaddr_un, sun_path) - + snprintf(sa_un.sun_path, sizeof(sa_un.sun_path), "%s", - expand_string(notifier_socket)); DEBUG(D_any) debug_printf(" %s\n", sa_un.sun_path); #endif -if (bind(fd, (const struct sockaddr *)&sa_un, len) < 0) +if (bind(fd, (const struct sockaddr *)&sa_un, (socklen_t)len) < 0) { where = US"bind"; goto bad; } #ifdef SO_PASSCRED /* Linux */ @@ -1205,7 +1230,11 @@ bad: static uschar queuerun_msgid[MESSAGE_ID_LENGTH+1]; -/* Return TRUE if a sigalrm should be emulated */ +/* The notifier socket has something to read. Pull the message from it, decode +and do the action. + +Return TRUE if a sigalrm should be emulated */ + static BOOL daemon_notification(void) { @@ -1255,7 +1284,6 @@ for (struct cmsghdr * cp = CMSG_FIRSTHDR(&msg); { DEBUG(D_queue_run) debug_printf("%s: sender creds pid %d uid %d gid %d\n", __FUNCTION__, (int)cr->pid, (int)cr->uid, (int)cr->gid); - return FALSE; } # elif defined(LOCAL_CREDS) /* BSD-ish */ struct sockcred * cr = (struct sockcred *) CMSG_DATA(cp); @@ -1263,7 +1291,6 @@ for (struct cmsghdr * cp = CMSG_FIRSTHDR(&msg); { DEBUG(D_queue_run) debug_printf("%s: sender creds pid ??? uid %d gid %d\n", __FUNCTION__, (int)cr->sc_uid, (int)cr->sc_gid); - return FALSE; } # endif break; @@ -1294,15 +1321,18 @@ switch (buf[0]) (const struct sockaddr *)&sa_un, msg.msg_namelen) < 0) log_write(0, LOG_MAIN|LOG_PANIC, "%s: sendto: %s\n", __FUNCTION__, strerror(errno)); - return FALSE; + break; } + + case NOTIFY_REGEX: + regex_at_daemon(buf); + break; } return FALSE; } - /************************************************* * Exim Daemon Mainline * *************************************************/ diff --git a/src/src/deliver.c b/src/src/deliver.c index 8a9a174e3..725d0c872 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -7203,7 +7203,7 @@ local and remote LMTP deliveries. */ if (!regex_IGNOREQUOTA) regex_IGNOREQUOTA = - regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE); + regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", MCS_NOFLAGS, TRUE); /* Handle local deliveries */ diff --git a/src/src/dns.c b/src/src/dns.c index 7d7ee0c04..4071c5822 100644 --- a/src/src/dns.c +++ b/src/src/dns.c @@ -1324,7 +1324,7 @@ dns_pattern_init(void) { if (check_dns_names_pattern[0] != 0 && !regex_check_dns_names) regex_check_dns_names = - regex_must_compile(check_dns_names_pattern, FALSE, TRUE); + regex_must_compile(check_dns_names_pattern, MCS_NOFLAGS, TRUE); } /* vi: aw ai sw=2 diff --git a/src/src/drtables.c b/src/src/drtables.c index 513ef6c4a..b2f2a4b33 100644 --- a/src/src/drtables.c +++ b/src/src/drtables.c @@ -728,8 +728,8 @@ if (!(dd = exim_opendir(LOOKUP_MODULE_DIR))) } else { - const pcre2_code *regex_islookupmod = regex_must_compile( - US"\\." DYNLIB_FN_EXT "$", FALSE, TRUE); + const pcre2_code * regex_islookupmod = regex_must_compile( + US"\\." DYNLIB_FN_EXT "$", MCS_NOFLAGS, TRUE); DEBUG(D_lookup) debug_printf("Loading lookup modules from %s\n", LOOKUP_MODULE_DIR); while ((ent = readdir(dd))) diff --git a/src/src/exim.c b/src/src/exim.c index fd01d1355..dec8de4b4 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -17,6 +17,13 @@ Also a few functions that don't naturally fit elsewhere. */ # include <gnu/libc-version.h> #endif +#ifndef _TIME_H +# include <time.h> +#endif +#ifndef NO_EXECINFO +# include <execinfo.h> +#endif + #ifdef USE_GNUTLS # include <gnutls/gnutls.h> # if GNUTLS_VERSION_NUMBER < 0x030103 && !defined(DISABLE_OCSP) @@ -24,10 +31,6 @@ Also a few functions that don't naturally fit elsewhere. */ # endif #endif -#ifndef _TIME_H -# include <time.h> -#endif - extern void init_lookup_list(void); @@ -56,78 +59,40 @@ if (block) store_free(block); } +static void * +function_store_get(PCRE2_SIZE size, void * tag) +{ +return store_get((int)size, GET_UNTAINTED); /* loses track of taint */ +} - -/************************************************* -* Enums for cmdline interface * -*************************************************/ - -enum commandline_info { CMDINFO_NONE=0, - CMDINFO_HELP, CMDINFO_SIEVE, CMDINFO_DSCP }; +static void +function_store_nullfree(void * block, void * tag) +{ +} /************************************************* -* Compile regular expression and panic on fail * +* Enums for cmdline interface * *************************************************/ -/* This function is called when failure to compile a regular expression leads -to a panic exit. In other cases, pcre_compile() is called directly. In many -cases where this function is used, the results of the compilation are to be -placed in long-lived store, so we temporarily reset the store management -functions that PCRE uses if the use_malloc flag is set. - -Argument: - pattern the pattern to compile - caseless TRUE if caseless matching is required - use_malloc TRUE if compile into malloc store - -Returns: pointer to the compiled pattern -*/ - -const pcre2_code * -regex_must_compile(const uschar * pattern, BOOL caseless, BOOL use_malloc) -{ -size_t offset; -int options = caseless ? PCRE_COPT|PCRE2_CASELESS : PCRE_COPT; -const pcre2_code * yield; -int err; -pcre2_general_context * gctx; -pcre2_compile_context * cctx; - -if (use_malloc) - { - gctx = pcre2_general_context_create(function_store_malloc, function_store_free, NULL); - cctx = pcre2_compile_context_create(gctx); - } -else - cctx = pcre_cmp_ctx; +enum commandline_info { CMDINFO_NONE=0, + CMDINFO_HELP, CMDINFO_SIEVE, CMDINFO_DSCP }; -if (!(yield = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, options, - &err, &offset, cctx))) - { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - log_write(0, LOG_MAIN|LOG_PANIC_DIE, "regular expression error: " - "%s at offset %ld while compiling %s", errbuf, (long)offset, pattern); - } -if (use_malloc) - { - pcre2_compile_context_free(cctx); - pcre2_general_context_free(gctx); - } -return yield; -} static void pcre_init(void) { -pcre_gen_ctx = pcre2_general_context_create(function_store_malloc, function_store_free, NULL); -pcre_cmp_ctx = pcre2_compile_context_create(pcre_gen_ctx); -pcre_mtc_ctx = pcre2_match_context_create(pcre_gen_ctx); +pcre_mlc_ctx = pcre2_general_context_create(function_store_malloc, function_store_free, NULL); +pcre_gen_ctx = pcre2_general_context_create(function_store_get, function_store_nullfree, NULL); + +pcre_mlc_cmp_ctx = pcre2_compile_context_create(pcre_mlc_ctx); +pcre_gen_cmp_ctx = pcre2_compile_context_create(pcre_gen_ctx); + +pcre_gen_mtc_ctx = pcre2_match_context_create(pcre_gen_ctx); } @@ -157,7 +122,7 @@ regex_match_and_setup(const pcre2_code * re, const uschar * subject, int options { pcre2_match_data * md = pcre2_match_data_create_from_pattern(re, pcre_gen_ctx); int res = pcre2_match(re, (PCRE2_SPTR)subject, PCRE2_ZERO_TERMINATED, 0, - PCRE_EOPT | options, md, pcre_mtc_ctx); + PCRE_EOPT | options, md, pcre_gen_mtc_ctx); BOOL yield; if ((yield = (res >= 0))) @@ -179,7 +144,7 @@ else if (res != PCRE2_ERROR_NOMATCH) DEBUG(D_any) pcre2_get_error_message(res, errbuf, sizeof(errbuf)); debug_printf_indent("pcre2: %s\n", errbuf); } -pcre2_match_data_free(md); +/* pcre2_match_data_free(md); gen ctx needs no free */ return yield; } @@ -201,13 +166,18 @@ regex_match(const pcre2_code * re, const uschar * subject, int slen, uschar ** r pcre2_match_data * md = pcre2_match_data_create(1, pcre_gen_ctx); int rc = pcre2_match(re, (PCRE2_SPTR)subject, slen >= 0 ? slen : PCRE2_ZERO_TERMINATED, - 0, PCRE_EOPT, md, pcre_mtc_ctx); + 0, PCRE_EOPT, md, pcre_gen_mtc_ctx); PCRE2_SIZE * ovec = pcre2_get_ovector_pointer(md); -if (rc < 0) - return FALSE; -if (rptr) - *rptr = string_copyn(subject + ovec[0], ovec[1] - ovec[0]); -return TRUE; +BOOL ret = FALSE; + +if (rc >= 0) + { + if (rptr) + *rptr = string_copyn(subject + ovec[0], ovec[1] - ovec[0]); + ret = TRUE; + } +/* pcre2_match_data_free(md); gen ctx needs no free */ +return ret; } @@ -261,6 +231,31 @@ exit(1); * Handler for SIGSEGV * ***********************************************/ +#define STACKDUMP_MAX 24 +static void +stackdump(void) +{ +#ifndef NO_EXECINFO +void * buf[STACKDUMP_MAX]; +char ** ss; +int nptrs = backtrace(buf, STACKDUMP_MAX); + +log_write(0, LOG_MAIN|LOG_PANIC, "backtrace\n"); +log_write(0, LOG_MAIN|LOG_PANIC, "---\n"); +if ((ss = backtrace_symbols(buf, nptrs))) + { + for (int i = 0; i < nptrs; i++) + log_write(0, LOG_MAIN|LOG_PANIC, "\t%s\n", ss[i]); + free(ss); + } +else + log_write(0, LOG_MAIN|LOG_PANIC, "backtrace_symbols: %s\n", strerror(errno)); +log_write(0, LOG_MAIN|LOG_PANIC, "---\n"); +#endif +} +#undef STACKDUMP_MAX + + static void #ifdef SA_SIGINFO segv_handler(int sig, siginfo_t * info, void * uctx) @@ -281,6 +276,7 @@ else log_write(0, LOG_MAIN|LOG_PANIC, "SIGSEGV (maybe attempt to write to immutable memory)"); if (process_info_len > 0) log_write(0, LOG_MAIN|LOG_PANIC, "SIGSEGV (%.*s)", process_info_len, process_info); +stackdump(); signal(SIGSEGV, SIG_DFL); kill(getpid(), sig); } @@ -291,6 +287,7 @@ segv_handler(int sig) log_write(0, LOG_MAIN|LOG_PANIC, "SIGSEGV (maybe attempt to write to immutable memory)"); if (process_info_len > 0) log_write(0, LOG_MAIN|LOG_PANIC, "SIGSEGV (%.*s)", process_info_len, process_info); +stackdump(); signal(SIGSEGV, SIG_DFL); kill(getpid(), sig); } @@ -1983,7 +1980,7 @@ this here, because the -M options check their arguments for syntactic validity using mac_ismsgid, which uses this. */ regex_ismsgid = - regex_must_compile(US"^(?:[^\\W_]{6}-){2}[^\\W_]{2}$", FALSE, TRUE); + regex_must_compile(US"^(?:[^\\W_]{6}-){2}[^\\W_]{2}$", MCS_NOFLAGS, TRUE); /* Precompile the regular expression that is used for matching an SMTP error code, possibly extended, at the start of an error message. Note that the @@ -1991,14 +1988,14 @@ terminating whitespace character is included. */ regex_smtp_code = regex_must_compile(US"^\\d\\d\\d\\s(?:\\d\\.\\d\\d?\\d?\\.\\d\\d?\\d?\\s)?", - FALSE, TRUE); + MCS_NOFLAGS, TRUE); #ifdef WHITELIST_D_MACROS /* Precompile the regular expression used to filter the content of macros given to -D for permissibility. */ regex_whitelisted_macro = - regex_must_compile(US"^[A-Za-z0-9_/.-]*$", FALSE, TRUE); + regex_must_compile(US"^[A-Za-z0-9_/.-]*$", MCS_NOFLAGS, TRUE); #endif for (i = 0; i < REGEX_VARS; i++) regex_vars[i] = NULL; @@ -2216,7 +2213,7 @@ on the second character (the one after '-'), to save some effort. */ -bdf: Ditto, but in the foreground. */ case 'd': - f.daemon_listen = TRUE; + f.daemon_listen = f.daemon_scion = TRUE; if (*argrest == 'f') f.background_daemon = FALSE; else if (*argrest) badarg = TRUE; break; @@ -2476,7 +2473,7 @@ on the second character (the one after '-'), to save some effort. */ case 'w': f.inetd_wait_mode = TRUE; f.background_daemon = FALSE; - f.daemon_listen = TRUE; + f.daemon_listen = f.daemon_scion = TRUE; if (*argrest) if ((inetd_wait_timeout = readconf_readtime(argrest, 0, FALSE)) <= 0) exim_fail("exim: bad time value %s: abandoned\n", argv[i]); @@ -5003,7 +5000,7 @@ for (i = 0;;) if (gecos_pattern && gecos_name) { const pcre2_code *re; - re = regex_must_compile(gecos_pattern, FALSE, TRUE); /* Use malloc */ + re = regex_must_compile(gecos_pattern, MCS_NOFLAGS, TRUE); /* Use malloc */ if (regex_match_and_setup(re, name, 0, -1)) { @@ -5399,7 +5396,10 @@ if (host_checking) memset(sender_host_cache, 0, sizeof(sender_host_cache)); if (verify_check_host(&hosts_connection_nolog) == OK) + { BIT_CLEAR(log_selector, log_selector_size, Li_smtp_connection); + BIT_CLEAR(log_selector, log_selector_size, Li_smtp_no_mail); + } log_write(L_smtp_connection, LOG_MAIN, "%s", smtp_get_connection_info()); /* NOTE: We do *not* call smtp_log_no_mail() if smtp_start_session() fails, @@ -5588,7 +5588,10 @@ if (smtp_input) smtp_out = stdout; memset(sender_host_cache, 0, sizeof(sender_host_cache)); if (verify_check_host(&hosts_connection_nolog) == OK) + { BIT_CLEAR(log_selector, log_selector_size, Li_smtp_connection); + BIT_CLEAR(log_selector, log_selector_size, Li_smtp_no_mail); + } log_write(L_smtp_connection, LOG_MAIN, "%s", smtp_get_connection_info()); if (!smtp_start_session()) { diff --git a/src/src/expand.c b/src/src/expand.c index 36c9f423b..ffbdc14e5 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -12,9 +12,15 @@ #include "exim.h" +typedef unsigned esi_flags; +#define ESI_NOFLAGS 0 +#define ESI_BRACE_ENDS BIT(0) /* expansion should stop at } */ +#define ESI_HONOR_DOLLAR BIT(1) /* $ is meaningfull */ +#define ESI_SKIPPING BIT(2) /* value will not be needed */ + /* Recursively called function */ -static uschar *expand_string_internal(const uschar *, BOOL, const uschar **, BOOL, BOOL, BOOL *); +static uschar *expand_string_internal(const uschar *, esi_flags, const uschar **, BOOL *, BOOL *); static int_eximarith_t expanded_string_integer(const uschar *, BOOL); #ifdef STAND_ALONE @@ -686,6 +692,7 @@ static var_entry var_table[] = { { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure }, { "recipients", vtype_string_func, (void *) &fn_recipients }, { "recipients_count", vtype_int, &recipients_count }, + { "regex_cachesize", vtype_int, ®ex_cachesize },/* undocumented; devel observability */ #ifdef WITH_CONTENT_SCAN { "regex_match_string", vtype_stringptr, ®ex_match_string }, #endif @@ -1748,9 +1755,7 @@ uschar buf[16]; int fd; ssize_t len; const uschar * where; -#ifndef EXIM_HAVE_ABSTRACT_UNIX_SOCKETS uschar * sname; -#endif if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) < 0) { @@ -1758,17 +1763,9 @@ if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) < 0) return NULL; } -#ifdef EXIM_HAVE_ABSTRACT_UNIX_SOCKETS -sa_un.sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ -len = offsetof(struct sockaddr_un, sun_path) + 1 - + snprintf(sa_un.sun_path+1, sizeof(sa_un.sun_path)-1, "exim_%d", getpid()); -#else -sname = string_sprintf("%s/p_%d", spool_directory, getpid()); -len = offsetof(struct sockaddr_un, sun_path) - + snprintf(sa_un.sun_path, sizeof(sa_un.sun_path), "%s", sname); -#endif +len = daemon_client_sockname(&sa_un, &sname); -if (bind(fd, (const struct sockaddr *)&sa_un, len) < 0) +if (bind(fd, (const struct sockaddr *)&sa_un, (socklen_t)len) < 0) { where = US"bind"; goto bad; } #ifdef notdef @@ -1777,17 +1774,7 @@ debug_printf("local addr '%s%s'\n", sa_un.sun_path + (*sa_un.sun_path ? 0 : 1)); #endif -#ifdef EXIM_HAVE_ABSTRACT_UNIX_SOCKETS -sa_un.sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ -len = offsetof(struct sockaddr_un, sun_path) + 1 - + snprintf(sa_un.sun_path+1, sizeof(sa_un.sun_path)-1, "%s", - expand_string(notifier_socket)); -#else -len = offsetof(struct sockaddr_un, sun_path) - + snprintf(sa_un.sun_path, sizeof(sa_un.sun_path), "%s", - expand_string(notifier_socket)); -#endif - +len = daemon_notifier_sockname(&sa_un); if (connect(fd, (const struct sockaddr *)&sa_un, len) < 0) { where = US"connect"; goto bad2; } @@ -2114,27 +2101,33 @@ Arguments: n maximum number of substrings m minimum required sptr points to current string pointer - skipping the skipping flag + flags + skipping the skipping flag check_end if TRUE, check for final '}' name name of item, for error message resetok if not NULL, pointer to flag - write FALSE if unsafe to reset - the store. + the store + textonly_p if not NULL, pointer to bitmask of which subs were text-only + (did not change when expended) -Returns: 0 OK; string pointer updated +Returns: -1 OK; string pointer updated, but in "skipping" mode + 0 OK; string pointer updated 1 curly bracketing error (too few arguments) 2 too many arguments (only if check_end is set); message set 3 other error (expansion failure) */ static int -read_subs(uschar **sub, int n, int m, const uschar **sptr, BOOL skipping, - BOOL check_end, uschar *name, BOOL *resetok) +read_subs(uschar ** sub, int n, int m, const uschar ** sptr, esi_flags flags, + BOOL check_end, uschar * name, BOOL * resetok, unsigned * textonly_p) { -const uschar *s = *sptr; +const uschar * s = *sptr; +unsigned textonly_l = 0; Uskip_whitespace(&s); for (int i = 0; i < n; i++) { + BOOL textonly; if (*s != '{') { if (i < m) @@ -2146,11 +2139,14 @@ for (int i = 0; i < n; i++) sub[i] = NULL; break; } - if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok))) + if (!(sub[i] = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags & ESI_SKIPPING, &s, resetok, + textonly_p ? &textonly : NULL))) return 3; if (*s++ != '}') return 1; + if (textonly_p && textonly) textonly_l |= BIT(i); Uskip_whitespace(&s); - } + } /*{*/ if (check_end && *s++ != '}') { if (s[-1] == '{') @@ -2163,8 +2159,9 @@ if (check_end && *s++ != '}') return 1; } +if (textonly_p) *textonly_p = textonly_l; *sptr = s; -return 0; +return flags & ESI_SKIPPING ? -1 : 0; } @@ -2523,11 +2520,11 @@ Returns: a pointer to the first character after the condition, or */ static const uschar * -eval_condition(const uschar *s, BOOL *resetok, BOOL *yield) +eval_condition(const uschar * s, BOOL * resetok, BOOL * yield) { BOOL testfor = TRUE; BOOL tempcond, combined_cond; -BOOL *subcondptr; +BOOL * subcondptr; BOOL sub2_honour_dollar = TRUE; BOOL is_forany, is_json, is_jsons; int rc, cond_type; @@ -2535,7 +2532,8 @@ int_eximarith_t num[2]; struct stat statbuf; uschar * opname; uschar name[256]; -const uschar *sub[10]; +const uschar * sub[10]; +unsigned sub_textonly = 0; for (;;) if (Uskip_whitespace(&s) == '!') { testfor = !testfor; s++; } else break; @@ -2629,8 +2627,14 @@ switch(cond_type = identify_operator(&s, &opname)) if (Uskip_whitespace(&s) != '{') goto COND_FAILED_CURLY_START; /* }-for-text-editors */ - sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok); - if (!sub[0]) return NULL; + { + BOOL textonly; + sub[0] = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | (yield ? ESI_NOFLAGS : ESI_SKIPPING), + &s, resetok, &textonly); + if (!sub[0]) return NULL; + if (textonly) sub_textonly |= BIT(0); + } /* {-for-text-editors */ if (*s++ != '}') goto COND_FAILED_CURLY_END; @@ -2727,8 +2731,8 @@ switch(cond_type = identify_operator(&s, &opname)) Uskip_whitespace(&s); if (*s++ != '{') goto COND_FAILED_CURLY_START; /*}*/ - switch(read_subs(sub, nelem(sub), 1, - &s, yield == NULL, TRUE, name, resetok)) + switch(read_subs(sub, nelem(sub), 1, &s, + yield ? ESI_NOFLAGS : ESI_SKIPPING, TRUE, name, resetok, NULL)) { case 1: expand_string_message = US"too few arguments or bracketing " "error for acl"; @@ -2779,8 +2783,8 @@ switch(cond_type = identify_operator(&s, &opname)) uschar *sub[4]; Uskip_whitespace(&s); if (*s++ != '{') goto COND_FAILED_CURLY_START; /* }-for-text-editors */ - switch(read_subs(sub, nelem(sub), 2, &s, yield == NULL, TRUE, name, - resetok)) + switch(read_subs(sub, nelem(sub), 2, &s, + yield ? ESI_NOFLAGS : ESI_SKIPPING, TRUE, name, resetok, NULL)) { case 1: expand_string_message = US"too few arguments or bracketing " "error for saslauthd"; @@ -2848,12 +2852,16 @@ switch(cond_type = identify_operator(&s, &opname)) for (int i = 0; i < 2; i++) { + BOOL textonly; /* Sometimes, we don't expand substrings; too many insecure configurations created using match_address{}{} and friends, where the second param includes information from untrustworthy sources. */ - BOOL honour_dollar = TRUE; - if ((i > 0) && !sub2_honour_dollar) - honour_dollar = FALSE; + /*XXX is this moot given taint-tracking? */ + + esi_flags flags = ESI_BRACE_ENDS; + + if (!(i > 0 && !sub2_honour_dollar)) flags |= ESI_HONOR_DOLLAR; + if (!yield) flags |= ESI_SKIPPING; if (Uskip_whitespace(&s) != '{') { @@ -2862,9 +2870,9 @@ switch(cond_type = identify_operator(&s, &opname)) "after \"%s\"", opname); return NULL; } - if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL, - honour_dollar, resetok))) + if (!(sub[i] = expand_string_internal(s+1, flags, &s, resetok, &textonly))) return NULL; + if (textonly) sub_textonly |= BIT(i); DEBUG(D_expand) if (i == 1 && !sub2_honour_dollar && Ustrchr(sub[1], '$')) debug_printf_indent("WARNING: the second arg is NOT expanded," " for security reasons\n"); @@ -2944,19 +2952,11 @@ switch(cond_type = identify_operator(&s, &opname)) case ECOND_MATCH: /* Regular expression match */ { - const pcre2_code * re; - PCRE2_SIZE offset; - int err; - - if (!(re = pcre2_compile((PCRE2_SPTR)sub[1], PCRE2_ZERO_TERMINATED, - PCRE_COPT, &err, &offset, pcre_cmp_ctx))) - { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - expand_string_message = string_sprintf("regular expression error in " - "\"%s\": %s at offset %ld", sub[1], errbuf, (long)offset); + const pcre2_code * re = regex_compile(sub[1], + sub_textonly & BIT(1) ? MCS_CACHEABLE : MCS_NOFLAGS, + &expand_string_message, pcre_gen_cmp_ctx); + if (!re) return NULL; - } tempcond = regex_match_and_setup(re, sub[0], 0, -1); break; @@ -3274,7 +3274,9 @@ switch(cond_type = identify_operator(&s, &opname)) Uskip_whitespace(&s); if (*s++ != '{') goto COND_FAILED_CURLY_START; /* }-for-text-editors */ - if (!(sub[0] = expand_string_internal(s, TRUE, &s, yield == NULL, TRUE, resetok))) + if (!(sub[0] = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | (yield ? ESI_NOFLAGS : ESI_SKIPPING), + &s, resetok, NULL))) return NULL; /* {-for-text-editors */ if (*s++ != '}') goto COND_FAILED_CURLY_END; @@ -3362,7 +3364,8 @@ switch(cond_type = identify_operator(&s, &opname)) if (Uskip_whitespace(&s) != '{') goto COND_FAILED_CURLY_START; /* }-for-text-editors */ ourname = cond_type == ECOND_BOOL_LAX ? US"bool_lax" : US"bool"; - switch(read_subs(sub_arg, 1, 1, &s, yield == NULL, FALSE, ourname, resetok)) + switch(read_subs(sub_arg, 1, 1, &s, + yield ? ESI_NOFLAGS : ESI_SKIPPING, FALSE, ourname, resetok, NULL)) { case 1: expand_string_message = string_sprintf( "too few arguments or bracketing error for %s", @@ -3430,7 +3433,8 @@ switch(cond_type = identify_operator(&s, &opname)) uschar cksum[4]; BOOL boolvalue = FALSE; - switch(read_subs(sub, 2, 2, CUSS &s, yield == NULL, FALSE, name, resetok)) + switch(read_subs(sub, 2, 2, CUSS &s, + yield ? ESI_NOFLAGS : ESI_SKIPPING, FALSE, name, resetok, NULL)) { case 1: expand_string_message = US"too few arguments or bracketing " "error for inbound_srs"; @@ -3441,10 +3445,10 @@ switch(cond_type = identify_operator(&s, &opname)) /* Match the given local_part against the SRS-encoded pattern */ re = regex_must_compile(US"^(?i)SRS0=([^=]+)=([A-Z2-7]+)=([^=]*)=(.*)$", - TRUE, FALSE); + MCS_CASELESS | MCS_CACHEABLE, FALSE); md = pcre2_match_data_create(4+1, pcre_gen_ctx); if (pcre2_match(re, sub[0], PCRE2_ZERO_TERMINATED, 0, PCRE_EOPT, - md, pcre_mtc_ctx) < 0) + md, pcre_gen_mtc_ctx) < 0) { DEBUG(D_expand) debug_printf("no match for SRS'd local-part pattern\n"); goto srs_result; @@ -3521,6 +3525,7 @@ switch(cond_type = identify_operator(&s, &opname)) boolvalue = TRUE; srs_result: + /* pcre2_match_data_free(md); gen ctx needs no free */ if (yield) *yield = (boolvalue == testfor); return s; } @@ -3628,7 +3633,8 @@ expanded, to check their syntax, but "skipping" is set when the result is not needed - this avoids unnecessary nested lookups. Arguments: - skipping TRUE if we were skipping when this item was reached + flags + skipping TRUE if we were skipping when this item was reached yes TRUE if the first string is to be used, else use the second save_lookup a value to put back into lookup_value before the 2nd expansion sptr points to the input string pointer @@ -3644,7 +3650,7 @@ Returns: 0 OK; lookup_value has been reset to save_lookup */ static int -process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, const uschar **sptr, +process_yesno(esi_flags flags, BOOL yes, uschar *save_lookup, const uschar **sptr, gstring ** yieldptr, uschar *type, BOOL *resetok) { int rc = 0; @@ -3652,6 +3658,8 @@ const uschar *s = *sptr; /* Local value */ uschar *sub1, *sub2; const uschar * errwhere; +flags &= ESI_SKIPPING; /* Ignore all buf the skipping flag */ + /* If there are no following strings, we substitute the contents of $value for lookups and for extractions in the success case. For the ${if item, the string "true" is substituted. In the fail case, nothing is substituted for all three @@ -3661,12 +3669,12 @@ if (skip_whitespace(&s) == '}') { if (type[0] == 'i') { - if (yes && !skipping) + if (yes && !(flags & ESI_SKIPPING)) *yieldptr = string_catn(*yieldptr, US"true", 4); } else { - if (yes && lookup_value && !skipping) + if (yes && lookup_value && !(flags & ESI_SKIPPING)) *yieldptr = string_cat(*yieldptr, lookup_value); lookup_value = save_lookup; } @@ -3678,7 +3686,7 @@ if (skip_whitespace(&s) == '}') if (*s++ != '{') { - errwhere = US"'yes' part did not start with '{'"; + errwhere = US"'yes' part did not start with '{'"; /*}}*/ goto FAILED_CURLY; } @@ -3686,9 +3694,12 @@ if (*s++ != '{') want this string. Set skipping in the call in the fail case (this will always be the case if we were already skipping). */ -sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok); +sub1 = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | (yes ? ESI_NOFLAGS : ESI_SKIPPING), + &s, resetok, NULL); if (sub1 == NULL && (yes || !f.expand_string_forcedfail)) goto FAILED; f.expand_string_forcedfail = FALSE; + /*{{*/ if (*s++ != '}') { errwhere = US"'yes' part did not end with '}'"; @@ -3713,14 +3724,16 @@ time, forced failures are noticed only if we want the second string. We must set skipping in the nested call if we don't want this string, or if we were already skipping. */ -if (skip_whitespace(&s) == '{') +if (skip_whitespace(&s) == '{') /*}*/ { - sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok); - if (sub2 == NULL && (!yes || !f.expand_string_forcedfail)) goto FAILED; - f.expand_string_forcedfail = FALSE; + esi_flags s_flags = ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags; + if (yes) s_flags |= ESI_SKIPPING; + sub2 = expand_string_internal(s+1, s_flags, &s, resetok, NULL); + if (!sub2 && (!yes || !f.expand_string_forcedfail)) goto FAILED; + f.expand_string_forcedfail = FALSE; /*{*/ if (*s++ != '}') { - errwhere = US"'no' part did not start with '{'"; + errwhere = US"'no' part did not start with '{'"; /*}*/ goto FAILED_CURLY; } @@ -3729,7 +3742,7 @@ if (skip_whitespace(&s) == '{') if (!yes) *yieldptr = string_cat(*yieldptr, sub2); } - + /*{{*/ /* If there is no second string, but the word "fail" is present when the use of the second string is wanted, set a flag indicating it was a forced failure rather than a syntactic error. Swallow the terminating } in case this is nested @@ -3742,9 +3755,9 @@ else if (*s != '}') s = US read_name(name, sizeof(name), s, US"_"); if (Ustrcmp(name, "fail") == 0) { - if (!yes && !skipping) + if (!yes && !(flags & ESI_SKIPPING)) { - Uskip_whitespace(&s); + Uskip_whitespace(&s); /*{{*/ if (*s++ != '}') { errwhere = US"did not close with '}' after forcedfail"; @@ -3766,7 +3779,7 @@ else if (*s != '}') /* All we have to do now is to check on the final closing brace. */ -skip_whitespace(&s); +skip_whitespace(&s); /*{{*/ if (*s++ != '}') { errwhere = US"did not close with '}'"; @@ -4445,15 +4458,17 @@ string expansion becoming too powerful. Arguments: string the string to be expanded - ket_ends true if expansion is to stop at } + flags + brace_ends expansion is to stop at } + honour_dollar TRUE if $ is to be expanded, + FALSE if it's just another character + skipping TRUE for recursive calls when the value isn't actually going + to be used (to allow for optimisation) left if not NULL, a pointer to the first character after the - expansion is placed here (typically used with ket_ends) - skipping TRUE for recursive calls when the value isn't actually going - to be used (to allow for optimisation) - honour_dollar TRUE if $ is to be expanded, - FALSE if it's just another character + expansion is placed here (typically used with brace_ends) resetok_p if not NULL, pointer to flag - write FALSE if unsafe to reset the store. + textonly_p if not NULL, pointer to flag - write bool for only-met-text Returns: NULL if expansion fails: expand_string_forcedfail is set TRUE if failure was forced @@ -4462,8 +4477,8 @@ Returns: NULL if expansion fails: */ static uschar * -expand_string_internal(const uschar *string, BOOL ket_ends, const uschar **left, - BOOL skipping, BOOL honour_dollar, BOOL *resetok_p) +expand_string_internal(const uschar * string, esi_flags flags, const uschar ** left, + BOOL *resetok_p, BOOL * textonly_p) { rmark reset_point = store_mark(); gstring * yield = string_get(Ustrlen(string) + 64); @@ -4471,7 +4486,7 @@ int item_type; const uschar * s = string; const uschar * save_expand_nstring[EXPAND_MAXN+1]; int save_expand_nlength[EXPAND_MAXN+1]; -BOOL resetok = TRUE, first = TRUE; +BOOL resetok = TRUE, first = TRUE, textonly = TRUE; expand_level++; f.expand_string_forcedfail = FALSE; @@ -4494,11 +4509,11 @@ while (*s) DEBUG(D_noutf8) debug_printf_indent("%c%s: %s\n", first ? '/' : '|', - skipping ? "---scanning" : "considering", s); + flags & ESI_SKIPPING ? "---scanning" : "considering", s); else debug_printf_indent("%s%s: %s\n", first ? UTF8_DOWN_RIGHT : UTF8_VERT_RIGHT, - skipping + flags & ESI_SKIPPING ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" : "considering", s); @@ -4524,7 +4539,7 @@ while (*s) for (s = t; *s ; s++) if (*s == '\\' && s[1] == 'N') break; DEBUG(D_expand) - debug_expansion_interim(US"protected", t, (int)(s - t), skipping); + debug_expansion_interim(US"protected", t, (int)(s - t), !!(flags & ESI_SKIPPING)); yield = string_catn(yield, t, s - t); if (*s) s += 2; } @@ -4547,20 +4562,21 @@ while (*s) /* Anything other than $ is just copied verbatim, unless we are looking for a terminating } character. */ - if (ket_ends && *s == '}') break; + if (flags & ESI_BRACE_ENDS && *s == '}') break; - if (*s != '$' || !honour_dollar) + if (*s != '$' || !(flags & ESI_HONOR_DOLLAR)) { int i = 1; /*{*/ for (const uschar * t = s+1; *t && *t != '$' && *t != '}' && *t != '\\'; t++) i++; - DEBUG(D_expand) debug_expansion_interim(US"text", s, i, skipping); + DEBUG(D_expand) debug_expansion_interim(US"text", s, i, !!(flags & ESI_SKIPPING)); yield = string_catn(yield, s, i); s += i; continue; } + textonly = FALSE; /* No { after the $ - must be a plain name or a number for string match variable. There has to be a fudge for variables that are the @@ -4622,7 +4638,7 @@ while (*s) /* Variable */ - else if (!(value = find_variable(name, FALSE, skipping, &newsize))) + else if (!(value = find_variable(name, FALSE, !!(flags & ESI_SKIPPING), &newsize))) { expand_string_message = string_sprintf("unknown variable name \"%s\"", name); @@ -4722,14 +4738,13 @@ while (*s) uschar * user_msg; int rc; - switch(read_subs(sub, nelem(sub), 1, &s, skipping, TRUE, name, - &resetok)) + switch(read_subs(sub, nelem(sub), 1, &s, flags, TRUE, name, &resetok, NULL)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - if (skipping) continue; resetok = FALSE; switch(rc = eval_acl(sub, nelem(sub), &user_msg)) @@ -4758,13 +4773,13 @@ while (*s) { uschar * sub_arg[1]; - switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name, - &resetok)) + switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, flags, TRUE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } + /*XXX no skipping-optimisation? */ yield = string_append(yield, 3, US"Authentication-Results: ", sub_arg[0], US"; none"); @@ -4802,14 +4817,14 @@ while (*s) uschar * save_lookup_value = lookup_value; Uskip_whitespace(&s); - if (!(next_s = eval_condition(s, &resetok, skipping ? NULL : &cond))) + if (!(next_s = eval_condition(s, &resetok, flags & ESI_SKIPPING ? NULL : &cond))) goto EXPAND_FAILED; /* message already set */ DEBUG(D_expand) { - debug_expansion_interim(US"condition", s, (int)(next_s - s), skipping); + debug_expansion_interim(US"condition", s, (int)(next_s - s), !!(flags & ESI_SKIPPING)); debug_expansion_interim(US"result", - cond ? US"true" : US"false", cond ? 4 : 5, skipping); + cond ? US"true" : US"false", cond ? 4 : 5, !!(flags & ESI_SKIPPING)); } s = next_s; @@ -4818,12 +4833,12 @@ while (*s) function that is also used by ${lookup} and ${extract} and ${run}. */ switch(process_yesno( - skipping, /* were previously skipping */ - cond, /* success/failure indicator */ - lookup_value, /* value to reset for string2 */ - &s, /* input pointer */ - &yield, /* output pointer */ - US"if", /* condition type */ + flags, /* were previously skipping */ + cond, /* success/failure indicator */ + lookup_value, /* value to reset for string2 */ + &s, /* input pointer */ + &yield, /* output pointer */ + US"if", /* condition type */ &resetok)) { case 1: goto EXPAND_FAILED; /* when all is well, the */ @@ -4845,13 +4860,13 @@ while (*s) uschar *sub_arg[3]; uschar *encoded; - switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name, - &resetok)) + switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, flags, TRUE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } + /*XXX no skipping-optimisation? */ if (!sub_arg[1]) /* One argument */ { @@ -4867,7 +4882,7 @@ while (*s) goto EXPAND_FAILED; } - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; if (!(encoded = imap_utf7_encode(sub_arg[0], headers_charset, sub_arg[1][0], sub_arg[2], &expand_string_message))) @@ -4906,7 +4921,8 @@ while (*s) if (Uskip_whitespace(&s) == '{') /*}*/ { - key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); + key = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!key) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -4976,7 +4992,8 @@ while (*s) expand_string_message = US"missing '{' for lookup file-or-query arg"; goto EXPAND_FAILED_CURLY; /*}}*/ } - if (!(filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))) + if (!(filename = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL))) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') @@ -5007,7 +5024,7 @@ while (*s) since new variables will have been set. Note that at the end of this "lookup" section, the old numeric variables are restored. */ - if (skipping) + if (flags & ESI_SKIPPING) lookup_value = NULL; else { @@ -5033,12 +5050,12 @@ while (*s) function that is also used by ${if} and ${extract}. */ switch(process_yesno( - skipping, /* were previously skipping */ - lookup_value != NULL, /* success/failure indicator */ - save_lookup_value, /* value to reset for string2 */ - &s, /* input pointer */ - &yield, /* output pointer */ - US"lookup", /* condition type */ + flags, /* were previously skipping */ + lookup_value != NULL, /* success/failure indicator */ + save_lookup_value, /* value to reset for string2 */ + &s, /* input pointer */ + &yield, /* output pointer */ + US"lookup", /* condition type */ &resetok)) { case 1: goto EXPAND_FAILED; /* when all is well, the */ @@ -5051,7 +5068,7 @@ while (*s) restore_expand_strings(save_expand_nmax, save_expand_nstring, save_expand_nlength); - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } @@ -5079,18 +5096,15 @@ while (*s) goto EXPAND_FAILED; } - switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, skipping, TRUE, - name, &resetok)) + switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, flags, TRUE, + name, &resetok, NULL)) { + case -1: continue; /* If skipping, we don't actually do anything */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - /* If skipping, we don't actually do anything */ - - if (skipping) continue; - /* Start the interpreter if necessary */ if (!opt_perl_started) @@ -5150,16 +5164,14 @@ while (*s) { uschar * sub_arg[3], * p, * domain; - switch(read_subs(sub_arg, 3, 2, &s, skipping, TRUE, name, &resetok)) + switch(read_subs(sub_arg, 3, 2, &s, flags, TRUE, name, &resetok, NULL)) { + case -1: continue; /* If skipping, we don't actually do anything */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - /* If skipping, we don't actually do anything */ - if (skipping) continue; - /* sub_arg[0] is the address */ if ( !(domain = Ustrrchr(sub_arg[0],'@')) || domain == sub_arg[0] || Ustrlen(domain) == 1) @@ -5208,30 +5220,21 @@ while (*s) gstring * g; const pcre2_code * re; - /* TF: Ugliness: We want to expand parameter 1 first, then set - up expansion variables that are used in the expansion of - parameter 2. So we clone the string for the first - expansion, where we only expand parameter 1. - - PH: Actually, that isn't necessary. The read_subs() function is - designed to work this way for the ${if and ${lookup expansions. I've - tidied the code. - */ /*}}*/ - /* Reset expansion variables */ prvscheck_result = NULL; prvscheck_address = NULL; prvscheck_keynum = NULL; - switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, name, &resetok)) + switch(read_subs(sub_arg, 1, 1, &s, flags, FALSE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - re = regex_must_compile(US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$", - TRUE,FALSE); + re = regex_must_compile( + US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$", + MCS_CASELESS | MCS_CACHEABLE, FALSE); if (regex_match_and_setup(re,sub_arg[0],0,-1)) { @@ -5241,11 +5244,14 @@ while (*s) uschar * hash = string_copyn(expand_nstring[3],expand_nlength[3]); uschar * domain = string_copyn(expand_nstring[5],expand_nlength[5]); - DEBUG(D_expand) debug_printf_indent("prvscheck localpart: %s\n", local_part); - DEBUG(D_expand) debug_printf_indent("prvscheck key number: %s\n", key_num); - DEBUG(D_expand) debug_printf_indent("prvscheck daystamp: %s\n", daystamp); - DEBUG(D_expand) debug_printf_indent("prvscheck hash: %s\n", hash); - DEBUG(D_expand) debug_printf_indent("prvscheck domain: %s\n", domain); + DEBUG(D_expand) + { + debug_printf_indent("prvscheck localpart: %s\n", local_part); + debug_printf_indent("prvscheck key number: %s\n", key_num); + debug_printf_indent("prvscheck daystamp: %s\n", daystamp); + debug_printf_indent("prvscheck hash: %s\n", hash); + debug_printf_indent("prvscheck domain: %s\n", domain); + } /* Set up expansion variables */ g = string_cat (NULL, local_part); @@ -5255,7 +5261,7 @@ while (*s) prvscheck_keynum = string_copy(key_num); /* Now expand the second argument */ - switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, name, &resetok)) + switch(read_subs(sub_arg, 1, 1, &s, flags, FALSE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: @@ -5266,7 +5272,6 @@ while (*s) p = prvs_hmac_sha1(prvscheck_address, sub_arg[0], prvscheck_keynum, daystamp); - if (!p) { expand_string_message = US"hmac-sha1 conversion failed"; @@ -5309,7 +5314,7 @@ while (*s) /* Now expand the final argument. We leave this till now so that it can include $prvscheck_result. */ - switch(read_subs(sub_arg, 1, 0, &s, skipping, TRUE, name, &resetok)) + switch(read_subs(sub_arg, 1, 0, &s, flags, TRUE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: @@ -5330,14 +5335,14 @@ while (*s) We need to make sure all subs are expanded first, so as to skip over the entire item. */ - switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, name, &resetok)) + switch(read_subs(sub_arg, 2, 1, &s, flags, TRUE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } @@ -5354,7 +5359,7 @@ while (*s) goto EXPAND_FAILED; } - switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, name, &resetok)) + switch(read_subs(sub_arg, 2, 1, &s, flags, TRUE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: @@ -5363,7 +5368,7 @@ while (*s) /* If skipping, we don't actually do anything */ - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; /* Open the file and read it */ @@ -5395,7 +5400,7 @@ while (*s) /* Read up to 4 arguments, but don't do the end of item check afterwards, because there may be a string for expansion on failure. */ - switch(read_subs(sub_arg, 4, 2, &s, skipping, FALSE, name, &resetok)) + switch(read_subs(sub_arg, 4, 2, &s, flags, FALSE, name, &resetok, NULL)) { case 1: goto EXPAND_FAILED_CURLY; case 2: /* Won't occur: no end check */ @@ -5405,7 +5410,7 @@ while (*s) /* If skipping, we don't actually do anything. Otherwise, arrange to connect to either an IP or a Unix socket. */ - if (!skipping) + if (!(flags & ESI_SKIPPING)) { int stype = search_findtype(US"readsock", 8); gstring * g = NULL; @@ -5483,7 +5488,8 @@ while (*s) if (*s == '{') /*}*/ { - if (!expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok)) + if (!expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | ESI_SKIPPING, &s, &resetok, NULL)) goto EXPAND_FAILED; /*{*/ if (*s++ != '}') { /*{*/ @@ -5499,7 +5505,7 @@ while (*s) expand_string_message = US"missing '}' closing readsocket"; goto EXPAND_FAILED_CURLY; } - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; /* Come here on failure to create socket, connect socket, write to the @@ -5509,7 +5515,8 @@ while (*s) SOCK_FAIL: if (*s != '{') goto EXPAND_FAILED; /*}*/ DEBUG(D_any) debug_printf("%s\n", expand_string_message); - if (!(arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok))) + if (!(arg = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR, &s, &resetok, NULL))) goto EXPAND_FAILED; yield = string_cat(yield, arg); /*{*/ if (*s++ != '}') @@ -5560,14 +5567,15 @@ while (*s) s++; if (late_expand) /* this is the default case */ - { + { /*{*/ int n = Ustrcspn(s, "}"); - arg = skipping ? NULL : string_copyn(s, n); + arg = flags & ESI_SKIPPING ? NULL : string_copyn(s, n); s += n; } else { - if (!(arg = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok))) + if (!(arg = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL))) goto EXPAND_FAILED; Uskip_whitespace(&s); } @@ -5578,7 +5586,7 @@ while (*s) goto EXPAND_FAILED_CURLY; } - if (skipping) /* Just pretend it worked when we're skipping */ + if (flags & ESI_SKIPPING) /* Just pretend it worked when we're skipping */ { runrc = 0; lookup_value = NULL; @@ -5652,19 +5660,19 @@ while (*s) /* Process the yes/no strings; $value may be useful in both cases */ switch(process_yesno( - skipping, /* were previously skipping */ - runrc == 0, /* success/failure indicator */ - lookup_value, /* value to reset for string2 */ - &s, /* input pointer */ - &yield, /* output pointer */ - US"run", /* condition type */ + flags, /* were previously skipping */ + runrc == 0, /* success/failure indicator */ + lookup_value, /* value to reset for string2 */ + &s, /* input pointer */ + &yield, /* output pointer */ + US"run", /* condition type */ &resetok)) { case 1: goto EXPAND_FAILED; /* when all is well, the */ case 2: goto EXPAND_FAILED_CURLY; /* returned value is 0 */ } - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } @@ -5676,8 +5684,9 @@ while (*s) int o2m; uschar * sub[3]; - switch(read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok)) + switch(read_subs(sub, 3, 3, &s, flags, TRUE, name, &resetok, NULL)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; @@ -5696,7 +5705,6 @@ while (*s) } } - if (skipping) continue; break; } @@ -5717,9 +5725,10 @@ while (*s) Ensure that sub[2] is set in the ${length } case. */ sub[2] = NULL; - switch(read_subs(sub, (item_type == EITEM_LENGTH)? 2:3, 2, &s, skipping, - TRUE, name, &resetok)) + switch(read_subs(sub, item_type == EITEM_LENGTH ? 2:3, 2, &s, flags, + TRUE, name, &resetok, NULL)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; @@ -5760,7 +5769,6 @@ while (*s) if (!ret) goto EXPAND_FAILED; yield = string_catn(yield, ret, len); - if (skipping) continue; break; } @@ -5793,15 +5801,14 @@ while (*s) uschar innerkey[MAX_HASHBLOCKLEN]; uschar outerkey[MAX_HASHBLOCKLEN]; - switch (read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok)) + switch (read_subs(sub, 3, 3, &s, flags, TRUE, name, &resetok, NULL)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - if (skipping) continue; - if (Ustrcmp(sub[0], "md5") == 0) { type = HMAC_MD5; @@ -5881,32 +5888,29 @@ while (*s) { const pcre2_code * re; int moffset, moffsetextra, slen; - PCRE2_SIZE roffset; pcre2_match_data * md; - int err, emptyopt; + int emptyopt; uschar * subject, * sub[3]; int save_expand_nmax = save_expand_strings(save_expand_nstring, save_expand_nlength); + unsigned sub_textonly = 0; - switch(read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok)) + switch(read_subs(sub, 3, 3, &s, flags, TRUE, name, &resetok, &sub_textonly)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - /*XXX no handling of skipping? */ /* Compile the regular expression */ - if (!(re = pcre2_compile((PCRE2_SPTR)sub[1], PCRE2_ZERO_TERMINATED, - PCRE_COPT, &err, &roffset, pcre_cmp_ctx))) - { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - expand_string_message = string_sprintf("regular expression error in " - "\"%s\": %s at offset %ld", sub[1], errbuf, (long)roffset); + re = regex_compile(sub[1], + sub_textonly & BIT(1) ? MCS_CACHEABLE : MCS_NOFLAGS, + &expand_string_message, pcre_gen_cmp_ctx); + if (!re) goto EXPAND_FAILED; - } + md = pcre2_match_data_create(EXPAND_MAXN + 1, pcre_gen_ctx); /* Now run a loop to do the substitutions as often as necessary. It ends @@ -5922,7 +5926,7 @@ while (*s) { PCRE2_SIZE * ovec = pcre2_get_ovector_pointer(md); int n = pcre2_match(re, (PCRE2_SPTR)subject, slen, moffset + moffsetextra, - PCRE_EOPT | emptyopt, md, pcre_mtc_ctx); + PCRE_EOPT | emptyopt, md, pcre_gen_mtc_ctx); uschar * insert; /* No match - if we previously set PCRE_NOTEMPTY after a null match, this @@ -5984,9 +5988,9 @@ while (*s) /* All done - restore numerical variables. */ + /* pcre2_match_data_free(md); gen ctx needs no free */ restore_expand_strings(save_expand_nmax, save_expand_nstring, save_expand_nlength); - if (skipping) continue; break; } @@ -6021,11 +6025,12 @@ while (*s) available (eg. $item) hence cannot decide on numeric vs. keyed. Read a maximum of 5 arguments (including the yes/no) */ - if (skipping) + if (flags & ESI_SKIPPING) { for (int j = 5; j > 0 && *s == '{'; j--) /*'}'*/ { - if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok)) + if (!expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL)) goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { @@ -6052,7 +6057,8 @@ while (*s) { if (Uskip_whitespace(&s) == '{') /*'}'*/ { - if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))) + if (!(sub[i] = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL))) goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { @@ -6111,7 +6117,7 @@ while (*s) /* Extract either the numbered or the keyed substring into $value. If skipping, just pretend the extraction failed. */ - if (skipping) + if (flags & ESI_SKIPPING) lookup_value = NULL; else switch (fmt) { @@ -6205,12 +6211,12 @@ while (*s) be yes/no strings, as for lookup or if. */ switch(process_yesno( - skipping, /* were previously skipping */ - lookup_value != NULL, /* success/failure indicator */ - save_lookup_value, /* value to reset for string2 */ - &s, /* input pointer */ - &yield, /* output pointer */ - US"extract", /* condition type */ + flags, /* were previously skipping */ + lookup_value != NULL, /* success/failure indicator */ + save_lookup_value, /* value to reset for string2 */ + &s, /* input pointer */ + &yield, /* output pointer */ + US"extract", /* condition type */ &resetok)) { case 1: goto EXPAND_FAILED; /* when all is well, the */ @@ -6222,7 +6228,7 @@ while (*s) restore_expand_strings(save_expand_nmax, save_expand_nstring, save_expand_nlength); - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } @@ -6246,7 +6252,8 @@ while (*s) goto EXPAND_FAILED_CURLY; } - sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); + sub[i] = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!sub[i]) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -6271,7 +6278,7 @@ while (*s) while (len > 0 && isspace(p[len-1])) len--; p[len] = 0; - if (!*p && !skipping) + if (!*p && !(flags & ESI_SKIPPING)) { expand_string_message = US"first argument of \"listextract\" must " "not be empty"; @@ -6297,18 +6304,18 @@ while (*s) /* Extract the numbered element into $value. If skipping, just pretend the extraction failed. */ - lookup_value = skipping ? NULL : expand_getlistele(field_number, sub[1]); + lookup_value = flags & ESI_SKIPPING ? NULL : expand_getlistele(field_number, sub[1]); /* If no string follows, $value gets substituted; otherwise there can be yes/no strings, as for lookup or if. */ switch(process_yesno( - skipping, /* were previously skipping */ - lookup_value != NULL, /* success/failure indicator */ - save_lookup_value, /* value to reset for string2 */ - &s, /* input pointer */ - &yield, /* output pointer */ - US"listextract", /* condition type */ + flags, /* were previously skipping */ + lookup_value != NULL, /* success/failure indicator */ + save_lookup_value, /* value to reset for string2 */ + &s, /* input pointer */ + &yield, /* output pointer */ + US"listextract", /* condition type */ &resetok)) { case 1: goto EXPAND_FAILED; /* when all is well, the */ @@ -6320,15 +6327,16 @@ while (*s) restore_expand_strings(save_expand_nmax, save_expand_nstring, save_expand_nlength); - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } case EITEM_LISTQUOTE: { uschar * sub[2]; - switch(read_subs(sub, 2, 2, &s, skipping, TRUE, name, &resetok)) + switch(read_subs(sub, 2, 2, &s, flags, TRUE, name, &resetok, NULL)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; @@ -6339,7 +6347,6 @@ while (*s) yield = string_catn(yield, sub[1], 1); } else yield = string_catn(yield, US" ", 1); - if (skipping) continue; break; } @@ -6356,7 +6363,8 @@ while (*s) expand_string_message = US"missing '{' for field arg of certextract"; goto EXPAND_FAILED_CURLY; /*}*/ } - sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); + sub[0] = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!sub[0]) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -6388,7 +6396,8 @@ while (*s) "be a certificate variable"; goto EXPAND_FAILED; } - sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok); + sub[1] = expand_string_internal(s+1, + ESI_BRACE_ENDS | flags & ESI_SKIPPING, &s, &resetok, NULL); if (!sub[1]) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -6396,7 +6405,7 @@ while (*s) goto EXPAND_FAILED_CURLY; } - if (skipping) + if (flags & ESI_SKIPPING) lookup_value = NULL; else { @@ -6404,12 +6413,12 @@ while (*s) if (*expand_string_message) goto EXPAND_FAILED; } switch(process_yesno( - skipping, /* were previously skipping */ - lookup_value != NULL, /* success/failure indicator */ - save_lookup_value, /* value to reset for string2 */ - &s, /* input pointer */ - &yield, /* output pointer */ - US"certextract", /* condition type */ + flags, /* were previously skipping */ + lookup_value != NULL, /* success/failure indicator */ + save_lookup_value, /* value to reset for string2 */ + &s, /* input pointer */ + &yield, /* output pointer */ + US"certextract", /* condition type */ &resetok)) { case 1: goto EXPAND_FAILED; /* when all is well, the */ @@ -6418,7 +6427,7 @@ while (*s) restore_expand_strings(save_expand_nmax, save_expand_nstring, save_expand_nlength); - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } #endif /*DISABLE_TLS*/ @@ -6443,7 +6452,8 @@ while (*s) goto EXPAND_FAILED_CURLY; /*}*/ } - if (!(list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok))) + if (!(list = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL))) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -6461,7 +6471,8 @@ while (*s) expand_string_message = US"missing '{' for second arg of reduce"; goto EXPAND_FAILED_CURLY; /*}*/ } - t = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok); + t = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!t) goto EXPAND_FAILED; lookup_value = t; /*{{*/ if (*s++ != '}') @@ -6488,7 +6499,8 @@ while (*s) the normal internal expansion function. */ if (item_type != EITEM_FILTER) - temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok); + temp = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | ESI_SKIPPING, &s, &resetok, NULL); else if ((temp = eval_condition(expr, &resetok, NULL))) s = temp; @@ -6519,7 +6531,7 @@ while (*s) /* If we are skipping, we can now just move on to the next item. When processing for real, we perform the iteration. */ - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0))) { *outsep = (uschar)sep; /* Separator as a string */ @@ -6550,7 +6562,8 @@ while (*s) else { - uschar * t = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok); + uschar * t = expand_string_internal(expr, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, NULL, &resetok, NULL); temp = t; if (!temp) { @@ -6622,7 +6635,7 @@ while (*s) /* Restore preserved $item */ iterate_item = save_iterate_item; - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } @@ -6641,7 +6654,8 @@ while (*s) goto EXPAND_FAILED_CURLY; /*}*/ } - srclist = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok); + srclist = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!srclist) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -6656,7 +6670,8 @@ while (*s) goto EXPAND_FAILED_CURLY; /*}*/ } - cmp = expand_string_internal(s, TRUE, &s, skipping, FALSE, &resetok); + cmp = expand_string_internal(s, + ESI_BRACE_ENDS | flags & ESI_SKIPPING, &s, &resetok, NULL); if (!cmp) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -6691,7 +6706,8 @@ while (*s) } xtract = s; - if (!(tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok))) + if (!(tmp = expand_string_internal(s, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | ESI_SKIPPING, &s, &resetok, NULL))) goto EXPAND_FAILED; xtract = string_copyn(xtract, s - xtract); /*{{*/ @@ -6707,7 +6723,7 @@ while (*s) goto EXPAND_FAILED; } - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; while ((srcitem = string_nextinlist(&srclist, &sep, NULL, 0))) { @@ -6718,8 +6734,8 @@ while (*s) /* extract field for comparisons */ iterate_item = srcitem; - if ( !(srcfield = expand_string_internal(xtract, FALSE, NULL, FALSE, - TRUE, &resetok)) + if ( !(srcfield = expand_string_internal(xtract, + ESI_HONOR_DOLLAR, NULL, &resetok, NULL)) || !*srcfield) { expand_string_message = string_sprintf( @@ -6824,18 +6840,15 @@ while (*s) goto EXPAND_FAILED; } - switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, skipping, - TRUE, name, &resetok)) + switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, flags, + TRUE, name, &resetok, NULL)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - /* If skipping, we don't actually do anything */ - - if (skipping) continue; - /* Look up the dynamically loaded object handle in the tree. If it isn't found, dlopen() the file and put the handle in the tree for next time. */ @@ -6901,7 +6914,8 @@ while (*s) if (Uskip_whitespace(&s) != '{') /*}*/ goto EXPAND_FAILED; - key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); + key = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!key) goto EXPAND_FAILED; /*{{*/ if (*s++ != '}') { @@ -6912,18 +6926,18 @@ while (*s) lookup_value = US getenv(CS key); switch(process_yesno( - skipping, /* were previously skipping */ - lookup_value != NULL, /* success/failure indicator */ - save_lookup_value, /* value to reset for string2 */ - &s, /* input pointer */ - &yield, /* output pointer */ - US"env", /* condition type */ + flags, /* were previously skipping */ + lookup_value != NULL, /* success/failure indicator */ + save_lookup_value, /* value to reset for string2 */ + &s, /* input pointer */ + &yield, /* output pointer */ + US"env", /* condition type */ &resetok)) { case 1: goto EXPAND_FAILED; /* when all is well, the */ case 2: goto EXPAND_FAILED_CURLY; /* returned value is 0 */ } - if (skipping) continue; + if (flags & ESI_SKIPPING) continue; break; } @@ -6936,13 +6950,13 @@ while (*s) gstring * g = NULL; BOOL quoted = FALSE; - switch (read_subs(sub, 3, 3, CUSS &s, skipping, TRUE, name, &resetok)) + switch (read_subs(sub, 3, 3, CUSS &s, flags, TRUE, name, &resetok, NULL)) { + case -1: continue; /* skipping */ case 1: goto EXPAND_FAILED_CURLY; case 2: case 3: goto EXPAND_FAILED; } - if (skipping) continue; if (sub[1] && *(sub[1])) { @@ -7024,7 +7038,7 @@ while (*s) DEBUG(D_expand) if (yield && (start > 0 || *s)) /* only if not the sole expansion of the line */ debug_expansion_interim(US"item-res", - yield->s + start, yield->ptr - start, skipping); + yield->s + start, yield->ptr - start, !!(flags & ESI_SKIPPING)); continue; NOT_ITEM: ; @@ -7070,8 +7084,8 @@ NOT_ITEM: ; if (s[1] == '$') { const uschar * s1 = s; - sub = expand_string_internal(s+2, TRUE, &s1, skipping, - FALSE, &resetok); + sub = expand_string_internal(s+2, + ESI_BRACE_ENDS | flags & ESI_SKIPPING, &s1, &resetok, NULL); if (!sub) goto EXPAND_FAILED; /*{*/ if (*s1 != '}') { /*{*/ @@ -7089,7 +7103,8 @@ NOT_ITEM: ; /*FALLTHROUGH*/ #endif default: - sub = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); + sub = expand_string_internal(s+1, + ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!sub) goto EXPAND_FAILED; s++; break; @@ -7101,7 +7116,7 @@ NOT_ITEM: ; for the existence of $sender_host_address before trying to mask it. For other operations, doing them may not fail, but it is a waste of time. */ - if (skipping && c >= 0) continue; + if (flags & ESI_SKIPPING && c >= 0) continue; /* Otherwise, switch on the operator type. After handling go back to the main loop top. */ @@ -7186,7 +7201,8 @@ NOT_ITEM: ; case EOP_EXPAND: { - uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok); + uschar *expanded = expand_string_internal(sub, + ESI_HONOR_DOLLAR | flags & ESI_SKIPPING, NULL, &resetok, NULL); if (!expanded) { expand_string_message = @@ -8177,7 +8193,7 @@ NOT_ITEM: ; debug_printf_indent("|-----op-res: %.*s\n", i, s); if (tainted) { - debug_printf_indent("%s \\__", skipping ? "| " : " "); + debug_printf_indent("%s \\__", flags & ESI_SKIPPING ? "| " : " "); debug_print_taint(yield->s); } } @@ -8189,7 +8205,7 @@ NOT_ITEM: ; if (tainted) { debug_printf_indent("%s", - skipping + flags & ESI_SKIPPING ? UTF8_VERT " " : " " UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ); debug_print_taint(yield->s); } @@ -8223,7 +8239,7 @@ NOT_ITEM: ; reset_point = store_mark(); g = store_get(sizeof(gstring), GET_UNTAINTED); /* alloc _before_ calling find_variable() */ } - if (!(value = find_variable(name, FALSE, skipping, &newsize))) + if (!(value = find_variable(name, FALSE, !!(flags & ESI_SKIPPING), &newsize))) { expand_string_message = string_sprintf("unknown variable in \"${%s}\"", name); @@ -8251,11 +8267,11 @@ NOT_ITEM: ; goto EXPAND_FAILED; } -/* If we hit the end of the string when ket_ends is set, there is a missing +/* If we hit the end of the string when brace_ends is set, there is a missing terminating brace. */ -if (ket_ends && !*s) - { +if (flags & ESI_BRACE_ENDS && !*s) + { /*{{*/ expand_string_message = malformed_header ? US"missing } at end of string - could be header name not terminated by colon" : US"missing } at end of string"; @@ -8285,13 +8301,13 @@ DEBUG(D_expand) { debug_printf_indent("|--expanding: %.*s\n", (int)(s - string), string); debug_printf_indent("%sresult: %s\n", - skipping ? "|-----" : "\\_____", yield->s); + flags & ESI_SKIPPING ? "|-----" : "\\_____", yield->s); if (tainted) { - debug_printf_indent("%s \\__", skipping ? "| " : " "); + debug_printf_indent("%s \\__", flags & ESI_SKIPPING ? "| " : " "); debug_print_taint(yield->s); } - if (skipping) + if (flags & ESI_SKIPPING) debug_printf_indent("\\___skipping: result is not used\n"); } else @@ -8301,20 +8317,21 @@ DEBUG(D_expand) (int)(s - string), string); debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "result: %s\n", - skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, + flags & ESI_SKIPPING ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, yield->s); if (tainted) { debug_printf_indent("%s", - skipping + flags & ESI_SKIPPING ? UTF8_VERT " " : " " UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ); debug_print_taint(yield->s); } - if (skipping) + if (flags & ESI_SKIPPING) debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "skipping: result is not used\n"); } } +if (textonly_p) *textonly_p = textonly; expand_level--; return yield->s; @@ -8363,16 +8380,20 @@ return NULL; } + /* This is the external function call. Do a quick check for any expansion metacharacters, and if there are none, just return the input string. -Argument: the string to be expanded +Arguments + the string to be expanded + optional pointer for return boolean indicating no-dynamic-expansions + Returns: the expanded string, or NULL if expansion failed; if failure was due to a lookup deferring, search_find_defer will be TRUE */ const uschar * -expand_cstring(const uschar * string) +expand_string_2(const uschar * string, BOOL * textonly_p) { if (Ustrpbrk(string, "$\\") != NULL) { @@ -8382,19 +8403,22 @@ if (Ustrpbrk(string, "$\\") != NULL) f.search_find_defer = FALSE; malformed_header = FALSE; store_pool = POOL_MAIN; - s = expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); + s = expand_string_internal(string, ESI_HONOR_DOLLAR, NULL, NULL, textonly_p); store_pool = old_pool; return s; } +if (textonly_p) *textonly_p = TRUE; return string; } +const uschar * +expand_cstring(const uschar * string) +{ return expand_string_2(string, NULL); } uschar * expand_string(uschar * string) -{ -return US expand_cstring(CUS string); -} +{ return US expand_string_2(CUS string, NULL); } + diff --git a/src/src/filter.c b/src/src/filter.c index ad017e567..cc4af230e 100644 --- a/src/src/filter.c +++ b/src/src/filter.c @@ -1424,213 +1424,203 @@ Returns: TRUE if the condition is met */ static BOOL -test_condition(condition_block *c, BOOL toplevel) +test_condition(condition_block * c, BOOL toplevel) { -BOOL yield = FALSE; -const uschar *exp[2], * p, * pp; +BOOL yield = FALSE, textonly_re; +const uschar * exp[2], * p, * pp; int val[2]; -int i; -if (c == NULL) return TRUE; /* does this ever occur? */ +if (!c) return TRUE; /* does this ever occur? */ switch (c->type) { case cond_and: - yield = test_condition(c->left.c, FALSE) && - *error_pointer == NULL && - test_condition(c->right.c, FALSE); - break; + yield = test_condition(c->left.c, FALSE) && + *error_pointer == NULL && + test_condition(c->right.c, FALSE); + break; case cond_or: - yield = test_condition(c->left.c, FALSE) || - (*error_pointer == NULL && - test_condition(c->right.c, FALSE)); - break; + yield = test_condition(c->left.c, FALSE) || + (*error_pointer == NULL && + test_condition(c->right.c, FALSE)); + break; - /* The personal test is meaningless in a system filter. The tests are now in - a separate function (so Sieve can use them). However, an Exim filter does not - scan Cc: (hence the FALSE argument). */ + /* The personal test is meaningless in a system filter. The tests are now in + a separate function (so Sieve can use them). However, an Exim filter does not + scan Cc: (hence the FALSE argument). */ case cond_personal: - yield = f.system_filtering? FALSE : filter_personal(c->left.a, FALSE); - break; + yield = f.system_filtering? FALSE : filter_personal(c->left.a, FALSE); + break; case cond_delivered: - yield = filter_delivered; - break; + yield = filter_delivered; + break; - /* Only TRUE if a message is actually being processed; FALSE for address - testing and verification. */ + /* Only TRUE if a message is actually being processed; FALSE for address + testing and verification. */ case cond_errormsg: - yield = message_id[0] != 0 && - (sender_address == NULL || sender_address[0] == 0); - break; + yield = message_id[0] != 0 && + (sender_address == NULL || sender_address[0] == 0); + break; - /* Only FALSE if a message is actually being processed; TRUE for address - and filter testing and verification. */ + /* Only FALSE if a message is actually being processed; TRUE for address + and filter testing and verification. */ case cond_firsttime: - yield = filter_test != FTEST_NONE || message_id[0] == 0 || f.deliver_firsttime; - break; + yield = filter_test != FTEST_NONE || message_id[0] == 0 || f.deliver_firsttime; + break; - /* Only TRUE if a message is actually being processed; FALSE for address - testing and verification. */ + /* Only TRUE if a message is actually being processed; FALSE for address + testing and verification. */ case cond_manualthaw: - yield = message_id[0] != 0 && f.deliver_manual_thaw; - break; + yield = message_id[0] != 0 && f.deliver_manual_thaw; + break; - /* The foranyaddress condition loops through a list of addresses */ + /* The foranyaddress condition loops through a list of addresses */ case cond_foranyaddress: - p = c->left.u; - if (!(pp = expand_cstring(p))) - { - *error_pointer = string_sprintf("failed to expand \"%s\" in " - "filter file: %s", p, expand_string_message); - return FALSE; - } + p = c->left.u; + if (!(pp = expand_cstring(p))) + { + *error_pointer = string_sprintf("failed to expand \"%s\" in " + "filter file: %s", p, expand_string_message); + return FALSE; + } - yield = FALSE; - f.parse_allow_group = TRUE; /* Allow group syntax */ + yield = FALSE; + f.parse_allow_group = TRUE; /* Allow group syntax */ - while (*pp) - { - uschar *error; - int start, end, domain; - uschar * s; + while (*pp) + { + uschar *error; + int start, end, domain; + uschar * s; - p = parse_find_address_end(pp, FALSE); - s = string_copyn(pp, p - pp); + p = parse_find_address_end(pp, FALSE); + s = string_copyn(pp, p - pp); - filter_thisaddress = - parse_extract_address(s, &error, &start, &end, &domain, FALSE); + filter_thisaddress = + parse_extract_address(s, &error, &start, &end, &domain, FALSE); - if (filter_thisaddress) - { - if ((filter_test != FTEST_NONE && debug_selector != 0) || - (debug_selector & D_filter) != 0) - { - indent(); - debug_printf_indent("Extracted address %s\n", filter_thisaddress); - } - yield = test_condition(c->right.c, FALSE); - } + if (filter_thisaddress) + { + if ((filter_test != FTEST_NONE && debug_selector != 0) || + (debug_selector & D_filter) != 0) + { + indent(); + debug_printf_indent("Extracted address %s\n", filter_thisaddress); + } + yield = test_condition(c->right.c, FALSE); + } - if (yield) break; - if (!*p) break; - pp = p + 1; - } + if (yield) break; + if (!*p) break; + pp = p + 1; + } - f.parse_allow_group = FALSE; /* Reset group syntax flags */ - f.parse_found_group = FALSE; - break; + f.parse_allow_group = FALSE; /* Reset group syntax flags */ + f.parse_found_group = FALSE; + break; - /* All other conditions have left and right values that need expanding; - on error, it doesn't matter what value is returned. */ + /* All other conditions have left and right values that need expanding; + on error, it doesn't matter what value is returned. */ - default: - p = c->left.u; - for (i = 0; i < 2; i++) - { - if (!(exp[i] = expand_cstring(p))) + default: + p = c->left.u; + for (int i = 0; i < 2; i++) { - *error_pointer = string_sprintf("failed to expand \"%s\" in " - "filter file: %s", p, expand_string_message); - return FALSE; + if (!(exp[i] = expand_string_2(p, &textonly_re))) + { + *error_pointer = string_sprintf("failed to expand \"%s\" in " + "filter file: %s", p, expand_string_message); + return FALSE; + } + p = c->right.u; } - p = c->right.u; - } - /* Inner switch for the different cases */ - - switch(c->type) - { - case cond_is: - yield = strcmpic(exp[0], exp[1]) == 0; - break; + /* Inner switch for the different cases */ - case cond_IS: - yield = Ustrcmp(exp[0], exp[1]) == 0; - break; - - case cond_contains: - yield = strstric_c(exp[0], exp[1], FALSE) != NULL; - break; + switch(c->type) + { + case cond_is: + yield = strcmpic(exp[0], exp[1]) == 0; + break; - case cond_CONTAINS: - yield = Ustrstr(exp[0], exp[1]) != NULL; - break; + case cond_IS: + yield = Ustrcmp(exp[0], exp[1]) == 0; + break; - case cond_begins: - yield = strncmpic(exp[0], exp[1], Ustrlen(exp[1])) == 0; - break; + case cond_contains: + yield = strstric_c(exp[0], exp[1], FALSE) != NULL; + break; - case cond_BEGINS: - yield = Ustrncmp(exp[0], exp[1], Ustrlen(exp[1])) == 0; - break; + case cond_CONTAINS: + yield = Ustrstr(exp[0], exp[1]) != NULL; + break; - case cond_ends: - case cond_ENDS: - { - int len = Ustrlen(exp[1]); - const uschar *s = exp[0] + Ustrlen(exp[0]) - len; - yield = s < exp[0] - ? FALSE - : (c->type == cond_ends ? strcmpic(s, exp[1]) : Ustrcmp(s, exp[1])) == 0; - } - break; + case cond_begins: + yield = strncmpic(exp[0], exp[1], Ustrlen(exp[1])) == 0; + break; - case cond_matches: - case cond_MATCHES: - { - const pcre2_code *re; - int err; - PCRE2_SIZE offset; + case cond_BEGINS: + yield = Ustrncmp(exp[0], exp[1], Ustrlen(exp[1])) == 0; + break; - if ((filter_test != FTEST_NONE && debug_selector != 0) || - (debug_selector & D_filter) != 0) + case cond_ends: + case cond_ENDS: { - debug_printf_indent("Match expanded arguments:\n"); - debug_printf_indent(" Subject = %s\n", exp[0]); - debug_printf_indent(" Pattern = %s\n", exp[1]); + int len = Ustrlen(exp[1]); + const uschar *s = exp[0] + Ustrlen(exp[0]) - len; + yield = s < exp[0] + ? FALSE + : (c->type == cond_ends ? strcmpic(s, exp[1]) : Ustrcmp(s, exp[1])) == 0; + break; } - if (!(re = pcre2_compile((PCRE2_SPTR)exp[1], PCRE2_ZERO_TERMINATED, - PCRE_COPT | (c->type == cond_matches ? PCRE2_CASELESS : 0), - &err, &offset, pcre_cmp_ctx))) + case cond_matches: + case cond_MATCHES: { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - *error_pointer = string_sprintf("error while compiling " - "regular expression \"%s\": %s at offset %ld", - exp[1], errbuf, (long)offset); - return FALSE; - } + const pcre2_code * re; + mcs_flags flags = textonly_re ? MCS_CACHEABLE : MCS_NOFLAGS; - yield = regex_match_and_setup(re, exp[0], PCRE_EOPT, -1); - break; - } + if ((filter_test != FTEST_NONE && debug_selector != 0) || + (debug_selector & D_filter) != 0) + { + debug_printf_indent("Match expanded arguments:\n"); + debug_printf_indent(" Subject = %s\n", exp[0]); + debug_printf_indent(" Pattern = %s\n", exp[1]); + } - /* For above and below, convert the strings to numbers */ + if (c->type == cond_matches) flags |= MCS_CASELESS; + if (!(re = regex_compile(exp[1], flags, error_pointer, pcre_gen_cmp_ctx))) + return FALSE; - case cond_above: - case cond_below: - for (i = 0; i < 2; i++) - { - val[i] = get_number(exp[i], &yield); - if (!yield) - { - *error_pointer = string_sprintf("malformed numerical string \"%s\"", - exp[i]); - return FALSE; - } + yield = regex_match_and_setup(re, exp[0], PCRE_EOPT, -1); + break; + } + + /* For above and below, convert the strings to numbers */ + + case cond_above: + case cond_below: + for (int i = 0; i < 2; i++) + { + val[i] = get_number(exp[i], &yield); + if (!yield) + { + *error_pointer = string_sprintf("malformed numerical string \"%s\"", + exp[i]); + return FALSE; + } + } + yield = c->type == cond_above ? (val[0] > val[1]) : (val[0] < val[1]); + break; } - yield = (c->type == cond_above)? (val[0] > val[1]) : (val[0] < val[1]); break; - } - break; } if ((filter_test != FTEST_NONE && debug_selector != 0) || @@ -2356,7 +2346,7 @@ while (commands) commands = commands->next; } -return filter_delivered? FF_DELIVERED : FF_NOTDELIVERED; +return filter_delivered ? FF_DELIVERED : FF_NOTDELIVERED; } diff --git a/src/src/functions.h b/src/src/functions.h index 224666cb1..4caae346d 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -182,6 +182,10 @@ extern BOOL cutthrough_predata(void); extern void release_cutthrough_connection(const uschar *); extern void daemon_go(void); +#ifndef COMPILE_UTILITY +extern ssize_t daemon_client_sockname(struct sockaddr_un *, uschar **); +extern ssize_t daemon_notifier_sockname(struct sockaddr_un *); +#endif #ifdef EXPERIMENTAL_DCC extern int dcc_process(uschar **); @@ -260,6 +264,7 @@ extern int exp_bool(address_item *addr, extern BOOL expand_check_condition(uschar *, uschar *, uschar *); extern uschar *expand_file_big_buffer(const uschar *); extern uschar *expand_string(uschar *); /* public, cannot make const */ +extern const uschar *expand_string_2(const uschar *, BOOL *); extern const uschar *expand_cstring(const uschar *); /* ... so use this one */ extern uschar *expand_getkeyed(const uschar *, const uschar *); @@ -332,7 +337,7 @@ extern BOOL macro_read_assignment(uschar *); extern uschar *macros_expand(int, int *, BOOL *); extern void mainlog_close(void); #ifdef WITH_CONTENT_SCAN -extern int malware(const uschar *, int); +extern int malware(const uschar *, BOOL, int); extern int malware_in_file(uschar *); extern void malware_init(void); extern gstring * malware_show_supported(gstring *); @@ -345,7 +350,7 @@ extern int match_check_list(const uschar **, int, tree_node **, unsigned int const uschar *, const uschar **); extern int match_isinlist(const uschar *, const uschar **, int, tree_node **, unsigned int *, int, BOOL, const uschar **); -extern int match_check_string(const uschar *, const uschar *, int, BOOL, BOOL, BOOL, +extern int match_check_string(const uschar *, const uschar *, int, mcs_flags, const uschar **); extern void message_start(void); @@ -360,7 +365,7 @@ extern int mime_acl_check(uschar *acl, FILE *f, struct mime_boundary_context *, uschar **, uschar **); extern int mime_decode(const uschar **); extern ssize_t mime_decode_base64(FILE *, FILE *, uschar *); -extern int mime_regex(const uschar **); +extern int mime_regex(const uschar **, BOOL); extern void mime_set_anomaly(int); #endif extern uschar *moan_check_errorcopy(uschar *); @@ -433,11 +438,14 @@ extern BOOL receive_msg(BOOL); extern int_eximarith_t receive_statvfs(BOOL, int *); extern void receive_swallow_smtp(void); #ifdef WITH_CONTENT_SCAN -extern int regex(const uschar **); +extern int regex(const uschar **, BOOL); #endif +extern void regex_at_daemon(const uschar *); extern BOOL regex_match(const pcre2_code *, const uschar *, int, uschar **); extern BOOL regex_match_and_setup(const pcre2_code *, const uschar *, int, int); -extern const pcre2_code *regex_must_compile(const uschar *, BOOL, BOOL); +extern const pcre2_code *regex_compile(const uschar *, mcs_flags, uschar **, + pcre2_compile_context *); +extern const pcre2_code *regex_must_compile(const uschar *, mcs_flags, BOOL); extern void retry_add_item(address_item *, uschar *, int); extern BOOL retry_check_address(const uschar *, host_item *, uschar *, BOOL, uschar **, uschar **); @@ -1221,6 +1229,7 @@ pid_t pid; DEBUG(D_any) debug_printf("%s forking for %s\n", process_purpose, purpose); if ((pid = fork()) == 0) { + f.daemon_listen = FALSE; process_purpose = purpose; DEBUG(D_any) debug_printf("postfork: %s\n", purpose); } diff --git a/src/src/globals.c b/src/src/globals.c index ff246feb4..c95d24b47 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -234,6 +234,7 @@ struct global_flags f = .continue_more = FALSE, .daemon_listen = FALSE, + .daemon_scion = FALSE, .debug_daemon = FALSE, .deliver_firsttime = FALSE, .deliver_force = FALSE, @@ -391,7 +392,7 @@ BOOL proxy_session = FALSE; #endif #ifndef DISABLE_QUEUE_RAMP -BOOL queue_fast_ramp = FALSE; +BOOL queue_fast_ramp = TRUE; #endif BOOL queue_list_requires_admin = TRUE; BOOL queue_only = FALSE; @@ -1207,9 +1208,12 @@ uid_t originator_uid; uschar *override_local_interfaces = NULL; uschar *override_pid_file_path = NULL; +BOOL panic_coredump = FALSE; pcre2_general_context * pcre_gen_ctx = NULL; -pcre2_compile_context * pcre_cmp_ctx = NULL; -pcre2_match_context * pcre_mtc_ctx = NULL; +pcre2_compile_context * pcre_gen_cmp_ctx = NULL; +pcre2_match_context * pcre_gen_mtc_ctx = NULL; +pcre2_general_context * pcre_mlc_ctx = NULL; +pcre2_compile_context * pcre_mlc_cmp_ctx = NULL; uschar *percent_hack_domains = NULL; uschar *pid_file_path = US PID_FILE_PATH @@ -1313,6 +1317,7 @@ const pcre2_code *regex_SIZE = NULL; #ifndef DISABLE_PIPE_CONNECT const pcre2_code *regex_EARLY_PIPE = NULL; #endif +int regex_cachesize = 0; const pcre2_code *regex_ismsgid = NULL; const pcre2_code *regex_smtp_code = NULL; const uschar *regex_vars[REGEX_VARS]; @@ -1323,7 +1328,7 @@ const pcre2_code *regex_whitelisted_macro = NULL; uschar *regex_match_string = NULL; #endif int remote_delivery_count = 0; -int remote_max_parallel = 2; +int remote_max_parallel = 4; uschar *remote_sort_domains = NULL; int retry_data_expire = 7*24*60*60; int retry_interval_max = 24*60*60; diff --git a/src/src/globals.h b/src/src/globals.h index fe099e402..c9ef5e484 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -199,6 +199,7 @@ extern struct global_flags { BOOL continue_more :1; /* Flag more addresses waiting */ BOOL daemon_listen :1; /* True if listening required */ + BOOL daemon_scion :1; /* Ancestor proc is daemon, and not re-exec'd */ BOOL debug_daemon :1; /* Debug the daemon process only */ BOOL deliver_firsttime :1; /* True for first delivery attempt */ BOOL deliver_force :1; /* TRUE if delivery was forced */ @@ -792,9 +793,12 @@ extern uid_t originator_uid; /* Uid of ditto */ extern uschar *override_local_interfaces; /* Value of -oX argument */ extern uschar *override_pid_file_path; /* Value of -oP argument */ +extern BOOL panic_coredump; /* SEGV rather than exit, on LOG_PANIC_DIE */ extern pcre2_general_context * pcre_gen_ctx; /* pcre memory management */ -extern pcre2_compile_context * pcre_cmp_ctx; -extern pcre2_match_context * pcre_mtc_ctx; +extern pcre2_compile_context * pcre_gen_cmp_ctx; +extern pcre2_match_context * pcre_gen_mtc_ctx; +extern pcre2_general_context * pcre_mlc_ctx; +extern pcre2_compile_context * pcre_mlc_cmp_ctx; extern uschar *percent_hack_domains; /* Local domains for which '% operates */ extern uschar *pid_file_path; /* For writing daemon pids */ @@ -895,6 +899,7 @@ extern const pcre2_code *regex_SIZE; /* For recognizing SIZE settings */ #ifndef DISABLE_PIPE_CONNECT extern const pcre2_code *regex_EARLY_PIPE; /* For recognizing PIPE_CONNCT */ #endif +extern int regex_cachesize; /* number of entries */ extern const pcre2_code *regex_ismsgid; /* Compiled r.e. for message ID */ extern const pcre2_code *regex_smtp_code; /* For recognizing SMTP codes */ extern const uschar *regex_vars[]; /* $regexN variables */ diff --git a/src/src/header.c b/src/src/header.c index 898d8d5c4..7ef59ff53 100644 --- a/src/src/header.c +++ b/src/src/header.c @@ -368,7 +368,7 @@ Returns: cond if the header exists and contains one of the strings; /* First we have a local subroutine to handle a single pattern */ static BOOL -one_pattern_match(uschar *name, int slen, BOOL has_addresses, uschar *pattern) +one_pattern_match(uschar * name, int slen, BOOL has_addresses, uschar * pattern) { BOOL yield = FALSE; const pcre2_code *re = NULL; @@ -376,7 +376,7 @@ const pcre2_code *re = NULL; /* If the pattern is a regex, compile it. Bomb out if compiling fails; these patterns are all constructed internally and should be valid. */ -if (*pattern == '^') re = regex_must_compile(pattern, TRUE, FALSE); +if (*pattern == '^') re = regex_must_compile(pattern, MCS_CASELESS, FALSE); /* Scan for the required header(s) and scan each one */ @@ -443,7 +443,7 @@ return yield; /* The externally visible interface */ BOOL -header_match(uschar *name, BOOL has_addresses, BOOL cond, string_item *strings, +header_match(uschar * name, BOOL has_addresses, BOOL cond, string_item * strings, int count, ...) { va_list ap; diff --git a/src/src/host.c b/src/src/host.c index e43b507e5..fed9f4b5f 100644 --- a/src/src/host.c +++ b/src/src/host.c @@ -363,80 +363,6 @@ while ((name = string_nextinlist(&list, &sep, NULL, 0))) } - - - -/************************************************* -* Extract port from address string * -*************************************************/ - -/* In the -oMa and -oMi options, a host plus port is given as an IP address -followed by a dot and a port number. This function decodes this. - -An alternative format for the -oMa and -oMi options is [ip address]:port which -is what Exim uses for output, because it seems to becoming commonly used, -whereas the dot form confuses some programs/people. So we recognize that form -too. - -The spool file used to use the first form, but this breaks with a v4mapped ipv6 -hybrid, because the parsing here is not clever. So for spool we now use the -second form. - -Argument: - address points to the string; if there is a port, the '.' in the string - is overwritten with zero to terminate the address; if the string - is in the [xxx]:ppp format, the address is shifted left and the - brackets are removed - -Returns: 0 if there is no port, else the port number. If there's a syntax - error, leave the incoming address alone, and return 0. -*/ - -int -host_address_extract_port(uschar *address) -{ -int port = 0; -uschar *endptr; - -/* Handle the "bracketed with colon on the end" format */ - -if (*address == '[') - { - uschar *rb = address + 1; - while (*rb != 0 && *rb != ']') rb++; - if (*rb++ == 0) return 0; /* Missing ]; leave invalid address */ - if (*rb == ':') - { - port = Ustrtol(rb + 1, &endptr, 10); - if (*endptr != 0) return 0; /* Invalid port; leave invalid address */ - } - else if (*rb != 0) return 0; /* Bad syntax; leave invalid address */ - memmove(address, address + 1, rb - address - 2); - rb[-2] = 0; - } - -/* Handle the "dot on the end" format */ - -else - { - int skip = -3; /* Skip 3 dots in IPv4 addresses */ - address--; - while (*(++address) != 0) - { - int ch = *address; - if (ch == ':') skip = 0; /* Skip 0 dots in IPv6 addresses */ - else if (ch == '.' && skip++ >= 0) break; - } - if (*address == 0) return 0; - port = Ustrtol(address + 1, &endptr, 10); - if (*endptr != 0) return 0; /* Invalid port; leave invalid address */ - *address = 0; - } - -return port; -} - - /************************************************* * Get port from a host item's name * *************************************************/ diff --git a/src/src/host_address.c b/src/src/host_address.c new file mode 100644 index 000000000..9e6f958be --- /dev/null +++ b/src/src/host_address.c @@ -0,0 +1,80 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* Copyright (c) The Exim Maintainers 2020 - 2022 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ +/* See the file NOTICE for conditions of use and distribution. */ + +#include "exim.h" + +/************************************************* +* Extract port from address string * +*************************************************/ + +/* In the spool file, and in the -oMa and -oMi options, a host plus port is +given as an IP address followed by a dot and a port number. This function +decodes this. + +An alternative format for the -oMa and -oMi options is [ip address]:port which +is what Exim 4 uses for output, because it seems to becoming commonly used, +whereas the dot form confuses some programs/people. So we recognize that form +too. + +Argument: + address points to the string; if there is a port, the '.' in the string + is overwritten with zero to terminate the address; if the string + is in the [xxx]:ppp format, the address is shifted left and the + brackets are removed + +Returns: 0 if there is no port, else the port number. If there's a syntax + error, leave the incoming address alone, and return 0. +*/ + +int +host_address_extract_port(uschar * address) +{ +int port = 0; +uschar *endptr; + +/* Handle the "bracketed with colon on the end" format */ + +if (*address == '[') + { + uschar *rb = address + 1; + while (*rb != 0 && *rb != ']') rb++; + if (*rb++ == 0) return 0; /* Missing ]; leave invalid address */ + if (*rb == ':') + { + port = Ustrtol(rb + 1, &endptr, 10); + if (*endptr != 0) return 0; /* Invalid port; leave invalid address */ + } + else if (*rb != 0) return 0; /* Bad syntax; leave invalid address */ + memmove(address, address + 1, rb - address - 2); + rb[-2] = 0; + } + +/* Handle the "dot on the end" format */ + +else + { + int skip = -3; /* Skip 3 dots in IPv4 addresses */ + address--; + while (*(++address) != 0) + { + int ch = *address; + if (ch == ':') skip = 0; /* Skip 0 dots in IPv6 addresses */ + else if (ch == '.' && skip++ >= 0) break; + } + if (*address == 0) return 0; + port = Ustrtol(address + 1, &endptr, 10); + if (*endptr != 0) return 0; /* Invalid port; leave invalid address */ + *address = 0; + } + +return port; +} + +/* vi: aw ai sw=2 +*/ +/* End of host.c */ diff --git a/src/src/log.c b/src/src/log.c index 8ca973f2d..a46d523db 100644 --- a/src/src/log.c +++ b/src/src/log.c @@ -1278,7 +1278,10 @@ if (flags & LOG_PANIC) /* Give up if the DIE flag is set */ if ((flags & LOG_PANIC_DIE) != LOG_PANIC) - die(NULL, US"Unexpected failure, please try later"); + if (panic_coredump) + kill(getpid(), SIGSEGV); /* deliberate trap */ + else + die(NULL, US"Unexpected failure, please try later"); } } diff --git a/src/src/macros.h b/src/src/macros.h index fa89de12d..adbe6a267 100644 --- a/src/src/macros.h +++ b/src/src/macros.h @@ -1111,7 +1111,17 @@ should not be one active. */ #define NOTIFIER_SOCKET_NAME "exim_daemon_notify" -#define NOTIFY_MSG_QRUN 1 /* Notify message types */ +/* Notify message types */ +#define NOTIFY_MSG_QRUN 1 #define NOTIFY_QUEUE_SIZE_REQ 2 +#define NOTIFY_REGEX 3 + +/* Flags for match_check_string() */ +typedef unsigned mcs_flags; +#define MCS_NOFLAGS 0 +#define MCS_PARTIAL BIT(0) /* permit partial- search types */ +#define MCS_CASELESS BIT(1) /* caseless matching where possible */ +#define MCS_AT_SPECIAL BIT(2) /* recognize @, @[], etc. */ +#define MCS_CACHEABLE BIT(3) /* no dynamic expansions used for pattern */ /* End of macros.h */ diff --git a/src/src/malware.c b/src/src/malware.c index 4719a5d61..8b5ec27c4 100644 --- a/src/src/malware.c +++ b/src/src/malware.c @@ -299,39 +299,29 @@ return sock; } static const pcre2_code * -m_pcre_compile(const uschar * re, uschar ** errstr) +m_pcre_compile(const uschar * re, BOOL cacheable, uschar ** errstr) { -int err; -PCRE2_SIZE roffset; -const pcre2_code * cre; - -if (!(cre = pcre2_compile((PCRE2_SPTR)re, PCRE2_ZERO_TERMINATED, - PCRE_COPT, &err, &roffset, pcre_cmp_ctx))) - { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - *errstr= string_sprintf("regular expression error in '%s': %s at offset %ld", - re, errbuf, (long)roffset); - } -return cre; +return regex_compile(re, cacheable ? MCS_CACHEABLE : MCS_NOFLAGS, errstr, + pcre_gen_cmp_ctx); } uschar * m_pcre_exec(const pcre2_code * cre, uschar * text) { pcre2_match_data * md = pcre2_match_data_create(2, pcre_gen_ctx); -int i = pcre2_match(cre, text, PCRE2_ZERO_TERMINATED, 0, 0, md, pcre_mtc_ctx); +int i = pcre2_match(cre, text, PCRE2_ZERO_TERMINATED, 0, 0, md, pcre_gen_mtc_ctx); PCRE2_UCHAR * substr = NULL; PCRE2_SIZE slen; if (i >= 2) /* Got it */ - pcre2_substring_get_bynumber(md, 1, &substr, &slen); + pcre2_substring_get_bynumber(md, 1, &substr, &slen); /* uses same ctx as md */ +/* pcre2_match_data_free(md); gen ctx needs no free */ return US substr; } static const pcre2_code * m_pcre_nextinlist(const uschar ** list, int * sep, - char * listerr, uschar ** errstr) + BOOL cacheable, char * listerr, uschar ** errstr) { const uschar * list_ele; const pcre2_code * cre = NULL; @@ -342,7 +332,7 @@ else { DEBUG(D_acl) debug_printf_indent("%15s%10s'%s'\n", "", "RE: ", string_printing(list_ele)); - cre = m_pcre_compile(CUS list_ele, errstr); + cre = m_pcre_compile(CUS list_ele, cacheable, errstr); } return cre; } @@ -568,6 +558,7 @@ is via malware(), or there's malware_in_file() used for testing/debugging. Arguments: malware_re match condition for "malware=" + cacheable the RE did not use any dynamic elements during expansion scan_filename the file holding the email to be scanned, if we're faking this up for the -bmalware test, else NULL timeout if nonzero, non-default timeoutl @@ -576,11 +567,12 @@ Returns: Exim message processing code (OK, FAIL, DEFER, ...) where true means malware was found (condition applies) */ static int -malware_internal(const uschar * malware_re, const uschar * scan_filename, - int timeout) +malware_internal(const uschar * malware_re, BOOL cacheable, + const uschar * scan_filename, int timeout) { int sep = 0; const uschar *av_scanner_work = av_scanner; +BOOL av_scanner_textonly; uschar *scanner_name; unsigned long mbox_size; FILE *mbox_file; @@ -607,30 +599,30 @@ the name), so we can close it right away. Get the directory too. */ eml_dir = string_copyn(eml_filename, Ustrrchr(eml_filename, '/') - eml_filename); /* parse 1st option */ -if (strcmpic(malware_re, US"false") == 0 || Ustrcmp(malware_re,"0") == 0) +if (strcmpic(malware_re, US"false") == 0 || Ustrcmp(malware_re, "0") == 0) return FAIL; /* explicitly no matching */ /* special cases (match anything except empty) */ -if ( strcmpic(malware_re,US"true") == 0 - || Ustrcmp(malware_re,"*") == 0 - || Ustrcmp(malware_re,"1") == 0 +if ( strcmpic(malware_re, US"true") == 0 + || Ustrcmp(malware_re, "*") == 0 + || Ustrcmp(malware_re, "1") == 0 ) { if ( !malware_default_re - && !(malware_default_re = m_pcre_compile(malware_regex_default, &errstr))) + && !(malware_default_re = m_pcre_compile(malware_regex_default, FALSE, &errstr))) return malware_panic_defer(errstr); malware_re = malware_regex_default; re = malware_default_re; } /* compile the regex, see if it works */ -else if (!(re = m_pcre_compile(malware_re, &errstr))) +else if (!(re = m_pcre_compile(malware_re, cacheable, &errstr))) return malware_panic_defer(errstr); /* if av_scanner starts with a dollar, expand it first */ if (*av_scanner == '$') { - if (!(av_scanner_work = expand_string(av_scanner))) + if (!(av_scanner_work = expand_string_2(av_scanner, &av_scanner_textonly))) return malware_panic_defer( string_sprintf("av_scanner starts with $, but expansion failed: %s", expand_string_message)); @@ -641,6 +633,8 @@ if (*av_scanner == '$') malware_name = NULL; malware_ok = FALSE; } +else + av_scanner_textonly = TRUE; /* Do not scan twice (unless av_scanner is dynamic). */ if (!malware_ok) @@ -745,13 +739,11 @@ if (!malware_ok) case M_FPROT6D: /* "f-prot6d" scanner type ----------------------------------- */ { int bread; - uschar * e; - uschar * linebuffer; - uschar * scanrequest; + uschar * e, * linebuffer, * scanrequest; uschar av_buffer[1024]; - if ((!fprot6d_re_virus && !(fprot6d_re_virus = m_pcre_compile(fprot6d_re_virus_str, &errstr))) - || (!fprot6d_re_error && !(fprot6d_re_error = m_pcre_compile(fprot6d_re_error_str, &errstr)))) + if ((!fprot6d_re_virus && !(fprot6d_re_virus = m_pcre_compile(fprot6d_re_virus_str, FALSE, &errstr))) + || (!fprot6d_re_error && !(fprot6d_re_error = m_pcre_compile(fprot6d_re_error_str, FALSE, &errstr)))) return malware_panic_defer(errstr); scanrequest = string_sprintf("SCAN FILE %s\n", eml_filename); @@ -921,7 +913,7 @@ badseek: err = errno; /* set up match regex */ if (!drweb_re) - drweb_re = m_pcre_compile(drweb_re_str, &errstr); + drweb_re = m_pcre_compile(drweb_re_str, FALSE, &errstr); /* read and concatenate virus names into one string */ for (int i = 0; i < drweb_vnum; i++) @@ -945,7 +937,7 @@ badseek: err = errno; /* try matcher on the line, grab substring */ result = pcre2_match(drweb_re, (PCRE2_SPTR)tmpbuf, PCRE2_ZERO_TERMINATED, - 0, 0, md, pcre_mtc_ctx); + 0, 0, md, pcre_gen_mtc_ctx); if (result >= 2) { PCRE2_SIZE * ovec = pcre2_get_ovector_pointer(md); @@ -959,6 +951,7 @@ badseek: err = errno; g = string_catn(g, US ovec[2], ovec[3] - ovec[2]); } } + /* pcre2_match_data_free(md); gen ctx needs no free */ } malware_name = string_from_gstring(g); } @@ -1099,7 +1092,7 @@ badseek: err = errno; /* set up match */ /* todo also SUSPICION\t */ if (!fsec_re) - fsec_re = m_pcre_compile(fsec_re_str, &errstr); + fsec_re = m_pcre_compile(fsec_re_str, FALSE, &errstr); /* read report, linewise. Apply a timeout as the Fsecure daemon sometimes wants an answer to "PING" but they won't tell us what */ @@ -1223,12 +1216,12 @@ badseek: err = errno; /* set up match regex, depends on retcode */ if (kav_rc == 3) { - if (!kav_re_sus) kav_re_sus = m_pcre_compile(kav_re_sus_str, &errstr); + if (!kav_re_sus) kav_re_sus = m_pcre_compile(kav_re_sus_str, FALSE, &errstr); kav_re = kav_re_sus; } else { - if (!kav_re_inf) kav_re_inf = m_pcre_compile(kav_re_inf_str, &errstr); + if (!kav_re_inf) kav_re_inf = m_pcre_compile(kav_re_inf_str, FALSE, &errstr); kav_re = kav_re_inf; } @@ -1277,13 +1270,13 @@ badseek: err = errno; return m_panic_defer(scanent, NULL, errstr); /* find scanner output trigger */ - cmdline_trigger_re = m_pcre_nextinlist(&av_scanner_work, &sep, + cmdline_trigger_re = m_pcre_nextinlist(&av_scanner_work, &sep, av_scanner_textonly, "missing trigger specification", &errstr); if (!cmdline_trigger_re) return m_panic_defer(scanent, NULL, errstr); /* find scanner name regex */ - cmdline_regex_re = m_pcre_nextinlist(&av_scanner_work, &sep, + cmdline_regex_re = m_pcre_nextinlist(&av_scanner_work, &sep, av_scanner_textonly, "missing virus name regex specification", &errstr); if (!cmdline_regex_re) return m_panic_defer(scanent, NULL, errstr); @@ -1906,13 +1899,13 @@ badseek: err = errno; string_printing(sockline_scanner)); /* find scanner output trigger */ - sockline_trig_re = m_pcre_nextinlist(&av_scanner_work, &sep, + sockline_trig_re = m_pcre_nextinlist(&av_scanner_work, &sep, av_scanner_textonly, "missing trigger specification", &errstr); if (!sockline_trig_re) return m_panic_defer_3(scanent, NULL, errstr, malware_daemon_ctx.sock); /* find virus name regex */ - sockline_name_re = m_pcre_nextinlist(&av_scanner_work, &sep, + sockline_name_re = m_pcre_nextinlist(&av_scanner_work, &sep, av_scanner_textonly, "missing virus name regex specification", &errstr); if (!sockline_name_re) return m_panic_defer_3(scanent, NULL, errstr, malware_daemon_ctx.sock); @@ -2043,11 +2036,11 @@ badseek: err = errno; */ if ( ( !ava_re_clean - && !(ava_re_clean = m_pcre_compile(ava_re_clean_str, &errstr))) + && !(ava_re_clean = m_pcre_compile(ava_re_clean_str, FALSE, &errstr))) || ( !ava_re_virus - && !(ava_re_virus = m_pcre_compile(ava_re_virus_str, &errstr))) + && !(ava_re_virus = m_pcre_compile(ava_re_virus_str, FALSE, &errstr))) || ( !ava_re_error - && !(ava_re_error = m_pcre_compile(ava_re_error_str, &errstr))) + && !(ava_re_error = m_pcre_compile(ava_re_error_str, FALSE, &errstr))) ) return malware_panic_defer(errstr); @@ -2209,15 +2202,16 @@ filename; it's a wrapper around the malware_file function. Arguments: malware_re match condition for "malware=" + cacheable the RE did not use any dynamic elements during expansion timeout if nonzero, timeout in seconds Returns: Exim message processing code (OK, FAIL, DEFER, ...) where true means malware was found (condition applies) */ int -malware(const uschar * malware_re, int timeout) +malware(const uschar * malware_re, BOOL cacheable, int timeout) { -int ret = malware_internal(malware_re, NULL, timeout); +int ret = malware_internal(malware_re, cacheable, NULL, timeout); if (ret == DEFER) av_failed = TRUE; return ret; @@ -2257,7 +2251,7 @@ recipients_list = NULL; receive_add_recipient(US"malware-victim@example.net", -1); f.enable_dollar_recipients = TRUE; -ret = malware_internal(US"*", eml_filename, 0); +ret = malware_internal(US"*", TRUE, eml_filename, 0); Ustrncpy(spooled_message_id, message_id, sizeof(spooled_message_id)); spool_mbox_ok = 1; @@ -2278,35 +2272,35 @@ void malware_init(void) { if (!malware_default_re) - malware_default_re = regex_must_compile(malware_regex_default, FALSE, TRUE); + malware_default_re = regex_must_compile(malware_regex_default, MCS_NOFLAGS, TRUE); #ifndef DISABLE_MAL_DRWEB if (!drweb_re) - drweb_re = regex_must_compile(drweb_re_str, FALSE, TRUE); + drweb_re = regex_must_compile(drweb_re_str, MCS_NOFLAGS, TRUE); #endif #ifndef DISABLE_MAL_FSECURE if (!fsec_re) - fsec_re = regex_must_compile(fsec_re_str, FALSE, TRUE); + fsec_re = regex_must_compile(fsec_re_str, MCS_NOFLAGS, TRUE); #endif #ifndef DISABLE_MAL_KAV if (!kav_re_sus) - kav_re_sus = regex_must_compile(kav_re_sus_str, FALSE, TRUE); + kav_re_sus = regex_must_compile(kav_re_sus_str, MCS_NOFLAGS, TRUE); if (!kav_re_inf) - kav_re_inf = regex_must_compile(kav_re_inf_str, FALSE, TRUE); + kav_re_inf = regex_must_compile(kav_re_inf_str, MCS_NOFLAGS, TRUE); #endif #ifndef DISABLE_MAL_AVAST if (!ava_re_clean) - ava_re_clean = regex_must_compile(ava_re_clean_str, FALSE, TRUE); + ava_re_clean = regex_must_compile(ava_re_clean_str, MCS_NOFLAGS, TRUE); if (!ava_re_virus) - ava_re_virus = regex_must_compile(ava_re_virus_str, FALSE, TRUE); + ava_re_virus = regex_must_compile(ava_re_virus_str, MCS_NOFLAGS, TRUE); if (!ava_re_error) - ava_re_error = regex_must_compile(ava_re_error_str, FALSE, TRUE); + ava_re_error = regex_must_compile(ava_re_error_str, MCS_NOFLAGS, TRUE); #endif #ifndef DISABLE_MAL_FFROT6D if (!fprot6d_re_error) - fprot6d_re_error = regex_must_compile(fprot6d_re_error_str, FALSE, TRUE); + fprot6d_re_error = regex_must_compile(fprot6d_re_error_str, MCS_NOFLAGS, TRUE); if (!fprot6d_re_virus) - fprot6d_re_virus = regex_must_compile(fprot6d_re_virus_str, FALSE, TRUE); + fprot6d_re_virus = regex_must_compile(fprot6d_re_virus_str, MCS_NOFLAGS, TRUE); #endif } diff --git a/src/src/match.c b/src/src/match.c index 2e4bff078..b4a0352ee 100644 --- a/src/src/match.c +++ b/src/src/match.c @@ -19,9 +19,7 @@ typedef struct check_string_block { const uschar *origsubject; /* caseful; keep these two first, in */ const uschar *subject; /* step with the block below */ int expand_setup; - BOOL use_partial; - BOOL caseless; - BOOL at_is_special; + mcs_flags flags; /* MCS_* defs in macros.h */ } check_string_block; @@ -32,7 +30,7 @@ typedef struct check_address_block { const uschar *origaddress; /* caseful; keep these two first, in */ uschar *address; /* step with the block above */ int expand_setup; - BOOL caseless; + mcs_flags flags; /* MCS_CASELESS, MCS_TEXTONLY_RE */ } check_address_block; @@ -93,9 +91,10 @@ Returns: OK if matched */ static int -check_string(void *arg, const uschar *pattern, const uschar **valueptr, uschar **error) +check_string(void * arg, const uschar * pattern, const uschar ** valueptr, + uschar ** error) { -const check_string_block *cb = arg; +const check_string_block * cb = arg; int search_type, partial, affixlen, starflags; int expand_setup = cb->expand_setup; const uschar * affix, * opts; @@ -128,7 +127,8 @@ required. */ if (pattern[0] == '^') { - const pcre2_code * re = regex_must_compile(pattern, cb->caseless, FALSE); + const pcre2_code * re = regex_must_compile(pattern, + cb->flags & (MCS_CACHEABLE | MCS_CASELESS), FALSE); if (expand_setup < 0 ? !regex_match(re, s, -1, NULL) : !regex_match_and_setup(re, s, 0, expand_setup) @@ -147,7 +147,7 @@ if (pattern[0] == '*') patlen = Ustrlen(++pattern); if (patlen > slen) return FAIL; - if (cb->caseless + if (cb->flags & MCS_CASELESS ? strncmpic(s + slen - patlen, pattern, patlen) != 0 : Ustrncmp(s + slen - patlen, pattern, patlen) != 0) return FAIL; @@ -166,7 +166,7 @@ the primary host name - implement this by changing the pattern. For the other cases we have to do some more work. If we don't recognize a special pattern, just fall through - the match will fail. */ -if (cb->at_is_special && pattern[0] == '@') +if (cb->flags & MCS_AT_SPECIAL && pattern[0] == '@') { if (pattern[1] == 0) { @@ -260,10 +260,10 @@ NOT_AT_SPECIAL: if ((semicolon = Ustrchr(pattern, ';')) == NULL) { - if (cb->caseless ? strcmpic(s, pattern) != 0 : Ustrcmp(s, pattern) != 0) + if (cb->flags & MCS_CASELESS ? strcmpic(s, pattern) != 0 : Ustrcmp(s, pattern) != 0) return FAIL; - if (expand_setup >= 0) expand_nmax = expand_setup; /* Original code! $0 gets the matched subject */ - if (valueptr) *valueptr = pattern; /* "value" gets the pattern */ + if (expand_setup >= 0) expand_nmax = expand_setup; /* $0 gets the matched subject */ + if (valueptr) *valueptr = pattern; /* "value" gets the pattern */ return OK; } @@ -280,7 +280,7 @@ if (search_type < 0) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", /* Partial matching is not appropriate for certain lookups (e.g. when looking up user@domain for sender rejection). There's a flag to disable it. */ -if (!cb->use_partial) partial = -1; +if (!(cb->flags & MCS_PARTIAL)) partial = -1; /* Set the parameters for the three different kinds of lookup. */ @@ -316,9 +316,10 @@ Arguments: s the subject string to be checked pattern the pattern to check it against expand_setup expansion setup option (see check_string()) - use_partial if FALSE, override any partial- search types - caseless TRUE for caseless matching where possible - at_is_special TRUE to recognize @, @[], etc. + flags + use_partial if FALSE, override any partial- search types + caseless TRUE for caseless matching where possible + at_is_special TRUE to recognize @, @[], etc. valueptr if not NULL, and a file lookup was done, return the result here instead of discarding it; else set it to point to NULL @@ -328,16 +329,14 @@ Returns: OK if matched */ int -match_check_string(const uschar *s, const uschar *pattern, int expand_setup, - BOOL use_partial, BOOL caseless, BOOL at_is_special, const uschar **valueptr) +match_check_string(const uschar * s, const uschar * pattern, int expand_setup, + mcs_flags flags, const uschar ** valueptr) { check_string_block cb; cb.origsubject = s; -cb.subject = caseless ? string_copylc(s) : string_copy(s); +cb.subject = flags & MCS_CASELESS ? string_copylc(s) : string_copy(s); cb.expand_setup = expand_setup; -cb.use_partial = use_partial; -cb.caseless = caseless; -cb.at_is_special = at_is_special; +cb.flags = flags; return check_string(&cb, pattern, valueptr, NULL); } @@ -364,14 +363,9 @@ switch(type) { case MCL_STRING: case MCL_DOMAIN: - case MCL_LOCALPART: - return ((check_string_block *)arg)->subject; - - case MCL_HOST: - return ((check_host_block *)arg)->host_address; - - case MCL_ADDRESS: - return ((check_address_block *)arg)->address; + case MCL_LOCALPART: return ((check_string_block *)arg)->subject; + case MCL_HOST: return ((check_host_block *)arg)->host_address; + case MCL_ADDRESS: return ((check_address_block *)arg)->address; } return US""; /* In practice, should never happen */ } @@ -438,6 +432,7 @@ BOOL include_unknown = FALSE, ignore_unknown = FALSE, const uschar *list; uschar *sss; uschar *ot = NULL; +BOOL textonly_re; /* Save time by not scanning for the option name when we don't need it. */ @@ -465,6 +460,7 @@ if (type >= MCL_NOEXPAND) { list = *listptr; type -= MCL_NOEXPAND; /* Remove the "no expand" flag */ + textonly_re = TRUE; } else { @@ -475,11 +471,11 @@ else { check_string_block *cb = (check_string_block *)arg; deliver_domain = string_copy(cb->subject); - list = expand_cstring(*listptr); + list = expand_string_2(*listptr, &textonly_re); deliver_domain = NULL; } else - list = expand_cstring(*listptr); + list = expand_string_2(*listptr, &textonly_re); if (!list) { @@ -495,6 +491,15 @@ else } } +if (textonly_re) switch (type) + { + case MCL_STRING: + case MCL_DOMAIN: + case MCL_LOCALPART: ((check_string_block *)arg)->flags |= MCS_CACHEABLE; break; + case MCL_HOST: ((check_host_block *)arg)->flags |= MCS_CACHEABLE; break; + case MCL_ADDRESS: ((check_address_block *)arg)->flags |= MCS_CACHEABLE; break; + } + /* For an unnamed list, use the expanded version in comments */ #define LIST_LIMIT_PR 2048 @@ -530,7 +535,7 @@ while ((sss = string_nextinlist(&list, &sep, NULL, 0))) if (at) Ustrncpy(cb->address, cb->origaddress, at - cb->origaddress); - cb->caseless = FALSE; + cb->flags &= ~MCS_CASELESS; continue; } } @@ -543,7 +548,7 @@ while ((sss = string_nextinlist(&list, &sep, NULL, 0))) { check_string_block *cb = (check_string_block *)arg; Ustrcpy(US cb->subject, cb->origsubject); - cb->caseless = FALSE; + cb->flags &= ~MCS_CASELESS; continue; } } @@ -958,15 +963,13 @@ unsigned int *local_cache_bits = cache_bits; check_string_block cb; cb.origsubject = s; cb.subject = caseless ? string_copylc(s) : string_copy(s); -cb.at_is_special = FALSE; +cb.flags = caseless ? MCS_PARTIAL+MCS_CASELESS : MCS_PARTIAL; switch (type & ~MCL_NOEXPAND) { - case MCL_DOMAIN: cb.at_is_special = TRUE; /*FALLTHROUGH*/ + case MCL_DOMAIN: cb.flags |= MCS_AT_SPECIAL; /*FALLTHROUGH*/ case MCL_LOCALPART: cb.expand_setup = 0; break; default: cb.expand_setup = sep > UCHAR_MAX ? 0 : -1; break; } -cb.use_partial = TRUE; -cb.caseless = caseless; if (valueptr) *valueptr = NULL; return match_check_list(listptr, sep, anchorptr, &local_cache_bits, check_string, &cb, type, s, valueptr); @@ -1003,7 +1006,8 @@ Returns: OK for a match */ static int -check_address(void *arg, const uschar *pattern, const uschar **valueptr, uschar **error) +check_address(void * arg, const uschar * pattern, const uschar ** valueptr, + uschar ** error) { check_address_block * cb = (check_address_block *)arg; check_string_block csb; @@ -1026,7 +1030,7 @@ sdomain = Ustrrchr(subject, '@'); /* The only case where a subject may not have a domain is if the subject is empty. Otherwise, a subject with no domain is a serious configuration error. */ -if (sdomain == NULL && *subject != 0) +if (!sdomain && *subject) { log_write(0, LOG_MAIN|LOG_PANIC, "no @ found in the subject of an " "address list match: subject=\"%s\" pattern=\"%s\"", subject, pattern); @@ -1037,14 +1041,14 @@ if (sdomain == NULL && *subject != 0) This may be the empty address. */ if (*pattern == '^') - return match_check_string(subject, pattern, cb->expand_setup, TRUE, - cb->caseless, FALSE, NULL); + return match_check_string(subject, pattern, cb->expand_setup, + cb->flags | MCS_PARTIAL, NULL); /* Handle a pattern that is just a lookup. Skip over possible lookup names (letters, digits, hyphens). Skip over a possible * or *@ at the end. Then we must have a semicolon for it to be a lookup. */ -for (s = pattern; isalnum(*s) || *s == '-'; s++); +for (s = pattern; isalnum(*s) || *s == '-'; s++) ; if (*s == '*') s++; if (*s == '@') s++; @@ -1057,8 +1061,7 @@ if (*s == ';') if (Ustrncmp(pattern, "partial-", 8) == 0) log_write(0, LOG_MAIN|LOG_PANIC, "partial matching is not applicable to " "whole-address lookups: ignored \"partial-\" in \"%s\"", pattern); - return match_check_string(subject, pattern, -1, FALSE, cb->caseless, FALSE, - valueptr); + return match_check_string(subject, pattern, -1, cb->flags, valueptr); } /* For the remaining cases, an empty subject matches only an empty pattern, @@ -1085,19 +1088,20 @@ if (pattern[0] == '@' && pattern[1] == '@') { int sep = 0; - if ((rc = match_check_string(key, pattern + 2, -1, TRUE, FALSE, FALSE, - CUSS &list)) != OK) return rc; + if ((rc = match_check_string(key, pattern + 2, -1, MCS_PARTIAL, CUSS &list)) + != OK) + return rc; /* Check for chaining from the last item; set up the next key if one is found. */ ss = Ustrrchr(list, ':'); - if (ss == NULL) ss = list; else ss++; - while (isspace(*ss)) ss++; + if (!ss) ss = list; else ss++; + Uskip_whitespace(&ss); if (*ss == '>') { *ss++ = 0; - while (isspace(*ss)) ss++; + Uskip_whitespace(&ss); key = string_copy(ss); } else key = NULL; @@ -1117,8 +1121,7 @@ if (pattern[0] == '@' && pattern[1] == '@') else local_yield = OK; *sdomain = 0; - rc = match_check_string(subject, ss, -1, TRUE, cb->caseless, FALSE, - valueptr); + rc = match_check_string(subject, ss, -1, cb->flags + MCS_PARTIAL, valueptr); *sdomain = '@'; switch(rc) @@ -1148,8 +1151,7 @@ if (pattern[0] == '@' && pattern[1] == '@') /* We get here if the pattern is not a lookup or a regular expression. If it contains an @ there is both a local part and a domain. */ -pdomain = Ustrrchr(pattern, '@'); -if (pdomain != NULL) +if ((pdomain = Ustrrchr(pattern, '@'))) { int pllen, sllen; @@ -1177,7 +1179,7 @@ if (pdomain != NULL) { int cllen = pllen - 1; if (sllen < cllen) return FAIL; - if (cb->caseless + if (cb->flags & MCS_CASELESS ? strncmpic(subject+sllen-cllen, pattern + 1, cllen) != 0 : Ustrncmp(subject+sllen-cllen, pattern + 1, cllen) != 0) return FAIL; @@ -1192,7 +1194,7 @@ if (pdomain != NULL) else { if (sllen != pllen) return FAIL; - if (cb->caseless + if (cb->flags & MCS_CASELESS ? strncmpic(subject, pattern, sllen) != 0 : Ustrncmp(subject, pattern, sllen) != 0) return FAIL; } @@ -1205,18 +1207,17 @@ original code read as follows: return match_check_string(sdomain + 1, pdomain ? pdomain + 1 : pattern, - cb->expand_setup + expand_inc, TRUE, cb->caseless, TRUE, NULL); + cb->expand_setup + expand_inc, cb->flags, NULL); This supported only literal domains and *.x.y patterns. In order to allow for -named domain lists (so that you can right, for example, "senders=+xxxx"), it +named domain lists (so that you can write, for example, "senders=+xxxx"), it was changed to use the list scanning function. */ csb.origsubject = sdomain + 1; -csb.subject = cb->caseless ? string_copylc(sdomain+1) : string_copy(sdomain+1); +csb.subject = cb->flags & MCS_CASELESS + ? string_copylc(sdomain+1) : string_copy(sdomain+1); csb.expand_setup = cb->expand_setup + expand_inc; -csb.use_partial = TRUE; -csb.caseless = cb->caseless; -csb.at_is_special = TRUE; +csb.flags = MCS_PARTIAL | MCS_AT_SPECIAL | cb->flags & MCS_CASELESS; listptr = pdomain ? pdomain + 1 : pattern; if (valueptr) *valueptr = NULL; @@ -1321,10 +1322,10 @@ if (expand_setup == 0) ab.origaddress = address; /* ab.address is above */ ab.expand_setup = expand_setup; -ab.caseless = caseless; +ab.flags = caseless ? MCS_CASELESS : 0; return match_check_list(listptr, sep, &addresslist_anchor, &local_cache_bits, - check_address, &ab, MCL_ADDRESS + (expand? 0:MCL_NOEXPAND), address, + check_address, &ab, MCL_ADDRESS + (expand ? 0 : MCL_NOEXPAND), address, valueptr); } diff --git a/src/src/queue.c b/src/src/queue.c index 4bdd6fb14..6e47d2c8a 100644 --- a/src/src/queue.c +++ b/src/src/queue.c @@ -423,11 +423,11 @@ if (!recurse) /* If deliver_selectstring is a regex, compile it. */ if (deliver_selectstring && f.deliver_selectstring_regex) - selectstring_regex = regex_must_compile(deliver_selectstring, TRUE, FALSE); + selectstring_regex = regex_must_compile(deliver_selectstring, MCS_CASELESS, FALSE); if (deliver_selectstring_sender && f.deliver_selectstring_sender_regex) selectstring_regex_sender = - regex_must_compile(deliver_selectstring_sender, TRUE, FALSE); + regex_must_compile(deliver_selectstring_sender, MCS_CASELESS, FALSE); /* If the spool is split into subdirectories, we want to process it one directory at a time, so as to spread out the directory scanning and the @@ -1562,19 +1562,9 @@ memcpy(buf+1, msgid, MESSAGE_ID_LENGTH+1); if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) >= 0) { struct sockaddr_un sa_un = {.sun_family = AF_UNIX}; + ssize_t len = daemon_notifier_sockname(&sa_un); -#ifdef EXIM_HAVE_ABSTRACT_UNIX_SOCKETS - int len = offsetof(struct sockaddr_un, sun_path) + 1 - + snprintf(sa_un.sun_path+1, sizeof(sa_un.sun_path)-1, "%s", - expand_string(notifier_socket)); - sa_un.sun_path[0] = 0; -#else - int len = offsetof(struct sockaddr_un, sun_path) - + snprintf(sa_un.sun_path, sizeof(sa_un.sun_path), "%s", - expand_string(notifier_socket)); -#endif - - if (sendto(fd, buf, sizeof(buf), 0, (struct sockaddr *)&sa_un, len) < 0) + if (sendto(fd, buf, sizeof(buf), 0, (struct sockaddr *)&sa_un, (socklen_t)len) < 0) DEBUG(D_queue_run) debug_printf("%s: sendto %s\n", __FUNCTION__, strerror(errno)); close(fd); diff --git a/src/src/readconf.c b/src/src/readconf.c index 06bc50fd8..5068dc60e 100644 --- a/src/src/readconf.c +++ b/src/src/readconf.c @@ -243,6 +243,7 @@ static optionlist optionlist_config[] = { #ifdef LOOKUP_ORACLE { "oracle_servers", opt_stringptr, {&oracle_servers} }, #endif + { "panic_coredump", opt_bool, {&panic_coredump} }, { "percent_hack_domains", opt_stringptr, {&percent_hack_domains} }, #ifdef EXIM_PERL { "perl_at_start", opt_bool, {&opt_perl_at_start} }, @@ -2667,8 +2668,8 @@ switch(ol->type & opt_mask) break; case opt_bit: - printf("%s%s\n", ((*((int *)value)) & (1 << ((ol->type >> 16) & 31)))? - "" : "no_", name); + printf("%s%s\n", (*((int *)value)) & (1 << ((ol->type >> 16) & 31)) + ? "" : "no_", name); break; case opt_expand_bool: @@ -2693,7 +2694,7 @@ switch(ol->type & opt_mask) case opt_bool: case opt_bool_verify: case opt_bool_set: - printf("%s%s\n", (*((BOOL *)value))? "" : "no_", name); + printf("%s%s\n", *((BOOL *)value) ? "" : "no_", name); break; case opt_func: @@ -3497,7 +3498,7 @@ if (!process_log_path || !*process_log_path) /* Compile the regex for matching a UUCP-style "From_" line in an incoming message. */ -regex_From = regex_must_compile(uucp_from_pattern, FALSE, TRUE); +regex_From = regex_must_compile(uucp_from_pattern, MCS_NOFLAGS, TRUE); /* Unpick the SMTP rate limiting options, if set */ diff --git a/src/src/regex.c b/src/src/regex.c index 5c0f7c4e0..5de1c1704 100644 --- a/src/src/regex.c +++ b/src/src/regex.c @@ -18,9 +18,9 @@ /* Structure to hold a list of Regular expressions */ typedef struct pcre_list { - pcre2_code *re; - uschar *pcre_text; - struct pcre_list *next; + const pcre2_code * re; + uschar * pcre_text; + struct pcre_list * next; } pcre_list; uschar regex_match_string_buffer[1024]; @@ -28,31 +28,27 @@ uschar regex_match_string_buffer[1024]; extern FILE *mime_stream; extern uschar *mime_current_boundary; + static pcre_list * -compile(const uschar * list) +compile(const uschar * list, BOOL cacheable) { int sep = 0; -uschar *regex_string; -pcre_list *re_list_head = NULL; -pcre_list *ri; +uschar * regex_string; +pcre_list * re_list_head = NULL; +pcre_list * ri; /* precompile our regexes */ while ((regex_string = string_nextinlist(&list, &sep, NULL, 0))) if (strcmpic(regex_string, US"false") != 0 && Ustrcmp(regex_string, "0") != 0) { - pcre2_code * re; - int err; - PCRE2_SIZE pcre_erroffset; - /* compile our regular expression */ - if (!(re = pcre2_compile( (PCRE2_SPTR) regex_string, PCRE2_ZERO_TERMINATED, - 0, &err, &pcre_erroffset, pcre_cmp_ctx))) + uschar * errstr; + const pcre2_code * re = regex_compile(regex_string, + cacheable ? MCS_CACHEABLE : MCS_NOFLAGS, &errstr, pcre_gen_cmp_ctx); + + if (!re) { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - log_write(0, LOG_MAIN, - "regex acl condition warning - error in regex '%s': %s at offset %ld, skipped.", - regex_string, errbuf, (long)pcre_erroffset); + log_write(0, LOG_MAIN, "regex acl condition warning - %s, skipped", errstr); continue; } @@ -75,7 +71,7 @@ for (pcre_list * ri = re_list_head; ri; ri = ri->next) int n; /* try matcher on the line */ - if ((n = pcre2_match(ri->re, (PCRE2_SPTR)linebuffer, len, 0, 0, md, pcre_mtc_ctx)) > 0) + if ((n = pcre2_match(ri->re, (PCRE2_SPTR)linebuffer, len, 0, 0, md, pcre_gen_mtc_ctx)) > 0) { Ustrncpy(regex_match_string_buffer, ri->pcre_text, sizeof(regex_match_string_buffer)-1); @@ -85,19 +81,21 @@ for (pcre_list * ri = re_list_head; ri; ri = ri->next) { PCRE2_UCHAR * cstr; PCRE2_SIZE cslen; - pcre2_substring_get_bynumber(md, nn, &cstr, &cslen); + pcre2_substring_get_bynumber(md, nn, &cstr, &cslen); /* uses same ctx as md */ regex_vars[nn-1] = CUS cstr; } return OK; } } -pcre2_match_data_free(md); +/* pcre2_match_data_free(md); gen ctx needs no free */ return FAIL; } + + int -regex(const uschar **listptr) +regex(const uschar **listptr, BOOL cacheable) { unsigned long mbox_size; FILE *mbox_file; @@ -130,7 +128,7 @@ else } /* precompile our regexes */ -if (!(re_list_head = compile(*listptr))) +if (!(re_list_head = compile(*listptr, cacheable))) return FAIL; /* no regexes -> nothing to do */ /* match each line against all regexes */ @@ -167,7 +165,7 @@ return ret; int -mime_regex(const uschar **listptr) +mime_regex(const uschar **listptr, BOOL cacheable) { pcre_list *re_list_head = NULL; FILE *f; @@ -179,7 +177,7 @@ int ret; regex_match_string = NULL; /* precompile our regexes */ -if (!(re_list_head = compile(*listptr))) +if (!(re_list_head = compile(*listptr, cacheable))) return FAIL; /* no regexes -> nothing to do */ /* check if the file is already decoded */ diff --git a/src/src/regex_cache.c b/src/src/regex_cache.c new file mode 100644 index 000000000..63cddce1d --- /dev/null +++ b/src/src/regex_cache.c @@ -0,0 +1,252 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* + * Copyright (c) The Exim Maintainers 2022 + * License: GPL + */ + +/* Caching layers for compiled REs. There is a local layer in the process, +implemented as a tree for inserts and lookup. This cache is inherited from +the daemon, for the process tree deriving from there - but not by re-exec'd +proceses or commandline submission processes. + +If the process has to compile, and is not the daemon or a re-exec'd exim, +it notifies the use of the RE to the daemon via a unix-domain socket. +This is a fire-and-forget send with no response, hence cheap from the point-of +view of the sender. I have not measured the overall comms costs. The +daemon also compiles the RE, and caches the result. + +A second layer would be possible by asking the daemon via the notifier socket +(for a result from its cache, or a compile if it must). The comms overhead +is significant, not only for the channel but also for de/serialisation of +the compiled object. This makes it untenable for the primary use-case, the +transport process which has been re-exec'd to gain privs - and therefore does not +have the daemon-maintained cache. Using shared-memory might reduce that cost +(the attach time for the memory segment will matter); the implimentation +would require suitable R/W locks. +*/ + +#include "exim.h" + +typedef struct re_req { + uschar notifier_reqtype; + BOOL caseless; + uschar re[1]; /* extensible */ +} re_req; + +static tree_node * regex_cache = NULL; +static tree_node * regex_caseless_cache = NULL; + +#define REGEX_CACHESIZE_LIMIT 1000 + +/******************************************************************************/ + +static void +regex_to_daemon(const uschar * key, BOOL caseless) +{ +int klen = Ustrlen(key) + 1; +int rlen = sizeof(re_req) + klen; +re_req * req; +int fd, old_pool = store_pool; + +DEBUG(D_expand|D_lists) + debug_printf_indent("sending RE '%s' to daemon\n", key); + +store_pool = POOL_MAIN; + req = store_get(rlen, key); /* maybe need a size limit */ +store_pool = old_pool;; +req->notifier_reqtype = NOTIFY_REGEX; +req->caseless = caseless; +memcpy(req->re, key, klen); + +if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) >= 0) + { + struct sockaddr_un sa_un = {.sun_family = AF_UNIX}; + ssize_t len = daemon_notifier_sockname(&sa_un); + + if (sendto(fd, req, rlen, 0, (struct sockaddr *)&sa_un, (socklen_t)len) < 0) + DEBUG(D_queue_run) + debug_printf("%s: sendto %s\n", __FUNCTION__, strerror(errno)); + close(fd); + } +else DEBUG(D_queue_run) debug_printf(" socket: %s\n", strerror(errno)); +} + + +static const pcre2_code * +regex_from_cache(const uschar * key, BOOL caseless) +{ +tree_node * node = + tree_search(caseless ? regex_caseless_cache : regex_cache, key); +DEBUG(D_expand|D_lists) + debug_printf_indent("compiled %sRE '%s' %sfound in local cache\n", + caseless ? "caseless " : "", key, node ? "" : "not "); + +return node ? node->data.ptr : NULL; +} + + +static void +regex_to_cache(const uschar * key, BOOL caseless, const pcre2_code * cre) +{ +PCRE2_SIZE srelen; +uschar * sre; +tree_node * node; + +node = store_get(sizeof(tree_node) + Ustrlen(key) + 1, key); /* we are called with STORE_PERM */ +Ustrcpy(node->name, key); +node->data.ptr = (void *)cre; + +if (!tree_insertnode(caseless ? ®ex_caseless_cache : ®ex_cache, node)) + { DEBUG(D_expand|D_lists) debug_printf_indent("duplicate key!\n"); } +else DEBUG(D_expand|D_lists) + debug_printf_indent("compiled RE '%s' saved in local cache\n", key); + +/* Additionally, if not re-execed and not the daemon, tell the daemon of the RE +so it can add to the cache */ + +if (f.daemon_scion && !f.daemon_listen) + regex_to_daemon(key, caseless); + +return; +} + +/******************************************************************************/ + +/************************************************* +* Compile regular expression and panic on fail * +*************************************************/ + +/* This function is called when failure to compile a regular expression leads +to a panic exit. In other cases, pcre_compile() is called directly. In many +cases where this function is used, the results of the compilation are to be +placed in long-lived store, so we temporarily reset the store management +functions that PCRE uses if the use_malloc flag is set. + +Argument: + pattern the pattern to compile + flags + caseless caseless matching is required + cacheable use (writeback) cache + use_malloc TRUE if compile into malloc store + +Returns: pointer to the compiled pattern +*/ + +const pcre2_code * +regex_must_compile(const uschar * pattern, mcs_flags flags, BOOL use_malloc) +{ +BOOL caseless = !!(flags & MCS_CASELESS); +size_t offset; +const pcre2_code * yield; +int old_pool = store_pool, err; + +/* Optionall, check the cache and return if found */ + +if ( flags & MCS_CACHEABLE + && (yield = regex_from_cache(pattern, caseless))) + return yield; + +store_pool = POOL_PERM; + +if (!(yield = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, + caseless ? PCRE_COPT|PCRE2_CASELESS : PCRE_COPT, + &err, &offset, use_malloc ? pcre_mlc_cmp_ctx : pcre_gen_cmp_ctx))) + { + uschar errbuf[128]; + pcre2_get_error_message(err, errbuf, sizeof(errbuf)); + log_write(0, LOG_MAIN|LOG_PANIC_DIE, "regular expression error: " + "%s at offset %ld while compiling %s", errbuf, (long)offset, pattern); + } + +if (use_malloc) + { + /*pcre2_general_context_free(gctx);*/ + } + +if (flags & MCS_CACHEABLE) + regex_to_cache(pattern, caseless, yield); + +store_pool = old_pool; +return yield; +} + + + + +/* Wrapper for pcre2_compile() and error-message handling. + +Arguments: pattern regex to compile + flags + caseless flag for match variant + cacheable use (writeback) cache + errstr on error, filled in with error message + cctx compile-context for pcre2 + +Return: NULL on error, with errstr set. Otherwise, the compiled RE object +*/ + +const pcre2_code * +regex_compile(const uschar * pattern, mcs_flags flags, uschar ** errstr, + pcre2_compile_context * cctx) +{ +const uschar * key = pattern; +BOOL caseless = !!(flags & MCS_CASELESS); +int err; +PCRE2_SIZE offset; +const pcre2_code * yield; +int old_pool = store_pool; + +/* Optionally, check the cache and return if found */ + +if ( flags & MCS_CACHEABLE + && (yield = regex_from_cache(key, caseless))) + return yield; + +DEBUG(D_expand|D_lists) debug_printf_indent("compiling %sRE '%s'\n", + caseless ? "caseless " : "", pattern); + +store_pool = POOL_PERM; +if (!(yield = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, + caseless ? PCRE_COPT|PCRE2_CASELESS : PCRE_COPT, + &err, &offset, cctx))) + { + uschar errbuf[128]; + pcre2_get_error_message(err, errbuf, sizeof(errbuf)); + store_pool = old_pool; + *errstr = string_sprintf("regular expression error in " + "\"%s\": %s at offset %ld", pattern, errbuf, (long)offset); + } +else if (flags & MCS_CACHEABLE) + regex_to_cache(key, caseless, yield); +store_pool = old_pool; + +return yield; +} + + + +/* Handle a regex notify arriving at the daemon. We get sent the original RE; +compile it (again) and write to the cache. Later forked procs will be able to +read from the cache, unless they re-execed. Therefore, those latter never bother +sending us a notification. */ + +void +regex_at_daemon(const uschar * reqbuf) +{ +const re_req * req = (const re_req *)reqbuf; +uschar * errstr; +const pcre2_code * cre; + +if (regex_cachesize >= REGEX_CACHESIZE_LIMIT) + errstr = US"regex cache size limit reached"; +else if ((cre = regex_compile(req->re, + req->caseless ? MCS_CASELESS | MCS_CACHEABLE : MCS_CACHEABLE, + &errstr, pcre_gen_cmp_ctx))) + regex_cachesize++; + +DEBUG(D_any) if (!cre) debug_printf("%s\n", errstr); +return; +} diff --git a/src/src/rewrite.c b/src/src/rewrite.c index 005dc51fe..bfd78b5f1 100644 --- a/src/src/rewrite.c +++ b/src/src/rewrite.c @@ -136,7 +136,8 @@ for (rewrite_rule * rule = rewrite_rules; if (flag & rewrite_smtp) { - uschar *key = expand_string(rule->key); + BOOL textonly_re; + const uschar * key = expand_string_2(rule->key, &textonly_re); if (!key) { if (!f.expand_string_forcedfail) @@ -144,7 +145,8 @@ for (rewrite_rule * rule = rewrite_rules; "checking for SMTP rewriting: %s", rule->key, expand_string_message); continue; } - if (match_check_string(subject, key, 0, TRUE, FALSE, FALSE, NULL) != OK) + if (match_check_string(subject, key, 0, + textonly_re ? MCS_CACHEABLE | MCS_PARTIAL : MCS_PARTIAL, NULL) != OK) continue; new = expand_string(rule->replacement); } diff --git a/src/src/routers/iplookup.c b/src/src/routers/iplookup.c index 94cde4e04..8b67f3116 100644 --- a/src/src/routers/iplookup.c +++ b/src/src/routers/iplookup.c @@ -84,10 +84,10 @@ iplookup_router_options_block iplookup_router_option_defaults = { consistency checks to be done, or anything else that needs to be set up. */ void -iplookup_router_init(router_instance *rblock) +iplookup_router_init(router_instance * rblock) { -iplookup_router_options_block *ob = - (iplookup_router_options_block *)(rblock->options_block); +iplookup_router_options_block * ob = + (iplookup_router_options_block *) rblock->options_block; /* A port and a host list must be given */ @@ -95,13 +95,13 @@ if (ob->port < 0) log_write(0, LOG_PANIC_DIE|LOG_CONFIG_FOR, "%s router:\n " "a port must be specified", rblock->name); -if (ob->hosts == NULL) +if (!ob->hosts) log_write(0, LOG_PANIC_DIE|LOG_CONFIG_FOR, "%s router:\n " "a host list must be specified", rblock->name); /* Translate protocol name into value */ -if (ob->protocol_name != NULL) +if (ob->protocol_name) { if (Ustrcmp(ob->protocol_name, "udp") == 0) ob->protocol = ip_udp; else if (Ustrcmp(ob->protocol_name, "tcp") == 0) ob->protocol = ip_tcp; @@ -111,9 +111,9 @@ if (ob->protocol_name != NULL) /* If a response pattern is given, compile it now to get the error early. */ -if (ob->response_pattern != NULL) +if (ob->response_pattern) ob->re_response_pattern = - regex_must_compile(ob->response_pattern, FALSE, TRUE); + regex_must_compile(ob->response_pattern, MCS_NOFLAGS, TRUE); } diff --git a/src/src/structs.h b/src/src/structs.h index b38aa6a9d..06cd06084 100644 --- a/src/src/structs.h +++ b/src/src/structs.h @@ -895,7 +895,7 @@ typedef struct check_host_block { const uschar *host_name; const uschar *host_address; const uschar *host_ipv4; - BOOL negative; + mcs_flags flags; } check_host_block; /* Structure for remembering lookup data when caching the result of diff --git a/src/src/transports/appendfile.c b/src/src/transports/appendfile.c index 93281efda..7e29dd3bc 100644 --- a/src/src/transports/appendfile.c +++ b/src/src/transports/appendfile.c @@ -681,7 +681,7 @@ for (struct dirent *ent; ent = readdir(dir); ) { pcre2_match_data * md = pcre2_match_data_create(2, pcre_gen_ctx); int rc = pcre2_match(re, (PCRE2_SPTR)name, PCRE2_ZERO_TERMINATED, - 0, 0, md, pcre_mtc_ctx); + 0, 0, md, pcre_gen_mtc_ctx); PCRE2_SIZE * ovec = pcre2_get_ovector_pointer(md); if ( rc >= 0 && (rc = pcre2_get_ovector_count(md)) >= 2) @@ -694,9 +694,11 @@ for (struct dirent *ent; ent = readdir(dir); ) DEBUG(D_transport) debug_printf("check_dir_size: size from %s is " OFF_T_FMT "\n", name, size); + /* pcre2_match_data_free(md); gen ctx needs no free */ continue; } } + /* pcre2_match_data_free(md); gen ctx needs no free */ DEBUG(D_transport) debug_printf("check_dir_size: regex did not match %s\n", name); } @@ -2211,23 +2213,14 @@ else if (ob->quota_value > 0 || THRESHOLD_CHECK || ob->maildir_use_size_file) { - PCRE2_SIZE offset; - int err; - /* Compile the regex if there is one. */ if (ob->quota_size_regex) { - if (!(re = pcre2_compile((PCRE2_SPTR)ob->quota_size_regex, - PCRE2_ZERO_TERMINATED, PCRE_COPT, &err, &offset, pcre_cmp_ctx))) - { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - addr->message = string_sprintf("appendfile: regular expression " - "error: %s at offset %ld while compiling %s", errbuf, (long)offset, - ob->quota_size_regex); + if (!(re = regex_compile(ob->quota_size_regex, + MCS_NOFLAGS, &addr->message, pcre_gen_cmp_ctx))) return FALSE; - } + DEBUG(D_transport) debug_printf("using regex for file sizes: %s\n", ob->quota_size_regex); } @@ -2300,23 +2293,14 @@ else if (ob->maildir_use_size_file) { const pcre2_code * dir_regex = NULL; - PCRE2_SIZE offset; - int err; if (ob->maildir_dir_regex) { int check_path_len = Ustrlen(check_path); - if (!(dir_regex = pcre2_compile((PCRE2_SPTR)ob->maildir_dir_regex, - PCRE2_ZERO_TERMINATED, PCRE_COPT, &err, &offset, pcre_cmp_ctx))) - { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - addr->message = string_sprintf("appendfile: regular expression " - "error: %s at offset %ld while compiling %s", errbuf, (long)offset, - ob->maildir_dir_regex); + if (!(dir_regex = regex_compile(ob->maildir_dir_regex, + MCS_NOFLAGS, &addr->message, pcre_gen_cmp_ctx))) return FALSE; - } DEBUG(D_transport) debug_printf("using regex for maildir directory selection: %s\n", diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 7f529b7ca..6eee04d03 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -275,7 +275,7 @@ struct list for (struct list * l = list; l < list + nelem(list); l++) if (!*l->re) - *l->re = regex_must_compile(l->string, FALSE, TRUE); + *l->re = regex_must_compile(l->string, MCS_NOFLAGS, TRUE); } @@ -1000,7 +1000,7 @@ uschar authnum; unsigned short authbits = 0; if (!sx->esmtp) return 0; -if (!regex_AUTH) regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE); +if (!regex_AUTH) regex_AUTH = regex_must_compile(AUTHS_REGEX, MCS_NOFLAGS, TRUE); if (!regex_match_and_setup(regex_AUTH, sx->buffer, 0, -1)) return 0; expand_nmax = -1; /* reset */ names = string_copyn(expand_nstring[1], expand_nlength[1]); @@ -1563,7 +1563,7 @@ f.smtp_authenticated = FALSE; client_authenticator = client_authenticated_id = client_authenticated_sender = NULL; if (!regex_AUTH) - regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE); + regex_AUTH = regex_must_compile(AUTHS_REGEX, MCS_NOFLAGS, TRUE); /* Is the server offering AUTH? */ @@ -1849,57 +1849,57 @@ pcre2_match_data * md = pcre2_match_data_create(1, pcre_gen_ctx); #ifndef DISABLE_TLS if ( checks & OPTION_TLS && pcre2_match(regex_STARTTLS, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_TLS; if ( checks & OPTION_IGNQ && pcre2_match(regex_IGNOREQUOTA, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_IGNQ; if ( checks & OPTION_CHUNKING && pcre2_match(regex_CHUNKING, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_CHUNKING; #ifndef DISABLE_PRDR if ( checks & OPTION_PRDR && pcre2_match(regex_PRDR, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_PRDR; #ifdef SUPPORT_I18N if ( checks & OPTION_UTF8 && pcre2_match(regex_UTF8, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_UTF8; if ( checks & OPTION_DSN && pcre2_match(regex_DSN, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_DSN; if ( checks & OPTION_PIPE && pcre2_match(regex_PIPELINING, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_PIPE; if ( checks & OPTION_SIZE && pcre2_match(regex_SIZE, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_SIZE; #ifndef DISABLE_PIPE_CONNECT if ( checks & OPTION_EARLY_PIPE && pcre2_match(regex_EARLY_PIPE, - (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0) + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_EARLY_PIPE; -pcre2_match_data_free(md); +/* pcre2_match_data_free(md); gen ctx needs no free */ /* debug_printf("%s: found 0x%04x\n", __FUNCTION__, checks); */ return checks; } diff --git a/src/src/transports/tf_maildir.c b/src/src/transports/tf_maildir.c index a83fc6f09..205ee41cb 100644 --- a/src/src/transports/tf_maildir.c +++ b/src/src/transports/tf_maildir.c @@ -142,22 +142,13 @@ a subfolder, and should ensure that a maildirfolder file exists. */ if (maildirfolder_create_regex) { - int err; - PCRE2_SIZE offset; const pcre2_code * re; DEBUG(D_transport) debug_printf("checking for maildirfolder requirement\n"); - if (!(re = pcre2_compile((PCRE2_SPTR)maildirfolder_create_regex, - PCRE2_ZERO_TERMINATED, PCRE_COPT, &err, &offset, pcre_cmp_ctx))) - { - uschar errbuf[128]; - pcre2_get_error_message(err, errbuf, sizeof(errbuf)); - addr->message = string_sprintf("appendfile: regular expression " - "error: %s at offset %ld while compiling %s", errbuf, (long)offset, - maildirfolder_create_regex); + if (!(re = regex_compile(maildirfolder_create_regex, + MCS_NOFLAGS, &addr->message, pcre_gen_cmp_ctx))) return FALSE; - } if (regex_match(re, path, -1, NULL)) { diff --git a/src/src/verify.c b/src/src/verify.c index b4c2b9a8f..afc18d553 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -3074,7 +3074,7 @@ digits, full stops, and hyphens (the constituents of domain names). Allow underscores, as they are all too commonly found. Sigh. Also, if allow_utf8_domains is set, allow top-bit characters. */ -for (t = ss; *t != 0; t++) +for (t = ss; *t; t++) if (!isalnum(*t) && *t != '.' && *t != '-' && *t != '_' && (!allow_utf8_domains || *t < 128)) break; @@ -3082,7 +3082,7 @@ for (t = ss; *t != 0; t++) its IP address and match against that. Note that a multi-homed host will add items to the chain. */ -if (*t == 0) +if (!*t) { int rc; host_item h; @@ -3113,8 +3113,8 @@ outgoing hosts, the name is always given explicitly. If it is NULL, it means we must use sender_host_name and its aliases, looking them up if necessary. */ if (cb->host_name) /* Explicit host name given */ - return match_check_string(cb->host_name, ss, -1, TRUE, TRUE, TRUE, - valueptr); + return match_check_string(cb->host_name, ss, -1, + MCS_PARTIAL | MCS_CASELESS | MCS_AT_SPECIAL | cb->flags, valueptr); /* Host name not given; in principle we need the sender host name and its aliases. However, for query-style lookups, we do not need the name if the @@ -3143,7 +3143,9 @@ if ((semicolon = Ustrchr(ss, ';'))) if (isquery) { - switch(match_check_string(US"", ss, -1, TRUE, TRUE, TRUE, valueptr)) + switch(match_check_string(US"", ss, -1, + MCS_PARTIAL| MCS_CASELESS| MCS_AT_SPECIAL | (cb->flags & MCS_CACHEABLE), + valueptr)) { case OK: return OK; case DEFER: return DEFER; @@ -3169,7 +3171,9 @@ if (!sender_host_name) /* Match on the sender host name, using the general matching function */ -switch(match_check_string(sender_host_name, ss, -1, TRUE, TRUE, TRUE, valueptr)) +switch(match_check_string(sender_host_name, ss, -1, + MCS_PARTIAL| MCS_CASELESS| MCS_AT_SPECIAL | (cb->flags & MCS_CACHEABLE), + valueptr)) { case OK: return OK; case DEFER: return DEFER; @@ -3179,7 +3183,9 @@ switch(match_check_string(sender_host_name, ss, -1, TRUE, TRUE, TRUE, valueptr)) aliases = sender_host_aliases; while (*aliases) - switch(match_check_string(*aliases++, ss, -1, TRUE, TRUE, TRUE, valueptr)) + switch(match_check_string(*aliases++, ss, -1, + MCS_PARTIAL| MCS_CASELESS| MCS_AT_SPECIAL | (cb->flags & MCS_CACHEABLE), + valueptr)) { case OK: return OK; case DEFER: return DEFER; @@ -3255,8 +3261,8 @@ rc = match_check_list( check_host, /* function for testing */ &cb, /* argument for function */ MCL_HOST, /* type of check */ - (host_address == sender_host_address)? - US"host" : host_address, /* text for debugging */ + host_address == sender_host_address + ? US"host" : host_address, /* text for debugging */ valueptr); /* where to pass back data */ deliver_host_address = save_host_address; return rc; |