From 8d909960c69378a54d9493586d74ba361948bf49 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 18 Mar 2017 14:41:50 +0000 Subject: Expansions: check numeric values of IPv4 address components --- test/stdout/0002 | 3 +++ 1 file changed, 3 insertions(+) (limited to 'test/stdout') diff --git a/test/stdout/0002 b/test/stdout/0002 index fa445b01c..5593f06cc 100644 --- a/test/stdout/0002 +++ b/test/stdout/0002 @@ -393,6 +393,8 @@ newline tab\134backslash ~tilde\177DEL\200\201. > isip: y 1.2.3.4 > isip4: y 1.2.3.4 > isip6: n 1.2.3.4 +> isip: n ::1.2.3.256 +> isip4: n 1.2.3.256 > isip: n 1:2:3:4 > isip4: n 1:2:3:4 > isip6: n 1:2:3:4 @@ -402,6 +404,7 @@ newline tab\134backslash ~tilde\177DEL\200\201. > isip: y fe80::a00:20ff:fe86:a061 > isip4: n fe80::a00:20ff:fe86:a061 > isip6: y fe80::a00:20ff:fe86:a061 +> isip: y fe80::1.2.3.4 > isip: n rhubarb > isip4: n rhubarb > isip6: n rhubarb -- cgit v1.2.3 From 328c5688dbe0f4c14418f22350ccd99b3fe8ac71 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Fri, 28 Apr 2017 23:54:35 +0100 Subject: Support wire-format spoolfiles --- doc/doc-docbook/spec.xfpt | 30 ++++++++++- doc/doc-txt/NewStuff | 6 +++ doc/doc-txt/OptionLists.txt | 1 + src/src/deliver.c | 2 - src/src/dkim_transport.c | 18 ++++--- src/src/globals.c | 2 + src/src/globals.h | 2 + src/src/pdkim/pdkim.c | 6 +-- src/src/readconf.c | 1 + src/src/receive.c | 49 +++++++++++++++-- src/src/spam.c | 4 +- src/src/spool_in.c | 7 +++ src/src/spool_mbox.c | 60 +++++++++++---------- src/src/spool_out.c | 5 +- src/src/transport.c | 111 +++++++++++++++++++++++++++++++------- src/src/transports/smtp.c | 3 ++ src/src/verify.c | 1 + test/confs/0906 | 89 +++++++++++++++++++++++++++++++ test/confs/4530 | 1 + test/log/0906 | 9 ++++ test/log/4530 | 17 ++++++ test/mail/0906.a | 109 ++++++++++++++++++++++++++++++++++++++ test/mail/4530.y | 26 +++++++++ test/mail/4530.z | 26 +++++++++ test/scripts/0000-Basic/0906 | 122 ++++++++++++++++++++++++++++++++++++++++++ test/scripts/4500-DKIM/4530 | 66 +++++++++++++++++++++++ test/stderr/0143 | 1 + test/stderr/0169 | 1 + test/stderr/0275 | 1 - test/stderr/0386 | 2 + test/stderr/0393 | 3 ++ test/stderr/0402 | 2 + test/stderr/0404 | 1 + test/stderr/0408 | 1 + test/stderr/0476 | 1 + test/stderr/0487 | 1 + test/stderr/0512 | 2 + test/stderr/2600 | 1 + test/stderr/4520 | 6 +-- test/stderr/5004 | 1 + test/stderr/5005 | 2 + test/stderr/5006 | 1 + test/stderr/5008 | 2 + test/stdout/0906 | 123 +++++++++++++++++++++++++++++++++++++++++++ test/stdout/4530 | 75 ++++++++++++++++++++++++++ 45 files changed, 930 insertions(+), 70 deletions(-) create mode 100644 test/confs/0906 create mode 120000 test/confs/4530 create mode 100644 test/log/0906 create mode 100644 test/log/4530 create mode 100644 test/mail/0906.a create mode 100644 test/mail/4530.y create mode 100644 test/mail/4530.z create mode 100644 test/scripts/0000-Basic/0906 create mode 100644 test/scripts/4500-DKIM/4530 create mode 100644 test/stdout/0906 create mode 100644 test/stdout/4530 (limited to 'test/stdout') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 7a1b491c0..e5c433fbb 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -13566,6 +13566,7 @@ listed in more than one group. .row &%message_body_visible%& "how much to show in &$message_body$&" .row &%mua_wrapper%& "run in &""MUA wrapper""& mode" .row &%print_topbitchars%& "top-bit characters are printing" +.row &%spool_wireformat%& "use wire-format spool data files when possible" .row &%timezone%& "force time zone" .endtable @@ -16821,6 +16822,25 @@ as failures in the configuration file. By using this option to override the compiled-in path, it is possible to run tests of Exim without using the standard spool. +.new +.option spool_wireformat main boolean false +.cindex "spool directory" "file formats" +If this option is set, Exim may for some messages use an alternate format +for data-files in the spool which matches the wire format. +Doing this permits more efficient message reception and transmission. +Currently it is only done for messages received using the EMSTP CHUNKING +option. + +Users of the local_scan() API (see &<>&), +and any external programs which are passed a reference to a message data file +(except via the &"regex"&, &"malware"& or &"spam"&) ACL conditions) +will need to be aware of the potential different format. + +Using any of the ACL conditions noted will negate the reception benefit +(as a Unix-mbox-format file is contructed for them). +The transimssion benefit is maintained. +.wen + .option sqlite_lock_timeout main time 5s .cindex "sqlite lookup type" "lock timeout" This option controls the timeout that the &(sqlite)& lookup uses when trying to @@ -38094,8 +38114,8 @@ The address of an authenticated sender &-- the value of the &$authenticated_sender$& variable. .vitem "&%-body_linecount%&&~<&'number'&>" -This records the number of lines in the body of the message, and is always -present. +This records the number of lines in the body of the message, and is +present unless &%-spool_file_wireformat%& is. .vitem "&%-body_zerocount%&&~<&'number'&>" This records the number of binary zero bytes in the body of the message, and is @@ -38177,6 +38197,12 @@ to ensure that the caller is displayed in queue listings). If a message was scanned by SpamAssassin, this is present. It records the value of &$spam_score_int$&. +.vitem &%-spool_file_wireformat%& +The -D file for this message is in wire-format (for ESMTP CHUNKING) +rather than Unix-format. +The line-ending is CRLF rather than newline. +There is still, however, no leading-dot-stuffing. + .vitem &%-tls_certificate_verified%& A TLS certificate was received from the client that sent this message, and the certificate was verified by the server. diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 5f446f792..cb2346f69 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -29,6 +29,12 @@ Version 4.90 6. The reproducible build $SOURCE_DATE_EPOCH environment variable is now supported. + 7. Optionally, an alternate format for spool data-files which matches the + wire format - meaning more efficient reception and transmission (at the + cost of difficulty with standard Unix tools). Only used for messages + received using the ESMTP CHUNKING option, and when a new main-section + option "spool_wireformat" (false by default) is set. + Version 4.89 ------------ diff --git a/doc/doc-txt/OptionLists.txt b/doc/doc-txt/OptionLists.txt index b6439e6ed..95d321e5c 100644 --- a/doc/doc-txt/OptionLists.txt +++ b/doc/doc-txt/OptionLists.txt @@ -529,6 +529,7 @@ socket string* unset lmtp spamd_address string* + main 4.50 with content scan split_spool_directory boolean false main 1.70 spool_directory string ++ main +spool_wireformat boolean false main 4.90 sqlite_lock_timeout time 5s main 4.53 strict_acl_vars boolean false main 4.64 srv_fail_domains domain list unset dnslookup 4.43 diff --git a/src/src/deliver.c b/src/src/deliver.c index 2787d0040..262ae454f 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -3890,14 +3890,12 @@ for (;;) /* Normally we do not repeat this loop */ maxpipe = 0; FD_ZERO(&select_pipes); for (poffset = 0; poffset < remote_max_parallel; poffset++) - { if (parlist[poffset].pid != 0) { int fd = parlist[poffset].fd; FD_SET(fd, &select_pipes); if (fd > maxpipe) maxpipe = fd; } - } /* Stick in a 60-second timeout, just in case. */ diff --git a/src/src/dkim_transport.c b/src/src/dkim_transport.c index 2aba56054..4538b36e3 100644 --- a/src/src/dkim_transport.c +++ b/src/src/dkim_transport.c @@ -45,9 +45,6 @@ DEBUG(D_transport) debug_printf("send file fd=%d size=%d\n", out_fd, size - off) /*XXX should implement timeout, like transport_write_block_fd() ? */ -/* Rewind file */ -lseek(in_fd, off, SEEK_SET); - #ifdef HAVE_LINUX_SENDFILE /* We can use sendfile() to shove the file contents to the socket. However only if we don't use TLS, @@ -69,6 +66,9 @@ else { int sread, wwritten; + /* Rewind file */ + lseek(in_fd, off, SEEK_SET); + /* Send file down the original fd */ while((sread = read(in_fd, deliver_out_buffer, DELIVER_OUT_BUFFER_SIZE)) >0) { @@ -118,6 +118,7 @@ dkt_direct(transport_ctx * tctx, struct ob_dkim * dkim, { int save_fd = tctx->u.fd; int save_options = tctx->options; +BOOL save_wireformat = spool_file_wireformat; uschar * hdrs, * dkim_signature; int siglen, hsize; const uschar * errstr; @@ -125,7 +126,8 @@ BOOL rc; DEBUG(D_transport) debug_printf("dkim signing direct-mode\n"); -/* Get headers in string for signing and transmission */ +/* Get headers in string for signing and transmission. Do CRLF +and dotstuffing (but no body nor dot-termination) */ tctx->u.msg = NULL; tctx->options = tctx->options & ~(topt_end_dot | topt_use_bdat) @@ -155,14 +157,18 @@ else if (!(rc = dkt_sign_fail(dkim, &errno))) /* Write the signature and headers into the deliver-out-buffer. This should mean they go out in the same packet as the MAIL, RCPT and (first) BDAT commands (transport_write_message() sizes the BDAT for the buffered amount) - for short -messages, the BDAT LAST command. We want no CRLF or dotstuffing expansion */ +messages, the BDAT LAST command. We want no dotstuffing expansion here, it +having already been done - but we have to say we want CRLF output format, and +temporarily set the marker for possible already-CRLF input. */ -tctx->options &= ~(topt_use_crlf | topt_escape_headers); +tctx->options &= ~topt_escape_headers; +spool_file_wireformat = TRUE; transport_write_reset(0); if ( !write_chunk(tctx, dkim_signature, siglen) || !write_chunk(tctx, hdrs, hsize)) return FALSE; +spool_file_wireformat = save_wireformat; tctx->options = save_options | topt_no_headers | topt_continuation; if (!(transport_write_message(tctx, 0))) diff --git a/src/src/globals.c b/src/src/globals.c index d61e894f3..f722fab12 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -1363,6 +1363,8 @@ uschar *spf_smtp_comment = NULL; BOOL split_spool_directory = FALSE; uschar *spool_directory = US SPOOL_DIRECTORY "\0<--------------Space to patch spool_directory->"; +BOOL spool_file_wireformat = FALSE; +BOOL spool_wireformat = FALSE; #ifdef EXPERIMENTAL_SRS uschar *srs_config = NULL; uschar *srs_db_address = NULL; diff --git a/src/src/globals.h b/src/src/globals.h index 1c58a936b..e31517bf4 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -872,6 +872,8 @@ extern uschar *spf_smtp_comment; /* spf comment to include in SMTP reply * #endif extern BOOL split_spool_directory; /* TRUE to use multiple subdirs */ extern uschar *spool_directory; /* Name of spool directory */ +extern BOOL spool_file_wireformat; /* current -D file has CRLF rather than NL */ +extern BOOL spool_wireformat; /* can write wireformat -D files */ #ifdef EXPERIMENTAL_SRS extern uschar *srs_config; /* SRS config secret:max age:hash length:use timestamp:use hash */ extern uschar *srs_db_address; /* SRS db address */ diff --git a/src/src/pdkim/pdkim.c b/src/src/pdkim/pdkim.c index e4384d7f7..61e3161d4 100644 --- a/src/src/pdkim/pdkim.c +++ b/src/src/pdkim/pdkim.c @@ -1419,7 +1419,7 @@ while (sig) } DEBUG(D_acl) debug_printf( - "PDKIM >> Header data for hash, canonicalized, in sequence >>>>>>>>>>>>>>\n"); + "PDKIM >> Header data for hash, canonicalized, in sequence >>>>>>>>>>>>\n"); /* SIGNING ---------------------------------------------------------------- */ /* When signing, walk through our header list and add them to the hash. As we @@ -1732,10 +1732,10 @@ DEBUG(D_acl) pdkim_signature s = *sig; ev_ctx vctx; - debug_printf("PDKIM (checking verify key)<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"); + debug_printf("PDKIM (checking verify key)>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); if (!pdkim_key_from_dns(ctx, &s, &vctx, errstr)) debug_printf("WARNING: bad dkim key in dns\n"); - debug_printf("PDKIM (finished checking verify key)<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"); + debug_printf("PDKIM (finished checking verify key)<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"); } return ctx; } diff --git a/src/src/readconf.c b/src/src/readconf.c index 340a0c0eb..f43a3d163 100644 --- a/src/src/readconf.c +++ b/src/src/readconf.c @@ -432,6 +432,7 @@ static optionlist optionlist_config[] = { #endif { "split_spool_directory", opt_bool, &split_spool_directory }, { "spool_directory", opt_stringptr, &spool_directory }, + { "spool_wireformat", opt_bool, &spool_wireformat }, #ifdef LOOKUP_SQLITE { "sqlite_lock_timeout", opt_int, &sqlite_lock_timeout }, #endif diff --git a/src/src/receive.c b/src/src/receive.c index 731c76d77..3d92a8479 100644 --- a/src/src/receive.c +++ b/src/src/receive.c @@ -1016,6 +1016,46 @@ for(;;) /*NOTREACHED*/ } +static int +read_message_bdat_smtp_wire(FILE *fout) +{ +int ch; + +/* Remember that this message uses wireformat. */ + +DEBUG(D_receive) debug_printf("CHUNKING: writing spoolfile in wire format\n"); +spool_file_wireformat = TRUE; + +/* Unfortunately cannot use sendfile() even if not TLS +as that requires (on linux) mmap-like operations on the input fd. + +XXX but worthwhile doing a block interface to the bdat_getc buffer +in the future */ + +for (;;) switch (ch = bdat_getc(GETC_BUFFER_UNLIMITED)) + { + case EOF: return END_EOF; + case EOD: return END_DOT; + case ERR: return END_PROTOCOL; + + default: + message_size++; +/*XXX not done: +linelength +max_received_linelength +body_linecount +body_zerocount +*/ + if (fout) + { + if (fputc(ch, fout) == EOF) return END_WERROR; + if (message_size > thismessage_size_limit) return END_SIZE; + } + break; + } +/*NOTREACHED*/ +} + @@ -3078,9 +3118,11 @@ if (!ferror(data_file) && !(receive_feof)() && message_ended != END_DOT) { if (smtp_input) { - message_ended = chunking_state > CHUNKING_OFFERED - ? read_message_bdat_smtp(data_file) - : read_message_data_smtp(data_file); + message_ended = chunking_state <= CHUNKING_OFFERED + ? read_message_data_smtp(data_file) + : spool_wireformat + ? read_message_bdat_smtp_wire(data_file) + : read_message_bdat_smtp(data_file); receive_linecount++; /* The terminating "." line */ } else message_ended = read_message_data(data_file); @@ -4258,6 +4300,7 @@ if (smtp_input) else if (chunking_state > CHUNKING_OFFERED) { +/*XXX rethink for spool_wireformat */ smtp_printf("250- %u byte chunk, total %d\r\n250 OK id=%s\r\n", chunking_datasize, message_size+message_linecount, message_id); chunking_state = CHUNKING_OFFERED; diff --git a/src/src/spam.c b/src/src/spam.c index 477ab62b4..49776a30d 100644 --- a/src/src/spam.c +++ b/src/src/spam.c @@ -265,9 +265,9 @@ if (spam_ok && Ustrcmp(prev_user_name, user_name) == 0) return override ? OK : spam_rc; /* make sure the eml mbox file is spooled up */ + if (!(mbox_file = spool_mbox(&mbox_size, NULL, NULL))) - { - /* error while spooling */ + { /* error while spooling */ log_write(0, LOG_MAIN|LOG_PANIC, "%s error while creating mbox spool file", loglabel); return DEFER; diff --git a/src/src/spool_in.c b/src/src/spool_in.c index 6ed566411..0bdf92e3b 100644 --- a/src/src/spool_in.c +++ b/src/src/spool_in.c @@ -284,6 +284,9 @@ sender_ident = NULL; sender_local = FALSE; sender_set_untrusted = FALSE; smtp_active_hostname = primary_hostname; +#ifndef COMPILE_UTILITY +spool_file_wireformat = FALSE; +#endif tree_nonrecipients = NULL; #ifdef EXPERIMENTAL_BRIGHTMAIL @@ -603,6 +606,10 @@ for (;;) else if (Ustrncmp(p, "pam_score_int ", 14) == 0) spam_score_int = string_copy(big_buffer + 16); #endif +#ifndef COMPILE_UTILITY + else if (Ustrncmp(p, "pool_file_wireformat", 20) == 0) + spool_file_wireformat = TRUE; +#endif #if defined(SUPPORT_I18N) && !defined(COMPILE_UTILITY) else if (Ustrncmp(p, "mtputf8", 7) == 0) message_smtputf8 = TRUE; diff --git a/src/src/spool_mbox.c b/src/src/spool_mbox.c index 89bdb7ddc..8ca468a85 100644 --- a/src/src/spool_mbox.c +++ b/src/src/spool_mbox.c @@ -62,8 +62,8 @@ if (!spool_mbox_ok) } /* open [message_id].eml file for writing */ - mbox_file = modefopen(mbox_path, "wb", SPOOL_MODE); - if (mbox_file == NULL) + + if (!(mbox_file = modefopen(mbox_path, "wb", SPOOL_MODE))) { log_write(0, LOG_MAIN|LOG_PANIC, "%s", string_open_failed(errno, "scan file %s", mbox_path)); @@ -80,33 +80,25 @@ if (!spool_mbox_ok) "${if def:sender_address{X-Envelope-From: <${sender_address}>\n}}" "${if def:recipients{X-Envelope-To: ${recipients}\n}}"); - if (temp_string != NULL) - { - i = fwrite(temp_string, Ustrlen(temp_string), 1, mbox_file); - if (i != 1) + if (temp_string) + if (fwrite(temp_string, Ustrlen(temp_string), 1, mbox_file) != 1) { log_write(0, LOG_MAIN|LOG_PANIC, "Error/short write while writing \ mailbox headers to %s", mbox_path); goto OUT; } - } - /* write all header lines to mbox file */ - my_headerlist = header_list; - for (my_headerlist = header_list; my_headerlist != NULL; - my_headerlist = my_headerlist->next) - { - /* skip deleted headers */ - if (my_headerlist->type == '*') continue; + /* write all non-deleted header lines to mbox file */ - i = fwrite(my_headerlist->text, my_headerlist->slen, 1, mbox_file); - if (i != 1) - { - log_write(0, LOG_MAIN|LOG_PANIC, "Error/short write while writing \ - message headers to %s", mbox_path); - goto OUT; - } - } + for (my_headerlist = header_list; my_headerlist; + my_headerlist = my_headerlist->next) + if (my_headerlist->type != '*') + if (fwrite(my_headerlist->text, my_headerlist->slen, 1, mbox_file) != 1) + { + log_write(0, LOG_MAIN|LOG_PANIC, "Error/short write while writing \ + message headers to %s", mbox_path); + goto OUT; + } /* End headers */ if (fwrite("\n", 1, 1, mbox_file) != 1) @@ -151,18 +143,32 @@ if (!spool_mbox_ok) do { - j = fread(buffer, 1, sizeof(buffer), data_file); + uschar * s; + + if (!spool_file_wireformat || source_file_override) + j = fread(buffer, 1, sizeof(buffer), data_file); + else /* needs CRLF -> NL */ + if ((s = fgets(buffer, sizeof(buffer), data_file))) + { + uschar * p = s + Ustrlen(s) - 1; + + if (*p == '\n' && p[-1] == '\r') + *--p = '\n'; + else if (*p == '\r') + ungetc(*p--, data_file); + + j = p - buffer; + } + else + j = 0; if (j > 0) - { - i = fwrite(buffer, j, 1, mbox_file); - if (i != 1) + if (fwrite(buffer, j, 1, mbox_file) != 1) { log_write(0, LOG_MAIN|LOG_PANIC, "Error/short write while writing \ message body to %s", mbox_path); goto OUT; } - } } while (j > 0); (void)fclose(mbox_file); diff --git a/src/src/spool_out.c b/src/src/spool_out.c index 652506fb3..ebe089d4f 100644 --- a/src/src/spool_out.c +++ b/src/src/spool_out.c @@ -197,7 +197,10 @@ tree_walk(acl_var_m, &acl_var_write, f); /* Now any other data that needs to be remembered. */ -fprintf(f, "-body_linecount %d\n", body_linecount); +if (spool_file_wireformat) + fprintf(f, "-spool_file_wireformat\n"); +else + fprintf(f, "-body_linecount %d\n", body_linecount); fprintf(f, "-max_received_linelength %d\n", max_received_linelength); if (body_zerocount > 0) fprintf(f, "-body_zerocount %d\n", body_zerocount); diff --git a/src/src/transport.c b/src/src/transport.c index 0f20efe1b..7806e3957 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -11,6 +11,10 @@ transports. */ #include "exim.h" +#ifdef HAVE_LINUX_SENDFILE +# include +#endif + /* Structure for keeping list of addresses that have been added to Envelope-To:, in order to avoid duplication. */ @@ -483,13 +487,22 @@ for (ptr = start; ptr < end; ptr++) chunk_ptr = deliver_out_buffer; } + /* Remove CR before NL if required */ + + if ( *ptr == '\r' && ptr[1] == '\n' + && (!tctx || !(tctx->options & topt_use_crlf)) + && spool_file_wireformat + ) + ptr++; + if ((ch = *ptr) == '\n') { int left = end - ptr - 1; /* count of chars left after NL */ /* Insert CR before NL if required */ - if (tctx && tctx->options & topt_use_crlf) *chunk_ptr++ = '\r'; + if (tctx && tctx->options & topt_use_crlf && !spool_file_wireformat) + *chunk_ptr++ = '\r'; *chunk_ptr++ = '\n'; transport_newlines++; @@ -749,9 +762,7 @@ for (h = header_list; h; h = h->next) if (h->type != htype_old) /* Header removed */ else - { DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text); - } } /* Add on any address-specific headers. If there are multiple addresses, @@ -890,7 +901,7 @@ Returns: TRUE on success; FALSE (with errno) on failure. BOOL internal_transport_write_message(transport_ctx * tctx, int size_limit) { -int len; +int len, size = 0; /* Initialize pointer in output buffer. */ @@ -906,17 +917,21 @@ if (tctx->check_string && tctx->escape_string) nl_escape_length = Ustrlen(nl_escape); } +/* Whether the escaping mechanism is applied to headers or not is controlled by +an option (set for SMTP, not otherwise). Negate the length if not wanted till +after the headers. */ + +if (!(tctx->options & topt_escape_headers)) + nl_check_length = -nl_check_length; + /* Write the headers if required, including any that have to be added. If there -are header rewriting rules, apply them. */ +are header rewriting rules, apply them. The datasource is not the -D spoolfile +so temporarily hide the global that adjusts for its format. */ if (!(tctx->options & topt_no_headers)) { - /* Whether the escaping mechanism is applied to headers or not is controlled by - an option (set for SMTP, not otherwise). Negate the length if not wanted till - after the headers. */ - - if (!(tctx->options & topt_escape_headers)) - nl_check_length = -nl_check_length; + BOOL save_wireformat = spool_file_wireformat; + spool_file_wireformat = FALSE; /* Add return-path: if requested. */ @@ -925,7 +940,7 @@ if (!(tctx->options & topt_no_headers)) uschar buffer[ADDRESS_MAXLENGTH + 20]; int n = sprintf(CS buffer, "Return-path: <%.*s>\n", ADDRESS_MAXLENGTH, return_path); - if (!write_chunk(tctx, buffer, n)) return FALSE; + if (!write_chunk(tctx, buffer, n)) goto bad; } /* Add envelope-to: if requested */ @@ -938,19 +953,18 @@ if (!(tctx->options & topt_no_headers)) struct aci *dlist = NULL; void *reset_point = store_get(0); - if (!write_chunk(tctx, US"Envelope-to: ", 13)) return FALSE; + if (!write_chunk(tctx, US"Envelope-to: ", 13)) goto bad; /* Pick up from all the addresses. The plist and dlist variables are anchors for lists of addresses already handled; they have to be defined at this level because write_env_to() calls itself recursively. */ for (p = tctx->addr; p; p = p->next) - if (!write_env_to(p, &plist, &dlist, &first, tctx)) - return FALSE; + if (!write_env_to(p, &plist, &dlist, &first, tctx)) goto bad; /* Add a final newline and reset the store used for tracking duplicates */ - if (!write_chunk(tctx, US"\n", 1)) return FALSE; + if (!write_chunk(tctx, US"\n", 1)) goto bad; store_reset(reset_point); } @@ -960,7 +974,7 @@ if (!(tctx->options & topt_no_headers)) { uschar buffer[100]; int n = sprintf(CS buffer, "Delivery-date: %s\n", tod_stamp(tod_full)); - if (!write_chunk(tctx, buffer, n)) return FALSE; + if (!write_chunk(tctx, buffer, n)) goto bad; } /* Then the message's headers. Don't write any that are flagged as "old"; @@ -970,7 +984,13 @@ if (!(tctx->options & topt_no_headers)) addr is not NULL. */ if (!transport_headers_send(tctx, &write_chunk)) + { +bad: + spool_file_wireformat = save_wireformat; return FALSE; + } + + spool_file_wireformat = save_wireformat; } /* When doing RFC3030 CHUNKING output, work out how much data would be in a @@ -988,7 +1008,7 @@ suboptimal. */ if (tctx->options & topt_use_bdat) { off_t fsize; - int hsize, size = 0; + int hsize; if ((hsize = chunk_ptr - deliver_out_buffer) < 0) hsize = 0; @@ -999,7 +1019,7 @@ if (tctx->options & topt_use_bdat) if (size_limit > 0 && fsize > size_limit) fsize = size_limit; size = hsize + fsize; - if (tctx->options & topt_use_crlf) + if (tctx->options & topt_use_crlf && !spool_file_wireformat) size += body_linecount; /* account for CRLF-expansion */ /* With topt_use_bdat we never do dot-stuffing; no need to @@ -1039,6 +1059,52 @@ negative in cases where it isn't to apply to the headers). Then ensure the body is positioned at the start of its file (following the message id), then write it, applying the size limit if required. */ +/* If we have a wireformat -D file (CRNL lines, non-dotstuffed, no ending dot) +and we want to send a body without dotstuffing or ending-dot, in-clear, +then we can just dump it using sendfile. +This should get used for CHUNKING output and also for writing the -K file for +dkim signing, when we had CHUNKING input. */ + +#ifdef HAVE_LINUX_SENDFILE +if ( spool_file_wireformat + && !(tctx->options & (topt_no_body | topt_end_dot)) + && !nl_check_length + && tls_out.active != tctx->u.fd + ) + { + ssize_t copied = 0; + off_t offset = SPOOL_DATA_START_OFFSET; + + /* Write out any header data in the buffer */ + + if ((len = chunk_ptr - deliver_out_buffer) > 0) + { + if (!transport_write_block(tctx, deliver_out_buffer, len, TRUE)) + return FALSE; + size -= len; + } + + DEBUG(D_transport) debug_printf("using sendfile for body\n"); + + while(size > 0) + { + if ((copied = sendfile(tctx->u.fd, deliver_datafile, &offset, size)) <= 0) break; + size -= copied; + } + return copied >= 0; + } +#else +DEBUG(D_transport) debug_printf("cannot use sendfile for body: no support\n"); +#endif + +DEBUG(D_transport) + if (!(tctx->options & topt_no_body)) + debug_printf("cannot use sendfile for body: %s\n", + !spool_file_wireformat ? "spoolfile not wireformat" + : tctx->options & topt_end_dot ? "terminating dot wanted" + : nl_check_length ? "dot- or From-stuffing wanted" + : "TLS output wanted"); + if (!(tctx->options & topt_no_body)) { int size = size_limit; @@ -1077,6 +1143,7 @@ return (len = chunk_ptr - deliver_out_buffer) <= 0 || + /************************************************* * External interface to write the message * *************************************************/ @@ -1098,6 +1165,7 @@ BOOL transport_write_message(transport_ctx * tctx, int size_limit) { BOOL last_filter_was_NL = TRUE; +BOOL save_spool_file_wireformat = spool_file_wireformat; int rc, len, yield, fd_read, fd_write, save_errno; int pfd[2] = {-1, -1}; pid_t filter_pid, write_pid; @@ -1215,8 +1283,10 @@ DEBUG(D_transport) debug_printf("copying from the filter\n"); /* Copy the output of the filter, remembering if the last character was NL. If no data is returned, that counts as "ended with NL" (default setting of the -variable is TRUE). */ +variable is TRUE). The output should always be unix-format as we converted +any wireformat source on writing input to the filter. */ +spool_file_wireformat = FALSE; chunk_ptr = deliver_out_buffer; for (;;) @@ -1256,6 +1326,7 @@ there has been an error, kill the processes before waiting for them, just to be sure. Also apply a paranoia timeout. */ TIDY_UP: +spool_file_wireformat = save_spool_file_wireformat; save_errno = errno; (void)close(fd_read); diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index e28a5bfe6..ecba054a2 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -2254,6 +2254,9 @@ included in the count.) */ if (sx->peer_offered & PEER_OFFERED_SIZE) { +/*XXX problem here under spool_files_wireformat? +Or just forget about lines? Or inflate by a fixed proportion? */ + sprintf(CS p, " SIZE=%d", message_size+message_linecount+sx->ob->size_addition); while (*p) p++; } diff --git a/src/src/verify.c b/src/src/verify.c index 706d42a0f..6a50af506 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -1344,6 +1344,7 @@ tctx.tblock = cutthrough.addr.transport; tctx.addr = &cutthrough.addr; tctx.check_string = US"."; tctx.escape_string = US".."; +/*XXX check under spool_files_wireformat. Might be irrelevant */ tctx.options = topt_use_crlf; if (!transport_headers_send(&tctx, &cutthrough_write_chunk)) diff --git a/test/confs/0906 b/test/confs/0906 new file mode 100644 index 000000000..2e8d35a91 --- /dev/null +++ b/test/confs/0906 @@ -0,0 +1,89 @@ +# Exim test configuration 0906 +SERVER= + +exim_path = EXIM_PATH +keep_environment = +host_lookup_order = bydns +spool_directory = DIR/spool +log_file_path = DIR/spool/log/SERVER%slog +gecos_pattern = "" +gecos_name = CALLER_NAME +chunking_advertise_hosts = * +tls_advertise_hosts = ${if eq {SRV}{tls} {*}} + +# ----- Main settings ----- + +spool_wireformat = true + +primary_hostname = testhost.test.ex +domainlist local_domains = @ : test.ex + +acl_smtp_rcpt = accept +log_selector = +received_recipients + +.ifdef _OPT_MAIN_TLS_CERTIFICATE +tls_certificate = DIR/aux-fixed/cert1 +tls_privatekey = DIR/aux-fixed/cert1 +.endif + +# ----- ACL ----- + +# ----- Routers ----- + +begin routers + +to_server: + driver = accept + condition = ${if = {$received_port}{PORT_S}} + transport = remote_smtp${if eq {OPT}{dkim} {_dkim}} + errors_to = "" + +fail_remote_domains: + driver = redirect + domains = ! +local_domains + data = :fail: unrouteable mail domain "$domain" + +localuser: + driver = accept + transport = local_delivery + + +# ----- Transports ----- + +begin transports + +local_delivery: + driver = appendfile + file = DIR/test-mail/$local_part + headers_add = "X-body-linecount: $body_linecount\n\ + X-message-linecount: $message_linecount\n\ + X-received-count: $received_count" + return_path_add + user = CALLER + +remote_smtp: + driver = smtp + hosts = 127.0.0.1 + port = PORT_D + allow_localhost + +remote_smtp_dkim: + driver = smtp + hosts = 127.0.0.1 + port = PORT_D + allow_localhost + +.ifdef OPT + dkim_domain = test.ex + dkim_selector = sel + dkim_private_key = DIR/aux-fixed/dkim/dkim.private +.ifndef HEADERS_MAXSIZE + dkim_sign_headers = LIST +.endif +.endif + +# ----- Retry ----- + +begin retry +* * F,30m,5m; +# End diff --git a/test/confs/4530 b/test/confs/4530 new file mode 120000 index 000000000..a8ce02c97 --- /dev/null +++ b/test/confs/4530 @@ -0,0 +1 @@ +0906 \ No newline at end of file diff --git a/test/log/0906 b/test/log/0906 new file mode 100644 index 000000000..e6a99e12d --- /dev/null +++ b/test/log/0906 @@ -0,0 +1,9 @@ + +******** SERVER ******** +1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225 port 1224 +1999-03-02 09:44:33 10HmaX-0005vi-00 <= sender@dom H=(test.com) [127.0.0.1] P=esmtp K S=sss for a@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for a@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 => a R=localuser T=local_delivery +1999-03-02 09:44:33 10HmaY-0005vi-00 Completed +1999-03-02 09:44:33 10HmaX-0005vi-00 => a@test.ex R=to_server T=remote_smtp H=127.0.0.1 [127.0.0.1] K C="250- 8392 byte chunk, total 8599\\n250 OK id=10HmaY-0005vi-00" +1999-03-02 09:44:33 10HmaX-0005vi-00 Completed diff --git a/test/log/4530 b/test/log/4530 new file mode 100644 index 000000000..d360cb102 --- /dev/null +++ b/test/log/4530 @@ -0,0 +1,17 @@ + +******** SERVER ******** +1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1224 port 1225 +1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@bloggs.com H=(xxx) [127.0.0.1] P=esmtp S=sss for z@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 DKIM: d=test.ex s=sel c=relaxed/relaxed a=rsa-sha256 b=1024 [verification succeeded] +1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for z@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 => z R=localuser T=local_delivery +1999-03-02 09:44:33 10HmaY-0005vi-00 Completed +1999-03-02 09:44:33 10HmaX-0005vi-00 => z@test.ex R=to_server T=remote_smtp_dkim H=127.0.0.1 [127.0.0.1] K C="250- 652 byte chunk, total 652\\n250 OK id=10HmaY-0005vi-00" +1999-03-02 09:44:33 10HmaX-0005vi-00 Completed +1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@bloggs.com H=(xxx) [127.0.0.1] P=esmtp K S=sss for y@test.ex +1999-03-02 09:44:33 10HmbA-0005vi-00 DKIM: d=test.ex s=sel c=relaxed/relaxed a=rsa-sha256 b=1024 [verification succeeded] +1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for y@test.ex +1999-03-02 09:44:33 10HmbA-0005vi-00 => y R=localuser T=local_delivery +1999-03-02 09:44:33 10HmbA-0005vi-00 Completed +1999-03-02 09:44:33 10HmaZ-0005vi-00 => y@test.ex R=to_server T=remote_smtp_dkim H=127.0.0.1 [127.0.0.1] K C="250- 652 byte chunk, total 652\\n250 OK id=10HmbA-0005vi-00" +1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed diff --git a/test/mail/0906.a b/test/mail/0906.a new file mode 100644 index 000000000..e5a07fc2d --- /dev/null +++ b/test/mail/0906.a @@ -0,0 +1,109 @@ +From MAILER-DAEMON Tue Mar 02 09:44:33 1999 +Return-path: <> +Received: from localhost ([127.0.0.1] helo=testhost.test.ex) + by testhost.test.ex with esmtp (Exim x.yz) + id 10HmaY-0005vi-00 + for a@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 +Received: from [127.0.0.1] (helo=test.com) + by testhost.test.ex with esmtp (Exim x.yz) + (envelope-from ) + id 10HmaX-0005vi-00 + for a@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 +Subject: foo +X-body-linecount: 0 +X-message-linecount: 10 +X-received-count: 2 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +.dot +tail + diff --git a/test/mail/4530.y b/test/mail/4530.y new file mode 100644 index 000000000..580e21dc7 --- /dev/null +++ b/test/mail/4530.y @@ -0,0 +1,26 @@ +From MAILER-DAEMON Tue Mar 02 09:44:33 1999 +Return-path: <> +Received: from localhost ([127.0.0.1] helo=testhost.test.ex) + by testhost.test.ex with esmtp (Exim x.yz) + id 10HmbA-0005vi-00 + for y@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=test.ex; + s=sel; h=LIST; bh=CVpkzY75tV/NCKk5pPx4GnM3NX83xwCiT0xVwo0G1Rs=; b=TIqPqpKM5qf + ZFlv2H8yio5RybWA3sLCtVmE6HmBhBKqW+uqLKG2grqJhVMJ3qXnvQQ3ixnMjMlJqfCpEBtxfsSR9 + MGLPP9ZMdlrBNEL6XKlgE+X8bAra5zkuLZs8gy8H3/mtEfoKPs4ltB/ZK/j2FHG2+CEx+TDTIkh9E + wkAMrA=; +Received: from [127.0.0.1] (helo=xxx) + by testhost.test.ex with esmtp (Exim x.yz) + (envelope-from ) + id 10HmaZ-0005vi-00 + for y@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 +Subject: simple test +X-body-linecount: 0 +X-message-linecount: 15 +X-received-count: 2 + +Line 1: This is a simple test. +Line 2: This is a simple test. +.Line 3 has a leading dot +last line: 4 + diff --git a/test/mail/4530.z b/test/mail/4530.z new file mode 100644 index 000000000..2e74291f5 --- /dev/null +++ b/test/mail/4530.z @@ -0,0 +1,26 @@ +From MAILER-DAEMON Tue Mar 02 09:44:33 1999 +Return-path: <> +Received: from localhost ([127.0.0.1] helo=testhost.test.ex) + by testhost.test.ex with esmtp (Exim x.yz) + id 10HmaY-0005vi-00 + for z@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=test.ex; + s=sel; h=LIST; bh=CVpkzY75tV/NCKk5pPx4GnM3NX83xwCiT0xVwo0G1Rs=; b=TIqPqpKM5qf + ZFlv2H8yio5RybWA3sLCtVmE6HmBhBKqW+uqLKG2grqJhVMJ3qXnvQQ3ixnMjMlJqfCpEBtxfsSR9 + MGLPP9ZMdlrBNEL6XKlgE+X8bAra5zkuLZs8gy8H3/mtEfoKPs4ltB/ZK/j2FHG2+CEx+TDTIkh9E + wkAMrA=; +Received: from [127.0.0.1] (helo=xxx) + by testhost.test.ex with esmtp (Exim x.yz) + (envelope-from ) + id 10HmaX-0005vi-00 + for z@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 +Subject: simple test +X-body-linecount: 0 +X-message-linecount: 15 +X-received-count: 2 + +Line 1: This is a simple test. +Line 2: This is a simple test. +.Line 3 has a leading dot +last line: 4 + diff --git a/test/scripts/0000-Basic/0906 b/test/scripts/0000-Basic/0906 new file mode 100644 index 000000000..f80899fee --- /dev/null +++ b/test/scripts/0000-Basic/0906 @@ -0,0 +1,122 @@ +# CHUNKING, spool_wireformat +# +exim -bd -DSERVER=server -oX PORT_D:PORT_S +**** +# +# Basic long message +client 127.0.0.1 PORT_S +??? 220 +EHLO test.com +??? 250- +??? 250- +??? 250- +??? 250- +??? 250- +??? 250 +MAIL FROM: +??? 250 +RCPT TO: +??? 250 +BDAT 8408 LAST +Subject: foo + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +.dot +tail +??? 250- +??? 250 +QUIT +**** +# +sleep 1 +killdaemon +no_msglog_check diff --git a/test/scripts/4500-DKIM/4530 b/test/scripts/4500-DKIM/4530 new file mode 100644 index 000000000..822f2fe02 --- /dev/null +++ b/test/scripts/4500-DKIM/4530 @@ -0,0 +1,66 @@ +# DKIM, CHUNKING, wireformat-spoolfile +# +exim -bd -DSERVER=server -DOPT=dkim -oX PORT_S:PORT_D +**** +# +# 1: non-CHUNKING injection; will not be stored as wireformat therefore +# onward-send will not use sendfile. Should still be signed, and verify correctly. +client 127.0.0.1 PORT_S +??? 220 +EHLO xxx +??? 250- +??? 250- +??? 250- +??? 250- +??? 250- +??? 250 +MAIL FROM: +??? 250 +RCPT TO: +??? 250 +DATA +??? 354 +Subject: simple test + +Line 1: This is a simple test. +Line 2: This is a simple test. +..Line 3 has a leading dot +last line: 4 +. +??? 250 +QUIT +??? 221 +**** +sleep 1 +# +# 2: CHUNKING injection; should be stored as wireformat therefore +# onward-send should not use sendfile. Should still be signed, and verify correctly. +client 127.0.0.1 PORT_S +??? 220 +EHLO xxx +??? 250- +??? 250- +??? 250- +??? 250- +??? 250- +??? 250 +MAIL FROM: +??? 250 +RCPT TO: +??? 250 +BDAT 129 LAST +Subject: simple test + +Line 1: This is a simple test. +Line 2: This is a simple test. +.Line 3 has a leading dot +last line: 4 +??? 250- +??? 250 +QUIT +??? 221 +**** +sleep 1 +# +killdaemon +no_msglog_check diff --git a/test/stderr/0143 b/test/stderr/0143 index 3de426629..b1d0ff9b6 100644 --- a/test/stderr/0143 +++ b/test/stderr/0143 @@ -38,6 +38,7 @@ cmd buf flush ddd bytes cmd buf flush ddd bytes SMTP<< 354 Send data SMTP>> writing message and terminating "." +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=300 SMTP<< 250 OK ok=1 send_quit=1 send_rset=0 continue_more=0 yield=0 first_address is NULL diff --git a/test/stderr/0169 b/test/stderr/0169 index 70f11bf4f..30ed85f8d 100644 --- a/test/stderr/0169 +++ b/test/stderr/0169 @@ -24,6 +24,7 @@ writing to file TESTSUITE/test-mail/userx Exim quota = 52428800 old size = sssss this message = sss (included) file count quota = 0 count = 0 writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 quota = 52428800 threshold = 21495808 old size = sssss message size = sss diff --git a/test/stderr/0275 b/test/stderr/0275 index 3ec0da1d4..5f76af09a 100644 --- a/test/stderr/0275 +++ b/test/stderr/0275 @@ -163,7 +163,6 @@ LOG: MAIN <= CALLER@test.ex U=CALLER P=local S=sss created log directory TESTSUITE/spool/log search_tidyup called -release cutthrough conn: msg passed for delivery exec TESTSUITE/eximdir/exim -DEXIM_PATH=TESTSUITE/eximdir/exim -C TESTSUITE/test-config -d=0xfbb95cfd -odi -Mc 10HmaX-0005vi-00 Exim version x.yz .... changed uid/gid: forcing real = effective diff --git a/test/stderr/0386 b/test/stderr/0386 index 44e856da2..6fb8a5536 100644 --- a/test/stderr/0386 +++ b/test/stderr/0386 @@ -310,6 +310,7 @@ lock file created mailbox TESTSUITE/test-mail/2 is locked writing to file TESTSUITE/test-mail/2 writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd @@ -490,6 +491,7 @@ lock file created mailbox TESTSUITE/test-mail/2 is locked writing to file TESTSUITE/test-mail/2 writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd diff --git a/test/stderr/0393 b/test/stderr/0393 index dd0887f09..7fb82e35c 100644 --- a/test/stderr/0393 +++ b/test/stderr/0393 @@ -28,6 +28,7 @@ mailbox TESTSUITE/test-mail/userx is locked writing to file TESTSUITE/test-mail/userx writing data block fd=dddd size=sss timeout=0 process pppp running as transport filter: fd_write=dddd fd_read=dddd +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 process pppp writing to transport filter copying from the filter @@ -70,6 +71,7 @@ lock file created mailbox TESTSUITE/test-mail/userx is locked writing to file TESTSUITE/test-mail/userx writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd @@ -108,6 +110,7 @@ lock file created mailbox TESTSUITE/test-mail/userx is locked writing to file TESTSUITE/test-mail/userx writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd diff --git a/test/stderr/0402 b/test/stderr/0402 index a007de31c..7f301651b 100644 --- a/test/stderr/0402 +++ b/test/stderr/0402 @@ -487,6 +487,7 @@ writing to file TESTSUITE/test-mail/junk └─────result: From CALLER@test.ex Tue Mar 02 09:44:33 1999 writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd @@ -542,6 +543,7 @@ writing to file TESTSUITE/test-mail/junk └─────result: From CALLER@test.ex Tue Mar 02 09:44:33 1999 writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd diff --git a/test/stderr/0404 b/test/stderr/0404 index d3d8f4c64..3b7605ba7 100644 --- a/test/stderr/0404 +++ b/test/stderr/0404 @@ -18004,6 +18004,7 @@ writing to file TESTSUITE/test-mail/sender writing data block fd=dddd size=sss timeout=0 flushing headers buffer writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd diff --git a/test/stderr/0408 b/test/stderr/0408 index f3d6d6b1a..9cdb41d96 100644 --- a/test/stderr/0408 +++ b/test/stderr/0408 @@ -159,6 +159,7 @@ lock file created mailbox TESTSUITE/test-mail/userx is locked writing to file TESTSUITE/test-mail/userx writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd diff --git a/test/stderr/0476 b/test/stderr/0476 index 59ea69f58..675c6239c 100644 --- a/test/stderr/0476 +++ b/test/stderr/0476 @@ -52,6 +52,7 @@ set_process_info: pppp delivering 10HmaX-0005vi-00 LOG: MAIN ** userx@test.ex R=r1 T=t1 H=127.0.0.1 [127.0.0.1]: SMTP error from remote mail server after RCPT TO:: 550 NO set_process_info: pppp tidying up after delivering 10HmaX-0005vi-00 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 Exim version x.yz .... configuration file is TESTSUITE/test-config diff --git a/test/stderr/0487 b/test/stderr/0487 index 7319b67dd..70da61fff 100644 --- a/test/stderr/0487 +++ b/test/stderr/0487 @@ -163,6 +163,7 @@ writing to file TESTSUITE/test-mail/userx writing data block fd=dddd size=sss timeout=0 rewrite_one_header: type=F: From: unqualified +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd diff --git a/test/stderr/0512 b/test/stderr/0512 index 1cdf5f013..07025d5d7 100644 --- a/test/stderr/0512 +++ b/test/stderr/0512 @@ -59,6 +59,7 @@ LOG: MAIN == userx@myhost.test.ex R=r1 T=t1 defer (dd): Connection refused LOG: MAIN ** userx@myhost.test.ex: retry timeout exceeded +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 Exim version x.yz .... configuration file is TESTSUITE/test-config @@ -122,6 +123,7 @@ LOG: MAIN == userx@myhost.test.ex R=r1 T=t1 defer (dd): Connection refused LOG: MAIN ** userx@myhost.test.ex: retry timeout exceeded +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 Exim version x.yz .... configuration file is TESTSUITE/test-config diff --git a/test/stderr/2600 b/test/stderr/2600 index 4e9974733..599be797a 100644 --- a/test/stderr/2600 +++ b/test/stderr/2600 @@ -439,6 +439,7 @@ lock file created mailbox TESTSUITE/test-mail/userx is locked writing to file TESTSUITE/test-mail/userx writing data block fd=dddd size=sss timeout=0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 writing data block fd=dddd size=sss timeout=0 appendfile yields 0 with errno=dd more_errno=dd diff --git a/test/stderr/4520 b/test/stderr/4520 index c2a856bb1..c1bafcdcc 100644 --- a/test/stderr/4520 +++ b/test/stderr/4520 @@ -23,18 +23,18 @@ cmd buf flush ddd bytes SMTP<< 250 OK SMTP<< 250 Accepted SMTP<< 354 Enter message, ending with "." on a line by itself -PDKIM (checking verify key)<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< +PDKIM (checking verify key)>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> PDKIM >> Parsing public key record >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Raw record: v=DKIM1\;{SP}p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXRFf+VhT+lCgFhhSkinZKcFNeRzjYdW8vT29Rbb3NadvTFwAd+cVLPFwZL8H5tUD/7JbUPqNTCPxmpgIL+V5T4tEZMorHatvvUM2qfcpQ45IfsZ+YdhbIiAslHCpy4xNxIR3zylgqRUF4+Dtsaqy3a5LhwMiKCLrnzhXk1F1hxwIDAQAB v=DKIM1\ p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXRFf+VhT+lCgFhhSkinZKcFNeRzjYdW8vT29Rbb3NadvTFwAd+cVLPFwZL8H5tUD/7JbUPqNTCPxmpgIL+V5T4tEZMorHatvvUM2qfcpQ45IfsZ+YdhbIiAslHCpy4xNxIR3zylgqRUF4+Dtsaqy3a5LhwMiKCLrnzhXk1F1hxwIDAQAB Error while parsing public key record WARNING: bad dkim key in dns -PDKIM (finished checking verify key)<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< +PDKIM (finished checking verify key)<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< content{CR}{LF} PDKIM [test.ex] Body bytes hashed: 9 PDKIM [test.ex] Body hash computed: fc06f48221d98ad6106c3845b33a2a41152482ab9e697f736ad26db4853fa657 -PDKIM >> Header data for hash, canonicalized, in sequence >>>>>>>>>>>>>> +PDKIM >> Header data for hash, canonicalized, in sequence >>>>>>>>>>>> sender:CALLER_NAME{SP}{CR}{LF} message-id:{CR}{LF} from:nobody@example.com{CR}{LF} diff --git a/test/stderr/5004 b/test/stderr/5004 index ad5c9b331..bedf5be2e 100644 --- a/test/stderr/5004 +++ b/test/stderr/5004 @@ -155,6 +155,7 @@ created directory TESTSUITE/test-mail/new created directory TESTSUITE/test-mail/cur delivering in maildir format in TESTSUITE/test-mail writing to tmp/MAILDIR.mail.test.ex +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 renaming temporary file renamed tmp/MAILDIR.mail.test.ex as new/MAILDIR.mail.test.ex diff --git a/test/stderr/5005 b/test/stderr/5005 index 8b28b7055..a68bc11c9 100644 --- a/test/stderr/5005 +++ b/test/stderr/5005 @@ -149,6 +149,7 @@ delivering in maildir format in TESTSUITE/test-mail/nofile writing to tmp/MAILDIR.myhost.test.ex Exim quota = 500 old size = sssss this message = sss (included) file count quota = 0 count = 0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 added 'ddd 1' to maildirsize file renaming temporary file @@ -338,6 +339,7 @@ delivering in maildir format in TESTSUITE/test-mail/userx writing to tmp/MAILDIR.myhost.test.ex Exim quota = 500 old size = sssss this message = sss (included) file count quota = 0 count = 0 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 added 'ddd 1' to maildirsize file renaming temporary file diff --git a/test/stderr/5006 b/test/stderr/5006 index dce0d0050..771f5fd56 100644 --- a/test/stderr/5006 +++ b/test/stderr/5006 @@ -147,6 +147,7 @@ maildir_compute_size (timestamp_only): ddddddd returning maildir size=sss filecount=0 delivering in maildir format in TESTSUITE/test-mail/userx writing to tmp/MAILDIR.myhost.test.ex +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 added 'ddd 1' to maildirsize file renaming temporary file diff --git a/test/stderr/5008 b/test/stderr/5008 index fd4ae87ec..b3bf8f785 100644 --- a/test/stderr/5008 +++ b/test/stderr/5008 @@ -25,6 +25,7 @@ delivering in maildir format in TESTSUITE/test-mail/userx writing to tmp/MAILDIR.myhost.test.ex Exim quota = 1048576 old size = sssss this message = sss (included) file count quota = 0 count = -1 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 renaming temporary file renamed tmp/MAILDIR.myhost.test.ex as new/MAILDIR.myhost.test.ex @@ -64,6 +65,7 @@ delivering in maildir format in TESTSUITE/test-mail/userx writing to tmp/MAILDIR.myhost.test.ex Exim quota = 1048576 old size = sssss this message = sss (included) file count quota = 20 count = 4 +cannot use sendfile for body: spoolfile not wireformat writing data block fd=dddd size=sss timeout=0 renaming temporary file renamed tmp/MAILDIR.myhost.test.ex as new/MAILDIR.myhost.test.ex diff --git a/test/stdout/0906 b/test/stdout/0906 new file mode 100644 index 000000000..57be5fe3d --- /dev/null +++ b/test/stdout/0906 @@ -0,0 +1,123 @@ +Connecting to 127.0.0.1 port 1224 ... connected +??? 220 +<<< 220 testhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +>>> EHLO test.com +??? 250- +<<< 250-testhost.test.ex Hello test.com [127.0.0.1] +??? 250- +<<< 250-SIZE 52428800 +??? 250- +<<< 250-8BITMIME +??? 250- +<<< 250-PIPELINING +??? 250- +<<< 250-CHUNKING +??? 250 +<<< 250 HELP +>>> MAIL FROM: +??? 250 +<<< 250 OK +>>> RCPT TO: +??? 250 +<<< 250 Accepted +>>> BDAT 8408 LAST +>>> Subject: foo +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +>>> .dot +>>> tail +??? 250- +<<< 250- 8408 byte chunk, total 8408 +??? 250 +<<< 250 OK id=10HmaX-0005vi-00 +>>> QUIT +End of script diff --git a/test/stdout/4530 b/test/stdout/4530 new file mode 100644 index 000000000..a1f5cb471 --- /dev/null +++ b/test/stdout/4530 @@ -0,0 +1,75 @@ +Connecting to 127.0.0.1 port 1224 ... connected +??? 220 +<<< 220 testhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +>>> EHLO xxx +??? 250- +<<< 250-testhost.test.ex Hello xxx [127.0.0.1] +??? 250- +<<< 250-SIZE 52428800 +??? 250- +<<< 250-8BITMIME +??? 250- +<<< 250-PIPELINING +??? 250- +<<< 250-CHUNKING +??? 250 +<<< 250 HELP +>>> MAIL FROM: +??? 250 +<<< 250 OK +>>> RCPT TO: +??? 250 +<<< 250 Accepted +>>> DATA +??? 354 +<<< 354 Enter message, ending with "." on a line by itself +>>> Subject: simple test +>>> +>>> Line 1: This is a simple test. +>>> Line 2: This is a simple test. +>>> ..Line 3 has a leading dot +>>> last line: 4 +>>> . +??? 250 +<<< 250 OK id=10HmaX-0005vi-00 +>>> QUIT +??? 221 +<<< 221 testhost.test.ex closing connection +End of script +Connecting to 127.0.0.1 port 1224 ... connected +??? 220 +<<< 220 testhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +>>> EHLO xxx +??? 250- +<<< 250-testhost.test.ex Hello xxx [127.0.0.1] +??? 250- +<<< 250-SIZE 52428800 +??? 250- +<<< 250-8BITMIME +??? 250- +<<< 250-PIPELINING +??? 250- +<<< 250-CHUNKING +??? 250 +<<< 250 HELP +>>> MAIL FROM: +??? 250 +<<< 250 OK +>>> RCPT TO: +??? 250 +<<< 250 Accepted +>>> BDAT 129 LAST +>>> Subject: simple test +>>> +>>> Line 1: This is a simple test. +>>> Line 2: This is a simple test. +>>> .Line 3 has a leading dot +>>> last line: 4 +??? 250- +<<< 250- 129 byte chunk, total 129 +??? 250 +<<< 250 OK id=10HmaZ-0005vi-00 +>>> QUIT +??? 221 +<<< 221 testhost.test.ex closing connection +End of script -- cgit v1.2.3 From 40525d07a858c90293bc09188fb539a1cec3f8aa Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 6 May 2017 21:01:45 +0100 Subject: Teach SMTP input sync check ("input sent too soon") about SMTP input buffering --- doc/doc-txt/ChangeLog | 5 +++++ src/src/smtp_in.c | 13 +++++++++++-- test/log/0901 | 2 +- test/rejectlog/0901 | 2 +- test/scripts/0000-Basic/0901 | 8 ++++---- test/scripts/2000-GnuTLS/2031 | 4 ++-- test/scripts/5730-OCSP-GnuTLS-events/5730 | 4 ++-- test/stdout/0901 | 8 ++++---- 8 files changed, 30 insertions(+), 16 deletions(-) (limited to 'test/stdout') diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index a7b441e64..aca12ea00 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -73,6 +73,11 @@ JH/11 Bug 2104: Fix continued use of a transport connection with TLS. In the which naturally failed, giving a failed delivery and bloating the retry database. Investigation and fix prototype from Wolfgang Breyha. +JH/12 Fix check on SMTP command input synchronisation. Previously there were + false-negatives in the check that the sender had not preempted a response + or prompt from Exim (running as a server), due to that code's lack of + awareness of the SMTP input buferring. + Exim version 4.89 ----------------- diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index 01c12caf6..8832908f3 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -343,6 +343,9 @@ if (!smtp_enforce_sync || sender_host_address == NULL || sender_host_notsocket || tls_in.active >= 0) return TRUE; +if (smtp_inptr < smtp_inend) + return FALSE; + fd = fileno(smtp_in); FD_ZERO(&fds); FD_SET(fd, &fds); @@ -532,12 +535,15 @@ for(;;) if (!pipelining_advertised && !check_sync()) { + unsigned n = smtp_inend - smtp_inptr; + if (n > 32) n = 32; + incomplete_transaction_log(US"sync failure"); log_write(0, LOG_MAIN|LOG_REJECT, "SMTP protocol synchronization error " "(next input sent too soon: pipelining was not advertised): " "rejected \"%s\" %s next input=\"%s\"", smtp_cmd_buffer, host_and_ident(TRUE), - string_printing(smtp_inptr)); + string_printing(string_copyn(smtp_inptr, n))); (void) synprot_error(L_smtp_protocol_error, 554, NULL, US"SMTP synchronization error"); goto repeat_until_rset; @@ -2863,10 +2869,13 @@ this synchronisation check is disabled. */ if (!check_sync()) { + unsigned n = smtp_inend - smtp_inptr; + if (n > 32) n = 32; + log_write(0, LOG_MAIN|LOG_REJECT, "SMTP protocol " "synchronization error (input sent without waiting for greeting): " "rejected connection from %s input=\"%s\"", host_and_ident(TRUE), - string_printing(smtp_inptr)); + string_printing(string_copyn(smtp_inptr, n))); smtp_printf("554 SMTP synchronization error\r\n"); return FALSE; } diff --git a/test/log/0901 b/test/log/0901 index cd8c52e75..0297a8915 100644 --- a/test/log/0901 +++ b/test/log/0901 @@ -10,6 +10,6 @@ 1999-03-02 09:44:33 SMTP connection from (tester) [127.0.0.1] lost while reading message data 1999-03-02 09:44:33 SMTP connection from (tester) [127.0.0.1] lost while reading message data 1999-03-02 09:44:33 10HmbD-0005vi-00 <= someone8@some.domain H=(tester) [127.0.0.1] P=esmtp K S=sss for CALLER@test.ex -1999-03-02 09:44:33 SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "bdat 1" H=(tester) [127.0.0.1] next input="bdat 87 last\r\n" +1999-03-02 09:44:33 SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "BDAT 1" H=(tester) [127.0.0.1] next input="BDAT 87 last\r\n" 1999-03-02 09:44:33 SMTP call from (tester) [127.0.0.1] dropped: too many syntax or protocol errors (last command was "From: Sam@random.com") 1999-03-02 09:44:33 SMTP connection from (tester) [127.0.0.1] lost while reading message data (header) diff --git a/test/rejectlog/0901 b/test/rejectlog/0901 index a7f8f0692..f75d9d270 100644 --- a/test/rejectlog/0901 +++ b/test/rejectlog/0901 @@ -1,6 +1,6 @@ ******** SERVER ******** -1999-03-02 09:44:33 SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "bdat 1" H=(tester) [127.0.0.1] next input="bdat 87 last\r\n" +1999-03-02 09:44:33 SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "BDAT 1" H=(tester) [127.0.0.1] next input="BDAT 87 last\r\n" Envelope-from: Envelope-to: 1999-03-02 09:44:33 SMTP call from (tester) [127.0.0.1] dropped: too many syntax or protocol errors (last command was "From: Sam@random.com") diff --git a/test/scripts/0000-Basic/0901 b/test/scripts/0000-Basic/0901 index 9908d5ecd..5e88c5ae9 100644 --- a/test/scripts/0000-Basic/0901 +++ b/test/scripts/0000-Basic/0901 @@ -243,12 +243,12 @@ ehlo tester ??? 250-8BITMIME ??? 250-CHUNKING ??? 250 HELP -mail from:someone9@some.domain +MAIL FROM:someone9@some.domain ??? 250 -rcpt to:CALLER@test.ex +RCPT TO:CALLER@test.ex ??? 250 -bdat 1\r\nTbdat 87 last -To: Susan@random.com +BDAT 1\r\nTBDAT 87 last +o: Susan@random.com From: Sam@random.com Subject: This is a bodyless test message diff --git a/test/scripts/2000-GnuTLS/2031 b/test/scripts/2000-GnuTLS/2031 index 65b529093..76186b5e4 100644 --- a/test/scripts/2000-GnuTLS/2031 +++ b/test/scripts/2000-GnuTLS/2031 @@ -6,14 +6,14 @@ exim -DSERVER=server -bd -oX PORT_D exim CALLER@test.ex Test message. **** -millisleep 500 +millisleep 700 # # # Extended: server uses SNI to choose certificate exim abcd@test.ex Test message. **** -millisleep 500 +millisleep 700 # # killdaemon diff --git a/test/scripts/5730-OCSP-GnuTLS-events/5730 b/test/scripts/5730-OCSP-GnuTLS-events/5730 index d22a1aa1f..11c3a867f 100644 --- a/test/scripts/5730-OCSP-GnuTLS-events/5730 +++ b/test/scripts/5730-OCSP-GnuTLS-events/5730 @@ -21,7 +21,7 @@ exim -bd -oX PORT_D -DSERVER=server \ exim norequire@test.ex test message. **** -millisleep 500 +millisleep 700 # # # @@ -30,7 +30,7 @@ millisleep 500 exim nostaple@test.ex test message. **** -millisleep 500 +millisleep 700 # # # diff --git a/test/stdout/0901 b/test/stdout/0901 index 99eb81231..a982ac8b6 100644 --- a/test/stdout/0901 +++ b/test/stdout/0901 @@ -324,14 +324,14 @@ Connecting to 127.0.0.1 port 1225 ... connected <<< 250-CHUNKING ??? 250 HELP <<< 250 HELP ->>> mail from:someone9@some.domain +>>> MAIL FROM:someone9@some.domain ??? 250 <<< 250 OK ->>> rcpt to:CALLER@test.ex +>>> RCPT TO:CALLER@test.ex ??? 250 <<< 250 Accepted ->>> bdat 1\r\nTbdat 87 last ->>> To: Susan@random.com +>>> BDAT 1\r\nTBDAT 87 last +>>> o: Susan@random.com >>> From: Sam@random.com >>> Subject: This is a bodyless test message >>> -- cgit v1.2.3 From b7e4352c99fe3dee2af93f06ef0ac74ee355d5ea Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 7 May 2017 15:37:18 +0100 Subject: Testsuite: add DANE testcase for TLSA lookup SERVFAIL --- test/confs/5840 | 4 ++-- test/dnszones-src/db.test.ex | 4 ++++ test/log/5840 | 3 +++ test/scripts/5840-DANE-OpenSSL/5840 | 15 +++++++++++++-- test/stderr/5840 | 15 +++++++-------- test/stdout/5840 | 10 ++++++---- 6 files changed, 35 insertions(+), 16 deletions(-) (limited to 'test/stdout') diff --git a/test/confs/5840 b/test/confs/5840 index ac3578dc9..01c114252 100644 --- a/test/confs/5840 +++ b/test/confs/5840 @@ -61,10 +61,10 @@ begin transports send_to_server: driver = smtp allow_localhost - port = PORT_D + port = ${if match {$host}{\Ntest.ex$\N} {PORT_D}{25}} hosts_try_dane = * - hosts_require_dane = !thishost.test.ex + hosts_require_dane = HOSTIPV4 tls_verify_cert_hostnames = ${if eq {OPT}{no_certname} {}{*}} tls_try_verify_hosts = thishost.test.ex tls_verify_certificates = CDIR2/ca_chain.pem diff --git a/test/dnszones-src/db.test.ex b/test/dnszones-src/db.test.ex index 349fbd4d3..50bd6b073 100644 --- a/test/dnszones-src/db.test.ex +++ b/test/dnszones-src/db.test.ex @@ -465,6 +465,10 @@ DNSSEC _1225._tcp.danelazy2 CNAME test.again.dns. DNSSEC dane.no.1 A HOSTIPV4 DNSSEC dane.no.2 A 127.0.0.1 +; a broken dane config (or under attack) where the TLSA lookup fails (as opposed to there not being one) +DNSSEC danebroken1 A 127.0.0.1 +_1225._tcp.danebroken1 CNAME test.fail.dns. + ; ------- Testing delays ------------ DELAY=500 delay500 A HOSTIPV4 diff --git a/test/log/5840 b/test/log/5840 index 8d309e088..d02a4c7d7 100644 --- a/test/log/5840 +++ b/test/log/5840 @@ -26,6 +26,7 @@ 1999-03-02 09:44:33 10HmbH-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@mxdanelazy.test.ex 1999-03-02 09:44:33 10HmbI-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@dane.no.1.test.ex 1999-03-02 09:44:33 10HmbJ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@dane.no.2.test.ex +1999-03-02 09:44:33 10HmbK-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@danebroken1.test.ex 1999-03-02 09:44:33 Start queue run: pid=pppp -qf 1999-03-02 09:44:33 10HmbH-0005vi-00 H=danelazy.test.ex [ip4.ip4.ip4.ip4]: DANE error: tlsa lookup DEFER 1999-03-02 09:44:33 10HmbH-0005vi-00 H=danelazy2.test.ex [127.0.0.1]: DANE error: tlsa lookup DEFER @@ -35,6 +36,8 @@ 1999-03-02 09:44:33 10HmbI-0005vi-00 Completed 1999-03-02 09:44:33 10HmbJ-0005vi-00 H=dane.no.2.test.ex [127.0.0.1]: DANE error: tlsa lookup DEFER 1999-03-02 09:44:33 10HmbJ-0005vi-00 == CALLER@dane.no.2.test.ex R=client T=send_to_server defer (-36): DANE error: tlsa lookup DEFER +1999-03-02 09:44:33 10HmbK-0005vi-00 H=danebroken1.test.ex [127.0.0.1]: DANE error: tlsa lookup DEFER +1999-03-02 09:44:33 10HmbK-0005vi-00 == CALLER@danebroken1.test.ex R=client T=send_to_server defer (-36): DANE error: tlsa lookup DEFER 1999-03-02 09:44:33 End queue run: pid=pppp -qf ******** SERVER ******** diff --git a/test/scripts/5840-DANE-OpenSSL/5840 b/test/scripts/5840-DANE-OpenSSL/5840 index d1da54913..fdff36119 100644 --- a/test/scripts/5840-DANE-OpenSSL/5840 +++ b/test/scripts/5840-DANE-OpenSSL/5840 @@ -61,14 +61,25 @@ exim -DSERVER=server -DDETAILS=ee -bd -oX PORT_D exim -odq CALLER@mxdanelazy.test.ex Testing **** -### A server lacking a TLSA, required +### A server lacking a TLSA, dane required (should fail) exim -odq CALLER@dane.no.1.test.ex Testing **** -### A server lacking a TLSA, requested only +### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) exim -odq CALLER@dane.no.2.test.ex Testing **** +### A server where the A is dnssec and the TLSA _fails_ +exim -odq CALLER@danebroken1.test.ex +Testing +**** +# ### A server securely saying "no TLSA records here", dane required (should fail) +# exim -odq CALLER@dane.no.3.test.ex +# Testing +# ### A server securely saying "no TLSA records here", dane requested only (should transmit) +# exim -odq CALLER@dane.no.4.test.ex +# Testing +# **** exim -qf **** killdaemon diff --git a/test/stderr/5840 b/test/stderr/5840 index e4cf15c51..75f938ab4 100644 --- a/test/stderr/5840 +++ b/test/stderr/5840 @@ -20,10 +20,7 @@ >>> Attempting full verification using callout >>> callout cache: no domain record found for dane256ee.test.ex >>> callout cache: no address record found for rcptuser@dane256ee.test.ex -MUNGED: ::1 will be omitted in what follows ->>> get[host|ipnode]byname[2] looked up these IP addresses: ->>> name=thishost.test.ex address=127.0.0.1 ->>> ip4.ip4.ip4.ip4 in hosts_require_dane? yes (end of list) +>>> ip4.ip4.ip4.ip4 in hosts_require_dane? yes (matched "ip4.ip4.ip4.ip4") >>> interface=NULL port=1225 >>> Connecting to dane256ee.test.ex [ip4.ip4.ip4.ip4]:1225 ... connected >>> SMTP<< 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 @@ -73,8 +70,9 @@ LOG: unexpected disconnection while reading SMTP command from [127.0.0.1] ### A server with a nonverifying cert and no TLSA ### A server with a verifying cert and no TLSA ### A server with two MXs for which both TLSA lookups return defer -### A server lacking a TLSA, required -### A server lacking a TLSA, requested only +### A server lacking a TLSA, dane required (should fail) +### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) +### A server where the A is dnssec and the TLSA _fails_ ******** SERVER ******** ### TLSA (3 1 1) @@ -84,5 +82,6 @@ LOG: unexpected disconnection while reading SMTP command from [127.0.0.1] ### A server with a nonverifying cert and no TLSA ### A server with a verifying cert and no TLSA ### A server with two MXs for which both TLSA lookups return defer -### A server lacking a TLSA, required -### A server lacking a TLSA, requested only +### A server lacking a TLSA, dane required (should fail) +### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) +### A server where the A is dnssec and the TLSA _fails_ diff --git a/test/stdout/5840 b/test/stdout/5840 index 1d94564ad..5071e7de5 100644 --- a/test/stdout/5840 +++ b/test/stdout/5840 @@ -14,8 +14,9 @@ ### A server with a nonverifying cert and no TLSA ### A server with a verifying cert and no TLSA ### A server with two MXs for which both TLSA lookups return defer -### A server lacking a TLSA, required -### A server lacking a TLSA, requested only +### A server lacking a TLSA, dane required (should fail) +### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) +### A server where the A is dnssec and the TLSA _fails_ ******** SERVER ******** ### TLSA (3 1 1) @@ -25,5 +26,6 @@ ### A server with a nonverifying cert and no TLSA ### A server with a verifying cert and no TLSA ### A server with two MXs for which both TLSA lookups return defer -### A server lacking a TLSA, required -### A server lacking a TLSA, requested only +### A server lacking a TLSA, dane required (should fail) +### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) +### A server where the A is dnssec and the TLSA _fails_ -- cgit v1.2.3 From ce889807c90746896f1310e9f4957215f46f7836 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 7 May 2017 17:40:41 +0100 Subject: Testsuite: add DANE cases for DNS secure no-TLSA lookups --- test/dnszones-src/db.test.ex | 12 +++++++++++- test/log/5840 | 12 ++++++++++++ test/scripts/5840-DANE-OpenSSL/5840 | 15 ++++++++------- test/src/fakens.c | 16 ++++++++++++++-- test/stderr/5840 | 4 ++++ test/stdout/5840 | 4 ++++ 6 files changed, 53 insertions(+), 10 deletions(-) (limited to 'test/stdout') diff --git a/test/dnszones-src/db.test.ex b/test/dnszones-src/db.test.ex index 50bd6b073..f7c9e313b 100644 --- a/test/dnszones-src/db.test.ex +++ b/test/dnszones-src/db.test.ex @@ -461,7 +461,8 @@ DNSSEC danelazy2 A 127.0.0.1 DNSSEC _1225._tcp.danelazy CNAME test.again.dns. DNSSEC _1225._tcp.danelazy2 CNAME test.again.dns. -; hosts with no TLSA +; hosts with no TLSA (just missing here, hence the TLSA NXDMAIN is _insecure_; a broken dane config) +; 1 for dane-required, 2 for merely requested DNSSEC dane.no.1 A HOSTIPV4 DNSSEC dane.no.2 A 127.0.0.1 @@ -469,6 +470,15 @@ DNSSEC dane.no.2 A 127.0.0.1 DNSSEC danebroken1 A 127.0.0.1 _1225._tcp.danebroken1 CNAME test.fail.dns. +; a good dns config saying there is no dane support, by securely returning NOXDOMAIN for TLSA lookups +; 3 for dane-required, 4 for merely requested +; the TLSA data here is dummy; ignored +DNSSEC dane.no.3 A HOSTIPV4 +DNSSEC dane.no.4 A 127.0.0.1 + +DNSSEC NXDOMAIN _1225._tcp.dane.no.3 TLSA 2 0 1 eec923139018c540a344c5191660ecba1ac3708525a98bfc338e17f31d3fa741 +DNSSEC NXDOMAIN _1225._tcp.dane.no.4 TLSA 2 0 1 eec923139018c540a344c5191660ecba1ac3708525a98bfc338e17f31d3fa741 + ; ------- Testing delays ------------ DELAY=500 delay500 A HOSTIPV4 diff --git a/test/log/5840 b/test/log/5840 index d02a4c7d7..b2f949009 100644 --- a/test/log/5840 +++ b/test/log/5840 @@ -27,6 +27,8 @@ 1999-03-02 09:44:33 10HmbI-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@dane.no.1.test.ex 1999-03-02 09:44:33 10HmbJ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@dane.no.2.test.ex 1999-03-02 09:44:33 10HmbK-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@danebroken1.test.ex +1999-03-02 09:44:33 10HmbL-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@dane.no.3.test.ex +1999-03-02 09:44:33 10HmbM-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss for CALLER@dane.no.4.test.ex 1999-03-02 09:44:33 Start queue run: pid=pppp -qf 1999-03-02 09:44:33 10HmbH-0005vi-00 H=danelazy.test.ex [ip4.ip4.ip4.ip4]: DANE error: tlsa lookup DEFER 1999-03-02 09:44:33 10HmbH-0005vi-00 H=danelazy2.test.ex [127.0.0.1]: DANE error: tlsa lookup DEFER @@ -38,6 +40,13 @@ 1999-03-02 09:44:33 10HmbJ-0005vi-00 == CALLER@dane.no.2.test.ex R=client T=send_to_server defer (-36): DANE error: tlsa lookup DEFER 1999-03-02 09:44:33 10HmbK-0005vi-00 H=danebroken1.test.ex [127.0.0.1]: DANE error: tlsa lookup DEFER 1999-03-02 09:44:33 10HmbK-0005vi-00 == CALLER@danebroken1.test.ex R=client T=send_to_server defer (-36): DANE error: tlsa lookup DEFER +1999-03-02 09:44:33 10HmbL-0005vi-00 ** CALLER@dane.no.3.test.ex R=client T=send_to_server: DANE error: tlsa lookup FAIL +1999-03-02 09:44:33 10HmbL-0005vi-00 CALLER@dane.no.3.test.ex: error ignored +1999-03-02 09:44:33 10HmbL-0005vi-00 Completed +1999-03-02 09:44:33 10HmbM-0005vi-00 [127.0.0.1] SSL verify error: depth=0 error=self signed certificate cert=/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock +1999-03-02 09:44:33 10HmbM-0005vi-00 [127.0.0.1] SSL verify error: certificate name mismatch: DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" H="dane.no.4.test.ex" +1999-03-02 09:44:33 10HmbM-0005vi-00 => CALLER@dane.no.4.test.ex R=client T=send_to_server H=dane.no.4.test.ex [127.0.0.1] X=TLSv1:AES256-SHA:256 CV=no DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" C="250 OK id=10HmbN-0005vi-00" +1999-03-02 09:44:33 10HmbM-0005vi-00 Completed 1999-03-02 09:44:33 End queue run: pid=pppp -qf ******** SERVER ******** @@ -61,3 +70,6 @@ 1999-03-02 09:44:33 10HmbG-0005vi-00 => :blackhole: R=server 1999-03-02 09:44:33 10HmbG-0005vi-00 Completed 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225 +1999-03-02 09:44:33 10HmbN-0005vi-00 <= <> H=localhost (myhost.test.ex) [127.0.0.1] P=esmtps X=TLSv1:AES256-SHA:256 CV=no S=sss id=E10HmbM-0005vi-00@myhost.test.ex for CALLER@dane.no.4.test.ex +1999-03-02 09:44:33 10HmbN-0005vi-00 => :blackhole: R=server +1999-03-02 09:44:33 10HmbN-0005vi-00 Completed diff --git a/test/scripts/5840-DANE-OpenSSL/5840 b/test/scripts/5840-DANE-OpenSSL/5840 index fdff36119..142a25ad4 100644 --- a/test/scripts/5840-DANE-OpenSSL/5840 +++ b/test/scripts/5840-DANE-OpenSSL/5840 @@ -73,13 +73,14 @@ Testing exim -odq CALLER@danebroken1.test.ex Testing **** -# ### A server securely saying "no TLSA records here", dane required (should fail) -# exim -odq CALLER@dane.no.3.test.ex -# Testing -# ### A server securely saying "no TLSA records here", dane requested only (should transmit) -# exim -odq CALLER@dane.no.4.test.ex -# Testing -# **** +### A server securely saying "no TLSA records here", dane required (should fail) +exim -odq CALLER@dane.no.3.test.ex +Testing +**** +### A server securely saying "no TLSA records here", dane requested only (should transmit) +exim -odq CALLER@dane.no.4.test.ex +Testing +**** exim -qf **** killdaemon diff --git a/test/src/fakens.c b/test/src/fakens.c index 34f5ea670..583b01282 100644 --- a/test/src/fakens.c +++ b/test/src/fakens.c @@ -53,11 +53,15 @@ HOST_NOT_FOUND. Any DNS record line in a zone file can be prefixed with "DELAY=" and a number of milliseconds (followed by one space). -Any DNS record line in a zone file can be prefixed with "DNSSEC "; +Any DNS record line can be prefixed with "DNSSEC "; if all the records found by a lookup are marked as such then the response will have the "AD" bit set. -Any DNS record line in a zone file can be prefixed with "AA " +Any DNS record line can be prefixed with "NXDOMAIN "; +The record will be ignored (but the prefix set still applied); +This lets us return a DNSSEC NXDOMAIN. + +Any DNS record line can be prefixed with "AA " if all the records found by a lookup are marked as such then the response will have the "AA" bit set. @@ -354,6 +358,7 @@ while (fgets(CS buffer, sizeof(buffer), f) != NULL) int qtlen = qtypelen; BOOL rr_sec = FALSE; BOOL rr_aa = FALSE; + BOOL rr_ignore = FALSE; int delay = 0; uint ttl = DEFAULT_TTL; @@ -379,6 +384,11 @@ while (fgets(CS buffer, sizeof(buffer), f) != NULL) rr_sec = TRUE; p += 7; } + if (Ustrncmp(p, US"NXDOMAIN ", 9) == 0) /* ignore record content */ + { + rr_ignore = TRUE; + p += 9; + } else if (Ustrncmp(p, US"AA ", 3) == 0) /* tagged as authoritative */ { rr_aa = TRUE; @@ -464,6 +474,8 @@ while (fgets(CS buffer, sizeof(buffer), f) != NULL) if (aa && !rr_aa) *aa = FALSE; /* cancel AA return */ + if (rr_ignore) continue; + yield = 0; *countptr = *countptr + 1; diff --git a/test/stderr/5840 b/test/stderr/5840 index 75f938ab4..5ccf7cda0 100644 --- a/test/stderr/5840 +++ b/test/stderr/5840 @@ -73,6 +73,8 @@ LOG: unexpected disconnection while reading SMTP command from [127.0.0.1] ### A server lacking a TLSA, dane required (should fail) ### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) ### A server where the A is dnssec and the TLSA _fails_ +### A server securely saying "no TLSA records here", dane required (should fail) +### A server securely saying "no TLSA records here", dane requested only (should transmit) ******** SERVER ******** ### TLSA (3 1 1) @@ -85,3 +87,5 @@ LOG: unexpected disconnection while reading SMTP command from [127.0.0.1] ### A server lacking a TLSA, dane required (should fail) ### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) ### A server where the A is dnssec and the TLSA _fails_ +### A server securely saying "no TLSA records here", dane required (should fail) +### A server securely saying "no TLSA records here", dane requested only (should transmit) diff --git a/test/stdout/5840 b/test/stdout/5840 index 5071e7de5..32425d2e2 100644 --- a/test/stdout/5840 +++ b/test/stdout/5840 @@ -17,6 +17,8 @@ ### A server lacking a TLSA, dane required (should fail) ### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) ### A server where the A is dnssec and the TLSA _fails_ +### A server securely saying "no TLSA records here", dane required (should fail) +### A server securely saying "no TLSA records here", dane requested only (should transmit) ******** SERVER ******** ### TLSA (3 1 1) @@ -29,3 +31,5 @@ ### A server lacking a TLSA, dane required (should fail) ### A server lacking a TLSA, dane requested only (should fail, as the NXDOMAIN is not DNSSEC) ### A server where the A is dnssec and the TLSA _fails_ +### A server securely saying "no TLSA records here", dane required (should fail) +### A server securely saying "no TLSA records here", dane requested only (should transmit) -- cgit v1.2.3 From 925ac8e4f1c5d365ddea2f7aee460cd0a3cd409d Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Wed, 17 May 2017 17:32:30 +0100 Subject: TLS: do decent PIPELINING under TLS, at least with GnuTLS I've not found an equivalent in OpenSSL of gnutls_record_cork() nor gnutls_record_check_pending() yet. --- doc/doc-txt/ChangeLog | 6 ++ src/src/auths/get_data.c | 2 +- src/src/auths/get_no64_data.c | 2 +- src/src/daemon.c | 12 +-- src/src/dkim_transport.c | 2 +- src/src/functions.h | 3 +- src/src/local_scan.h | 4 +- src/src/receive.c | 11 ++- src/src/smtp_in.c | 173 ++++++++++++++++++++++++----------------- src/src/smtp_out.c | 9 ++- src/src/tls-gnu.c | 35 ++++++++- src/src/tls-openssl.c | 16 +++- src/src/transport.c | 5 +- src/src/transports/smtp.c | 3 +- src/src/verify.c | 4 +- test/confs/2007 | 1 + test/scripts/2000-GnuTLS/2090 | 8 +- test/scripts/2100-OpenSSL/2190 | 8 +- test/stdout/2090 | 8 +- test/stdout/2190 | 8 +- 20 files changed, 185 insertions(+), 135 deletions(-) (limited to 'test/stdout') diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index 4bc5235ec..12c1ff35b 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -86,6 +86,12 @@ PP/04 Add commandline_checks_require_admin option. AND make fixes to the calling application, such as using `--` to stop processing options. +JH/13 Do pipelining under TLS, with GnuTLS. Previously, although safe, no + advantage was taken. Now take care to pack both (client) MAIL,RCPT,DATA, + and (server) responses to those, into a single TLS record each way (this + usually means a single packet). As a side issue, we can now detect + over-eager senders in non-pipelined mode. + Exim version 4.89 ----------------- diff --git a/src/src/auths/get_data.c b/src/src/auths/get_data.c index 11bc581b9..3857e0738 100644 --- a/src/src/auths/get_data.c +++ b/src/src/auths/get_data.c @@ -30,7 +30,7 @@ auth_get_data(uschar **aptr, uschar *challenge, int challen) { int c; int p = 0; -smtp_printf("334 %s\r\n", b64encode(challenge, challen)); +smtp_printf("334 %s\r\n", FALSE, b64encode(challenge, challen)); while ((c = receive_getc(GETC_BUFFER_UNLIMITED)) != '\n' && c != EOF) { if (p >= big_buffer_size - 1) return BAD64; diff --git a/src/src/auths/get_no64_data.c b/src/src/auths/get_no64_data.c index 71e71394c..1bb611c1c 100644 --- a/src/src/auths/get_no64_data.c +++ b/src/src/auths/get_no64_data.c @@ -31,7 +31,7 @@ auth_get_no64_data(uschar **aptr, uschar *challenge) { int c; int p = 0; -smtp_printf("334 %s\r\n", challenge); +smtp_printf("334 %s\r\n", FALSE, challenge); while ((c = receive_getc(GETC_BUFFER_UNLIMITED)) != '\n' && c != EOF) { if (p >= big_buffer_size - 1) return BAD64; diff --git a/src/src/daemon.c b/src/src/daemon.c index af5015454..06c2b258c 100644 --- a/src/src/daemon.c +++ b/src/src/daemon.c @@ -109,7 +109,7 @@ never_error(uschar *log_msg, uschar *smtp_msg, int was_errno) uschar *emsg = (was_errno <= 0)? US"" : string_sprintf(": %s", strerror(was_errno)); log_write(0, LOG_MAIN|LOG_PANIC, "%s%s", log_msg, emsg); -if (smtp_out != NULL) smtp_printf("421 %s\r\n", smtp_msg); +if (smtp_out != NULL) smtp_printf("421 %s\r\n", FALSE, smtp_msg); } @@ -189,7 +189,7 @@ if (getsockname(accept_socket, (struct sockaddr *)(&interface_sockaddr), { log_write(0, LOG_MAIN | ((errno == ECONNRESET)? 0 : LOG_PANIC), "getsockname() failed: %s", strerror(errno)); - smtp_printf("421 Local problem: getsockname() failed; please try again later\r\n"); + smtp_printf("421 Local problem: getsockname() failed; please try again later\r\n", FALSE); goto ERROR_RETURN; } @@ -222,7 +222,7 @@ if (smtp_accept_max > 0 && smtp_accept_count >= smtp_accept_max) DEBUG(D_any) debug_printf("rejecting SMTP connection: count=%d max=%d\n", smtp_accept_count, smtp_accept_max); smtp_printf("421 Too many concurrent SMTP connections; " - "please try again later.\r\n"); + "please try again later.\r\n", FALSE); log_write(L_connection_reject, LOG_MAIN, "Connection from %s refused: too many connections", whofrom); @@ -241,7 +241,7 @@ if (smtp_load_reserve >= 0) { DEBUG(D_any) debug_printf("rejecting SMTP connection: load average = %.2f\n", (double)load_average/1000.0); - smtp_printf("421 Too much load; please try again later.\r\n"); + smtp_printf("421 Too much load; please try again later.\r\n", FALSE); log_write(L_connection_reject, LOG_MAIN, "Connection from %s refused: load average = %.2f", whofrom, (double)load_average/1000.0); @@ -312,7 +312,7 @@ if ((max_for_this_host > 0) && "IP address: count=%d max=%d\n", host_accept_count, max_for_this_host); smtp_printf("421 Too many concurrent SMTP connections " - "from this IP address; please try again later.\r\n"); + "from this IP address; please try again later.\r\n", FALSE); log_write(L_connection_reject, LOG_MAIN, "Connection from %s refused: too many connections " "from that IP address", whofrom); @@ -396,7 +396,7 @@ if (pid == 0) "(smtp_active_hostname): %s", raw_active_hostname, expand_string_message); smtp_printf("421 Local configuration error; " - "please try again later.\r\n"); + "please try again later.\r\n", FALSE); mac_smtp_fflush(); search_tidyup(); _exit(EXIT_FAILURE); diff --git a/src/src/dkim_transport.c b/src/src/dkim_transport.c index 95e750e5e..4d26f9291 100644 --- a/src/src/dkim_transport.c +++ b/src/src/dkim_transport.c @@ -77,7 +77,7 @@ else { #ifdef SUPPORT_TLS wwritten = tls_out.active == out_fd - ? tls_write(FALSE, p, sread) + ? tls_write(FALSE, p, sread, FALSE) : write(out_fd, CS p, sread); #else wwritten = write(out_fd, CS p, sread); diff --git a/src/src/functions.h b/src/src/functions.h index 963f48ed4..4be375d77 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -51,6 +51,7 @@ extern int tls_client_start(int, host_item *, address_item *, # endif uschar **); extern void tls_close(BOOL, BOOL); +extern BOOL tls_could_read(void); extern int tls_export_cert(uschar *, size_t, void *); extern int tls_feof(void); extern int tls_ferror(void); @@ -63,7 +64,7 @@ extern int tls_read(BOOL, uschar *, size_t); extern int tls_server_start(const uschar *, uschar **); extern BOOL tls_smtp_buffered(void); extern int tls_ungetc(int); -extern int tls_write(BOOL, const uschar *, size_t); +extern int tls_write(BOOL, const uschar *, size_t, BOOL); extern uschar *tls_validate_require_cipher(void); extern void tls_version_report(FILE *); # ifndef USE_GNUTLS diff --git a/src/src/local_scan.h b/src/src/local_scan.h index bc4fc8e25..c03d87032 100644 --- a/src/src/local_scan.h +++ b/src/src/local_scan.h @@ -186,8 +186,8 @@ extern void receive_add_recipient(uschar *, int); extern BOOL receive_remove_recipient(uschar *); extern uschar *rfc2047_decode(uschar *, BOOL, uschar *, int, int *, uschar **); extern int smtp_fflush(void); -extern void smtp_printf(const char *, ...) PRINTF_FUNCTION(1,2); -extern void smtp_vprintf(const char *, va_list); +extern void smtp_printf(const char *, BOOL, ...) PRINTF_FUNCTION(1,3); +extern void smtp_vprintf(const char *, BOOL, va_list); extern uschar *string_copy(const uschar *); extern uschar *string_copyn(const uschar *, int); extern uschar *string_sprintf(const char *, ...) ALMOST_PRINTF(1,2); diff --git a/src/src/receive.c b/src/src/receive.c index 6316ff961..9561a4baf 100644 --- a/src/src/receive.c +++ b/src/src/receive.c @@ -2194,7 +2194,7 @@ for (;;) sender_address, sender_fullhost ? " H=" : "", sender_fullhost ? sender_fullhost : US"", sender_ident ? " U=" : "", sender_ident ? sender_ident : US""); - smtp_printf("552 Message header not CRLF terminated\r\n"); + smtp_printf("552 Message header not CRLF terminated\r\n", FALSE); bdat_flush_data(); smtp_reply = US""; goto TIDYUP; /* Skip to end of function */ @@ -3501,7 +3501,7 @@ else int all_pass = OK; int all_fail = FAIL; - smtp_printf("353 PRDR content analysis beginning\r\n"); + smtp_printf("353 PRDR content analysis beginning\r\n", TRUE); /* Loop through recipients, responses must be in same order received */ for (c = 0; recipients_count > c; c++) { @@ -4302,13 +4302,12 @@ if (smtp_input) else if (chunking_state > CHUNKING_OFFERED) { -/*XXX rethink for spool_wireformat */ - smtp_printf("250- %u byte chunk, total %d\r\n250 OK id=%s\r\n", + smtp_printf("250- %u byte chunk, total %d\r\n250 OK id=%s\r\n", FALSE, chunking_datasize, message_size+message_linecount, message_id); chunking_state = CHUNKING_OFFERED; } else - smtp_printf("250 OK id=%s\r\n", message_id); + smtp_printf("250 OK id=%s\r\n", FALSE, message_id); if (host_checking) fprintf(stdout, @@ -4322,7 +4321,7 @@ if (smtp_input) smtp_respond((fake_response == DEFER)? US"450" : US"550", 3, TRUE, fake_response_text); else - smtp_printf("%.1024s\r\n", smtp_reply); + smtp_printf("%.1024s\r\n", FALSE, smtp_reply); switch (cutthrough_done) { diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index e5885e477..5ff1b7f32 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -314,10 +314,10 @@ static void smtp_rset_handler(void); *************************************************/ /* Synchronization checks can never be perfect because a packet may be on its -way but not arrived when the check is done. Such checks can in any case only be -done when TLS is not in use. Normally, the checks happen when commands are -read: Exim ensures that there is no more input in the input buffer. In normal -cases, the response to the command will be fast, and there is no further check. +way but not arrived when the check is done. Normally, the checks happen when +commands are read: Exim ensures that there is no more input in the input buffer. +In normal cases, the response to the command will be fast, and there is no +further check. However, for some commands an ACL is run, and that can include delays. In those cases, it is useful to do another check on the input just before sending the @@ -333,15 +333,14 @@ Returns: TRUE if all is well; FALSE if there is input pending */ static BOOL -check_sync(void) +wouldblock_reading(void) { int fd, rc; fd_set fds; struct timeval tzero; -if (!smtp_enforce_sync || sender_host_address == NULL || - sender_host_notsocket || tls_in.active >= 0) - return TRUE; +if (tls_in.active >= 0 && tls_could_read()) + return FALSE; if (smtp_inptr < smtp_inend) return FALSE; @@ -364,6 +363,29 @@ smtp_inptr[rc] = 0; return FALSE; } +static BOOL +check_sync(void) +{ +if (!smtp_enforce_sync || sender_host_address == NULL || sender_host_notsocket) + return TRUE; + +return wouldblock_reading(); +} + + +/* If there's input waiting (and we're doing pipelineing) then we can pipeline +a reponse with the one following. */ + +static BOOL +pipeline_response(void) +{ +if ( !smtp_enforce_sync || !sender_host_address + || sender_host_notsocket || !pipelining_advertised) + return FALSE; + +return !wouldblock_reading(); +} + /************************************************* @@ -560,7 +582,7 @@ for(;;) return EOD; } - smtp_printf("250 %u byte chunk received\r\n", chunking_datasize); + smtp_printf("250 %u byte chunk received\r\n", FALSE, chunking_datasize); chunking_state = CHUNKING_OFFERED; DEBUG(D_receive) debug_printf("chunking state %d\n", (int)chunking_state); @@ -598,7 +620,7 @@ next_cmd: case NOOP_CMD: HAD(SCH_NOOP); - smtp_printf("250 OK\r\n"); + smtp_printf("250 OK\r\n", FALSE); goto next_cmd; case BDAT_CMD: @@ -774,18 +796,19 @@ they are also picked up later by smtp_fflush(). Arguments: format format string + more further data expected ... optional arguments Returns: nothing */ void -smtp_printf(const char *format, ...) +smtp_printf(const char *format, BOOL more, ...) { va_list ap; -va_start(ap, format); -smtp_vprintf(format, ap); +va_start(ap, more); +smtp_vprintf(format, more, ap); va_end(ap); } @@ -794,7 +817,7 @@ smtp_printf(), bearing in mind that in C a vararg function can't directly call another vararg function, only a function which accepts a va_list. */ void -smtp_vprintf(const char *format, va_list ap) +smtp_vprintf(const char *format, BOOL more, va_list ap) { BOOL yield; @@ -840,7 +863,7 @@ if (rcpt_in_progress) #ifdef SUPPORT_TLS if (tls_in.active >= 0) { - if (tls_write(TRUE, big_buffer, Ustrlen(big_buffer)) < 0) + if (tls_write(TRUE, big_buffer, Ustrlen(big_buffer), more) < 0) smtp_write_error = -1; } else @@ -1619,7 +1642,7 @@ smtp_closedown(uschar *message) { if (smtp_in == NULL || smtp_batched_input) return; receive_swallow_smtp(); -smtp_printf("421 %s\r\n", message); +smtp_printf("421 %s\r\n", FALSE, message); for (;;) switch(smtp_read_command(FALSE, GETC_BUFFER_UNLIMITED)) { @@ -1627,16 +1650,16 @@ for (;;) switch(smtp_read_command(FALSE, GETC_BUFFER_UNLIMITED)) return; case QUIT_CMD: - smtp_printf("221 %s closing connection\r\n", smtp_active_hostname); + smtp_printf("221 %s closing connection\r\n", FALSE, smtp_active_hostname); mac_smtp_fflush(); return; case RSET_CMD: - smtp_printf("250 Reset OK\r\n"); + smtp_printf("250 Reset OK\r\n", FALSE); break; default: - smtp_printf("421 %s\r\n", message); + smtp_printf("421 %s\r\n", FALSE, message); break; } } @@ -2490,7 +2513,7 @@ if (!sender_host_unknown) { log_write(0, LOG_MAIN, "getsockopt() failed from %s: %s", host_and_ident(FALSE), strerror(errno)); - smtp_printf("451 SMTP service not available\r\n"); + smtp_printf("451 SMTP service not available\r\n", FALSE); return FALSE; } } @@ -2595,7 +2618,7 @@ if (!sender_host_unknown) log_write(0, LOG_MAIN|LOG_REJECT, "connection from %s refused (IP options)", host_and_ident(FALSE)); - smtp_printf("554 SMTP service not available\r\n"); + smtp_printf("554 SMTP service not available\r\n", FALSE); return FALSE; } @@ -2647,7 +2670,7 @@ if (!sender_host_unknown) { log_write(L_connection_reject, LOG_MAIN|LOG_REJECT, "refused connection " "from %s (host_reject_connection)", host_and_ident(FALSE)); - smtp_printf("554 SMTP service not available\r\n"); + smtp_printf("554 SMTP service not available\r\n", FALSE); return FALSE; } @@ -2678,7 +2701,7 @@ if (!sender_host_unknown) log_write(L_connection_reject, LOG_MAIN|LOG_REJECT, "refused connection from %s " "(tcp wrappers)", host_and_ident(FALSE)); - smtp_printf("554 SMTP service not available\r\n"); + smtp_printf("554 SMTP service not available\r\n", FALSE); } else { @@ -2688,7 +2711,7 @@ if (!sender_host_unknown) log_write(L_connection_reject, LOG_MAIN|LOG_REJECT, "temporarily refused connection from %s " "(tcp wrappers errno=%d)", host_and_ident(FALSE), save_errno); - smtp_printf("451 Temporary local problem - please try later\r\n"); + smtp_printf("451 Temporary local problem - please try later\r\n", FALSE); } return FALSE; } @@ -2708,7 +2731,7 @@ if (!sender_host_unknown) host_and_ident(FALSE), smtp_accept_count - 1, smtp_accept_max, smtp_accept_reserve, (rc == DEFER)? " (lookup deferred)" : ""); smtp_printf("421 %s: Too many concurrent SMTP connections; " - "please try again later\r\n", smtp_active_hostname); + "please try again later\r\n", FALSE, smtp_active_hostname); return FALSE; } reserved_host = TRUE; @@ -2729,7 +2752,7 @@ if (!sender_host_unknown) LOG_MAIN, "temporarily refused connection from %s: not in " "reserve list and load average = %.2f", host_and_ident(FALSE), (double)load_average/1000.0); - smtp_printf("421 %s: Too much load; please try again later\r\n", + smtp_printf("421 %s: Too much load; please try again later\r\n", FALSE, smtp_active_hostname); return FALSE; } @@ -2876,13 +2899,13 @@ if (!check_sync()) "synchronization error (input sent without waiting for greeting): " "rejected connection from %s input=\"%s\"", host_and_ident(TRUE), string_printing(string_copyn(smtp_inptr, n))); - smtp_printf("554 SMTP synchronization error\r\n"); + smtp_printf("554 SMTP synchronization error\r\n", FALSE); return FALSE; } /* Now output the banner */ -smtp_printf("%s", ss); +smtp_printf("%s", FALSE, ss); return TRUE; } @@ -2929,10 +2952,10 @@ if (++synprot_error_count > smtp_max_synprot_errors) if (code > 0) { - smtp_printf("%d%c%s%s%s\r\n", code, (yield == 1)? '-' : ' ', - (data == NULL)? US"" : data, (data == NULL)? US"" : US": ", errmess); + smtp_printf("%d%c%s%s%s\r\n", FALSE, code, yield == 1 ? '-' : ' ', + data ? data : US"", data ? US": " : US"", errmess); if (yield == 1) - smtp_printf("%d Too many syntax or protocol errors\r\n", code); + smtp_printf("%d Too many syntax or protocol errors\r\n", FALSE, code); } return yield; @@ -2988,25 +3011,27 @@ if (rcpt_in_progress) rcpt_in_progress = FALSE; } -/* Not output the message, splitting it up into multiple lines if necessary. */ +/* Not output the message, splitting it up into multiple lines if necessary. +We only handle pipelining these responses as far as nonfinal/final groups, +not the whole MAIL/RCPT/DATA response set. */ for (;;) { uschar *nl = Ustrchr(msg, '\n'); if (nl == NULL) { - smtp_printf("%.3s%c%.*s%s\r\n", code, final? ' ':'-', esclen, esc, msg); + smtp_printf("%.3s%c%.*s%s\r\n", !final, code, final ? ' ':'-', esclen, esc, msg); return; } else if (nl[1] == 0 || no_multiline_responses) { - smtp_printf("%.3s%c%.*s%.*s\r\n", code, final? ' ':'-', esclen, esc, + smtp_printf("%.3s%c%.*s%.*s\r\n", !final, code, final ? ' ':'-', esclen, esc, (int)(nl - msg), msg); return; } else { - smtp_printf("%.3s-%.*s%.*s\r\n", code, esclen, esc, (int)(nl - msg), msg); + smtp_printf("%.3s-%.*s%.*s\r\n", TRUE, code, esclen, esc, (int)(nl - msg), msg); msg = nl + 1; while (isspace(*msg)) msg++; } @@ -3350,7 +3375,7 @@ if (code && defaultrespond) va_start(ap, defaultrespond); if (!string_vformat(buffer, sizeof(buffer), CS defaultrespond, ap)) log_write(0, LOG_MAIN|LOG_PANIC, "string too large in smtp_notquit_exit()"); - smtp_printf("%s %s\r\n", code, buffer); + smtp_printf("%s %s\r\n", FALSE, code, buffer); va_end(ap); } mac_smtp_fflush(); @@ -3642,7 +3667,7 @@ if (allow_unqualified_recipient || strcmpic(*recipient, US"postmaster") == 0) *recipient = rewrite_address_qualify(*recipient, TRUE); return rd; } -smtp_printf("501 %s: recipient address must contain a domain\r\n", +smtp_printf("501 %s: recipient address must contain a domain\r\n", FALSE, smtp_cmd_data); log_write(L_smtp_syntax_error, LOG_MAIN|LOG_REJECT, "unqualified %s rejected: <%s> %s%s", @@ -3668,7 +3693,7 @@ if (acl_smtp_quit) if (*user_msgp) smtp_respond(US"221", 3, TRUE, *user_msgp); else - smtp_printf("221 %s closing connection\r\n", smtp_active_hostname); + smtp_printf("221 %s closing connection\r\n", FALSE, smtp_active_hostname); #ifdef SUPPORT_TLS tls_close(TRUE, TRUE); @@ -3684,7 +3709,7 @@ smtp_rset_handler(void) { HAD(SCH_RSET); incomplete_transaction_log(US"RSET"); -smtp_printf("250 Reset OK\r\n"); +smtp_printf("250 Reset OK\r\n", FALSE); cmd_list[CMD_LIST_RSET].is_mail_cmd = FALSE; } @@ -3904,7 +3929,7 @@ while (done <= 0) { c = smtp_in_auth(au, &s, &ss); - smtp_printf("%s\r\n", s); + smtp_printf("%s\r\n", FALSE, s); if (c != OK) log_write(0, LOG_MAIN|LOG_REJECT, "%s authenticator failed for %s: %s", au->name, host_and_ident(FALSE), ss); @@ -3951,7 +3976,7 @@ while (done <= 0) if (!check_helo(smtp_cmd_data)) { - smtp_printf("501 Syntactically invalid %s argument(s)\r\n", hello); + smtp_printf("501 Syntactically invalid %s argument(s)\r\n", FALSE, hello); log_write(0, LOG_MAIN|LOG_REJECT, "rejected %s from %s: syntactically " "invalid argument(s): %s", hello, host_and_ident(FALSE), @@ -4015,7 +4040,7 @@ while (done <= 0) { if (helo_required) { - smtp_printf("%d %s argument does not match calling host\r\n", + smtp_printf("%d %s argument does not match calling host\r\n", FALSE, tempfail? 451 : 550, hello); log_write(0, LOG_MAIN|LOG_REJECT, "%srejected \"%s %s\" from %s", tempfail? "temporarily " : "", @@ -4285,7 +4310,7 @@ while (done <= 0) s[ptr] = 0; #ifdef SUPPORT_TLS - if (tls_in.active >= 0) (void)tls_write(TRUE, s, ptr); else + if (tls_in.active >= 0) (void)tls_write(TRUE, s, ptr, FALSE); else #endif { @@ -4328,7 +4353,7 @@ while (done <= 0) if (helo_required && !helo_seen) { - smtp_printf("503 HELO or EHLO required\r\n"); + smtp_printf("503 HELO or EHLO required\r\n", FALSE); log_write(0, LOG_MAIN|LOG_REJECT, "rejected MAIL from %s: no " "HELO/EHLO given", host_and_ident(FALSE)); break; @@ -4354,7 +4379,7 @@ while (done <= 0) if (smtp_accept_max_per_connection > 0 && smtp_mailcmd_count > smtp_accept_max_per_connection) { - smtp_printf("421 too many messages in this connection\r\n"); + smtp_printf("421 too many messages in this connection\r\n", FALSE); log_write(0, LOG_MAIN|LOG_REJECT, "rejected MAIL command %s: too many " "messages in one connection", host_and_ident(TRUE)); break; @@ -4619,7 +4644,7 @@ while (done <= 0) if (thismessage_size_limit > 0 && message_size > thismessage_size_limit) { - smtp_printf("552 Message size exceeds maximum permitted\r\n"); + smtp_printf("552 Message size exceeds maximum permitted\r\n", FALSE); log_write(L_size_reject, LOG_MAIN|LOG_REJECT, "rejected MAIL FROM:<%s> %s: " "message too big: size%s=%d max=%d", @@ -4644,7 +4669,7 @@ while (done <= 0) (smtp_check_spool_space && message_size >= 0)? message_size + 5000 : 0)) { - smtp_printf("452 Space shortage, please try later\r\n"); + smtp_printf("452 Space shortage, please try later\r\n", FALSE); sender_address = NULL; break; } @@ -4666,7 +4691,7 @@ while (done <= 0) } else { - smtp_printf("501 %s: sender address must contain a domain\r\n", + smtp_printf("501 %s: sender address must contain a domain\r\n", FALSE, smtp_cmd_data); log_write(L_smtp_syntax_error, LOG_MAIN|LOG_REJECT, @@ -4694,8 +4719,10 @@ while (done <= 0) if (rc == OK || rc == DISCARD) { + BOOL more = pipeline_response(); + if (!user_msg) - smtp_printf("%s%s%s", US"250 OK", + smtp_printf("%s%s%s", more, US"250 OK", #ifndef DISABLE_PRDR prdr_requested ? US", PRDR Requested" : US"", #else @@ -4741,7 +4768,7 @@ while (done <= 0) { if (pipelining_advertised && last_was_rej_mail) { - smtp_printf("503 sender not yet given\r\n"); + smtp_printf("503 sender not yet given\r\n", FALSE); was_rej_mail = TRUE; } else @@ -4889,7 +4916,7 @@ while (done <= 0) if (recipients_max_reject) { rcpt_fail_count++; - smtp_printf("552 too many recipients\r\n"); + smtp_printf("552 too many recipients\r\n", FALSE); if (!toomany) log_write(0, LOG_MAIN|LOG_REJECT, "too many recipients: message " "rejected: sender=<%s> %s", sender_address, host_and_ident(TRUE)); @@ -4897,7 +4924,7 @@ while (done <= 0) else { rcpt_defer_count++; - smtp_printf("452 too many recipients\r\n"); + smtp_printf("452 too many recipients\r\n", FALSE); if (!toomany) log_write(0, LOG_MAIN|LOG_REJECT, "too many recipients: excess " "temporarily rejected: sender=<%s> %s", sender_address, @@ -4939,10 +4966,12 @@ while (done <= 0) if (rc == OK) { + BOOL more = pipeline_response(); + if (user_msg) smtp_user_msg(US"250", user_msg); else - smtp_printf("250 Accepted\r\n"); + smtp_printf("250 Accepted\r\n", more); receive_add_recipient(recipient, -1); /* Set the dsn flags in the recipients_list */ @@ -4961,7 +4990,7 @@ while (done <= 0) if (user_msg) smtp_user_msg(US"250", user_msg); else - smtp_printf("250 Accepted\r\n"); + smtp_printf("250 Accepted\r\n", FALSE); rcpt_fail_count++; discarded = TRUE; log_write(0, LOG_MAIN|LOG_REJECT, "%s F=<%s> RCPT %s: " @@ -5054,7 +5083,7 @@ while (done <= 0) smtp_respond(code, 3, FALSE, rcpt_smtp_response); } if (pipelining_advertised && last_was_rcpt) - smtp_printf("503 Valid RCPT command must precede %s\r\n", + smtp_printf("503 Valid RCPT command must precede %s\r\n", FALSE, smtp_names[smtp_connection_had[smtp_ch_index-1]]); else done = synprot_error(L_smtp_protocol_error, 503, NULL, @@ -5071,7 +5100,7 @@ while (done <= 0) { sender_address = NULL; /* This will allow a new MAIL without RSET */ sender_address_unrewritten = NULL; - smtp_printf("554 Too many recipients\r\n"); + smtp_printf("554 Too many recipients\r\n", FALSE); break; } @@ -5106,7 +5135,7 @@ while (done <= 0) smtp_user_msg(US"354", user_msg); else smtp_printf( - "354 Enter message, ending with \".\" on a line by itself\r\n"); + "354 Enter message, ending with \".\" on a line by itself\r\n", FALSE); } #ifdef TCP_QUICKACK @@ -5129,7 +5158,7 @@ while (done <= 0) if (!(address = parse_extract_address(smtp_cmd_data, &errmess, &start, &end, &recipient_domain, FALSE))) { - smtp_printf("501 %s\r\n", errmess); + smtp_printf("501 %s\r\n", FALSE, errmess); break; } @@ -5168,7 +5197,7 @@ while (done <= 0) break; } - smtp_printf("%s\r\n", s); + smtp_printf("%s\r\n", FALSE, s); } break; } @@ -5297,7 +5326,7 @@ while (done <= 0) if (rc == DEFER) { - smtp_printf("454 TLS currently unavailable\r\n"); + smtp_printf("454 TLS currently unavailable\r\n", FALSE); break; } @@ -5330,14 +5359,14 @@ while (done <= 0) if (user_msg) smtp_respond(US"221", 3, TRUE, user_msg); else - smtp_printf("221 %s closing connection\r\n", smtp_active_hostname); + smtp_printf("221 %s closing connection\r\n", FALSE, smtp_active_hostname); log_write(L_smtp_connection, LOG_MAIN, "%s closed by QUIT", smtp_get_connection_info()); done = 2; break; default: - smtp_printf("554 Security failure\r\n"); + smtp_printf("554 Security failure\r\n", FALSE); break; } tls_close(TRUE, TRUE); @@ -5365,7 +5394,7 @@ while (done <= 0) case NOOP_CMD: HAD(SCH_NOOP); - smtp_printf("250 OK\r\n"); + smtp_printf("250 OK\r\n", FALSE); break; @@ -5376,7 +5405,7 @@ while (done <= 0) case HELP_CMD: HAD(SCH_HELP); - smtp_printf("214-Commands supported:\r\n"); + smtp_printf("214-Commands supported:\r\n", TRUE); { uschar buffer[256]; buffer[0] = 0; @@ -5391,7 +5420,7 @@ while (done <= 0) if (acl_smtp_etrn != NULL) Ustrcat(buffer, " ETRN"); if (acl_smtp_expn != NULL) Ustrcat(buffer, " EXPN"); if (acl_smtp_vrfy != NULL) Ustrcat(buffer, " VRFY"); - smtp_printf("214%s\r\n", buffer); + smtp_printf("214%s\r\n", FALSE, buffer); } break; @@ -5460,7 +5489,7 @@ while (done <= 0) { log_write(0, LOG_MAIN|LOG_PANIC, "failed to set up ETRN command: %s", error); - smtp_printf("458 Internal failure\r\n"); + smtp_printf("458 Internal failure\r\n", FALSE); break; } } @@ -5491,7 +5520,7 @@ while (done <= 0) debug_printf("ETRN command is: %s\n", etrn_command); debug_printf("ETRN command execution skipped\n"); } - if (user_msg == NULL) smtp_printf("250 OK\r\n"); + if (user_msg == NULL) smtp_printf("250 OK\r\n", FALSE); else smtp_user_msg(US"250", user_msg); break; } @@ -5502,7 +5531,7 @@ while (done <= 0) if (smtp_etrn_serialize && !enq_start(etrn_serialize_key, 1)) { - smtp_printf("458 Already processing %s\r\n", smtp_cmd_data); + smtp_printf("458 Already processing %s\r\n", FALSE, smtp_cmd_data); break; } @@ -5565,12 +5594,12 @@ while (done <= 0) { log_write(0, LOG_MAIN|LOG_PANIC, "fork of process for ETRN failed: %s", strerror(errno)); - smtp_printf("458 Unable to fork process\r\n"); + smtp_printf("458 Unable to fork process\r\n", FALSE); if (smtp_etrn_serialize) enq_end(etrn_serialize_key); } else { - if (user_msg == NULL) smtp_printf("250 OK\r\n"); + if (user_msg == NULL) smtp_printf("250 OK\r\n", FALSE); else smtp_user_msg(US"250", user_msg); } @@ -5589,7 +5618,7 @@ while (done <= 0) case BADCHAR_CMD: done = synprot_error(L_smtp_syntax_error, 0, NULL, /* Just logs */ US"NULL character(s) present (shown as '?')"); - smtp_printf("501 NULL characters are not allowed in SMTP commands\r\n"); + smtp_printf("501 NULL characters are not allowed in SMTP commands\r\n", FALSE); break; @@ -5626,7 +5655,7 @@ while (done <= 0) #ifdef SUPPORT_PROXY case PROXY_FAIL_IGNORE_CMD: - smtp_printf("503 Command refused, required Proxy negotiation failed\r\n"); + smtp_printf("503 Command refused, required Proxy negotiation failed\r\n", FALSE); break; #endif diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c index 4328cb342..253d7670d 100644 --- a/src/src/smtp_out.c +++ b/src/src/smtp_out.c @@ -323,7 +323,7 @@ pipelining. Argument: outblock the SMTP output block - mode more-expected, or plain + mode further data expected, or plain Returns: TRUE if OK, FALSE on error, with errno set */ @@ -333,18 +333,19 @@ flush_buffer(smtp_outblock * outblock, int mode) { int rc; int n = outblock->ptr - outblock->buffer; +BOOL more = mode == SCMD_MORE; HDEBUG(D_transport|D_acl) debug_printf_indent("cmd buf flush %d bytes%s\n", n, - mode == SCMD_MORE ? " (more expected)" : ""); + more ? " (more expected)" : ""); #ifdef SUPPORT_TLS if (tls_out.active == outblock->sock) - rc = tls_write(FALSE, outblock->buffer, n); + rc = tls_write(FALSE, outblock->buffer, n, more); else #endif rc = send(outblock->sock, outblock->buffer, n, #ifdef MSG_MORE - mode == SCMD_MORE ? MSG_MORE : 0 + more ? MSG_MORE : 0 #else 0 #endif diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c index d623d8e4a..c68d11d72 100644 --- a/src/src/tls-gnu.c +++ b/src/src/tls-gnu.c @@ -60,6 +60,9 @@ require current GnuTLS, then we'll drop support for the ancient libraries). #if GNUTLS_VERSION_NUMBER >= 0x030014 # define SUPPORT_SYSDEFAULT_CABUNDLE #endif +#if GNUTLS_VERSION_NUMBER >= 0x030109 +# define SUPPORT_CORK +#endif #ifndef DISABLE_OCSP # include @@ -1745,7 +1748,7 @@ exim_gnutls_state_st * state = NULL; if (tls_in.active >= 0) { tls_error(US"STARTTLS received after TLS started", "", NULL, errstr); - smtp_printf("554 Already in TLS\r\n"); + smtp_printf("554 Already in TLS\r\n", FALSE); return FAIL; } @@ -1806,7 +1809,7 @@ mode, the fflush() happens when smtp_getc() is called. */ if (!state->tlsp->on_connect) { - smtp_printf("220 TLS go ahead\r\n"); + smtp_printf("220 TLS go ahead\r\n", FALSE); fflush(smtp_out); } @@ -2288,6 +2291,14 @@ if (n > 0) } +BOOL +tls_could_read(void) +{ +return state_server.xfer_buffer_lwm < state_server.xfer_buffer_hwm + || gnutls_record_check_pending(state_server.session) > 0; +} + + /************************************************* @@ -2347,19 +2358,27 @@ Arguments: is_server channel specifier buff buffer of data len number of bytes + more more data expected soon Returns: the number of bytes after a successful write, -1 after a failed write */ int -tls_write(BOOL is_server, const uschar *buff, size_t len) +tls_write(BOOL is_server, const uschar *buff, size_t len, BOOL more) { ssize_t outbytes; size_t left = len; exim_gnutls_state_st *state = is_server ? &state_server : &state_client; +#ifdef SUPPORT_CORK +static BOOL corked = FALSE; + +if (more && !corked) gnutls_record_cork(state->session); +#endif + +DEBUG(D_tls) debug_printf("%s(%p, " SIZE_T_FMT "%s)\n", __FUNCTION__, + buff, left, more ? ", more" : ""); -DEBUG(D_tls) debug_printf("tls_do_write(%p, " SIZE_T_FMT ")\n", buff, left); while (left > 0) { DEBUG(D_tls) debug_printf("gnutls_record_send(SSL, %p, " SIZE_T_FMT ")\n", @@ -2390,6 +2409,14 @@ if (len > INT_MAX) len = INT_MAX; } +#ifdef SUPPORT_CORK +if (more != corked) + { + if (!more) (void) gnutls_record_uncork(state->session, 0); + corked = more; + } +#endif + return (int) len; } diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c index f17d94b5f..7f41c106e 100644 --- a/src/src/tls-openssl.c +++ b/src/src/tls-openssl.c @@ -1875,7 +1875,7 @@ static uschar cipherbuf[256]; if (tls_in.active >= 0) { tls_error(US"STARTTLS received after TLS started", NULL, US"", errstr); - smtp_printf("554 Already in TLS\r\n"); + smtp_printf("554 Already in TLS\r\n", FALSE); return FAIL; } @@ -1959,7 +1959,7 @@ mode, the fflush() happens when smtp_getc() is called. */ SSL_set_session_id_context(server_ssl, sid_ctx, Ustrlen(sid_ctx)); if (!tls_in.on_connect) { - smtp_printf("220 TLS go ahead\r\n"); + smtp_printf("220 TLS go ahead\r\n", FALSE); fflush(smtp_out); } @@ -2480,6 +2480,13 @@ if (n > 0) } +BOOL +tls_could_read(void) +{ +/* XXX no actual inquiry into library; only our buffer */ +return ssl_xfer_buffer_lwm < ssl_xfer_buffer_hwm; +} + /************************************************* * Read bytes from TLS channel * @@ -2533,6 +2540,7 @@ Arguments: is_server channel specifier buff buffer of data len number of bytes + more further data expected soon Returns: the number of bytes after a successful write, -1 after a failed write @@ -2541,14 +2549,14 @@ Used by both server-side and client-side TLS. */ int -tls_write(BOOL is_server, const uschar *buff, size_t len) +tls_write(BOOL is_server, const uschar *buff, size_t len, BOOL more) { int outbytes; int error; int left = len; SSL *ssl = is_server ? server_ssl : client_ssl; -DEBUG(D_tls) debug_printf("tls_do_write(%p, %d)\n", buff, left); +DEBUG(D_tls) debug_printf("%s(%p, %d)\n", __FUNCTION__, buff, left); while (left > 0) { DEBUG(D_tls) debug_printf("SSL_write(SSL, %p, %d)\n", buff, left); diff --git a/src/src/transport.c b/src/src/transport.c index 5bcf8c1ae..04b67f9e9 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -203,6 +203,7 @@ Arguments: tctx transport context: file descriptor or string to write to block block of bytes to write len number of bytes to write + more further data expected soon Returns: TRUE on success, FALSE on failure (with errno preserved); transport_count is incremented by the number of bytes written @@ -233,7 +234,7 @@ for (i = 0; i < 100; i++) { rc = #ifdef SUPPORT_TLS - (tls_out.active == fd) ? tls_write(FALSE, block, len) : + tls_out.active == fd ? tls_write(FALSE, block, len, more) : #endif #ifdef MSG_MORE more ? send(fd, block, len, MSG_MORE) : @@ -250,7 +251,7 @@ for (i = 0; i < 100; i++) rc = #ifdef SUPPORT_TLS - (tls_out.active == fd) ? tls_write(FALSE, block, len) : + tls_out.active == fd ? tls_write(FALSE, block, len, more) : #endif #ifdef MSG_MORE more ? send(fd, block, len, MSG_MORE) : diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 8f1e0bff8..454c0f74d 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -2618,7 +2618,8 @@ for (fd_bits = 3; fd_bits; ) else { for (nbytes = 0; rc - nbytes > 0; nbytes += i) - if ((i = tls_write(FALSE, buf + nbytes, rc - nbytes)) < 0) return; + if ((i = tls_write(FALSE, buf + nbytes, rc - nbytes, FALSE)) < 0) + return; } else if (fd_bits & 2) FD_SET(proxy_fd, &fds); diff --git a/src/src/verify.c b/src/src/verify.c index 6a50af506..3a8f9b6bc 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -1177,7 +1177,7 @@ if(cutthrough.fd < 0) if( #ifdef SUPPORT_TLS - (tls_out.active == cutthrough.fd) ? tls_write(FALSE, ctblock.buffer, n) : + tls_out.active == cutthrough.fd ? tls_write(FALSE, ctblock.buffer, n, FALSE) : #endif send(cutthrough.fd, ctblock.buffer, n, 0) > 0 ) @@ -1517,7 +1517,7 @@ va_list ap; va_start(ap, format); if (smtp_out && (f == smtp_out)) - smtp_vprintf(format, ap); + smtp_vprintf(format, FALSE, ap); else vfprintf(f, format, ap); va_end(ap); diff --git a/test/confs/2007 b/test/confs/2007 index 7b58b73ee..a16b9a57e 100644 --- a/test/confs/2007 +++ b/test/confs/2007 @@ -16,6 +16,7 @@ queue_only queue_run_in_order tls_advertise_hosts = * +tls_require_ciphers = NORMAL:!DHE-RSA:!DHE-DSS:!ECDHE-RSA:!ECDHE-ECDSA:!ECDHE-PSK # Set certificate only if server diff --git a/test/scripts/2000-GnuTLS/2090 b/test/scripts/2000-GnuTLS/2090 index 278f03429..91227e8ca 100644 --- a/test/scripts/2000-GnuTLS/2090 +++ b/test/scripts/2000-GnuTLS/2090 @@ -58,13 +58,7 @@ EHLO rhu.barb ??? 250-PIPELINING ??? 250-CHUNKING ??? 250 HELP -MAIL FROM: -RCPT TO: -BDAT 88 LAST -To: Susan@random.com -From: Sam@random.com -Subject: This is a bodyless test message - +MAIL FROM:\r\nRCPT TO:\r\nBDAT 88 LAST\r\nTo: Susan@random.com\r\nFrom: Sam@random.com\r\nSubject: This is a bodyless test message\r\n ??? 250 ??? 250 ??? 250- diff --git a/test/scripts/2100-OpenSSL/2190 b/test/scripts/2100-OpenSSL/2190 index 54095d49a..e004e5e02 100644 --- a/test/scripts/2100-OpenSSL/2190 +++ b/test/scripts/2100-OpenSSL/2190 @@ -57,13 +57,7 @@ EHLO rhu.barb ??? 250-PIPELINING ??? 250-CHUNKING ??? 250 HELP -MAIL FROM: -RCPT TO: -BDAT 88 LAST -To: Susan@random.com -From: Sam@random.com -Subject: This is a bodyless test message - +MAIL FROM:\r\nRCPT TO:\r\nBDAT 88 LAST\r\nTo: Susan@random.com\r\nFrom: Sam@random.com\r\nSubject: This is a bodyless test message\r\n ??? 250 ??? 250 ??? 250- diff --git a/test/stdout/2090 b/test/stdout/2090 index 7bf70310a..f08abd10b 100644 --- a/test/stdout/2090 +++ b/test/stdout/2090 @@ -89,13 +89,7 @@ Succeeded in starting TLS <<< 250-CHUNKING ??? 250 HELP <<< 250 HELP ->>> MAIL FROM: ->>> RCPT TO: ->>> BDAT 88 LAST ->>> To: Susan@random.com ->>> From: Sam@random.com ->>> Subject: This is a bodyless test message ->>> +>>> MAIL FROM:\r\nRCPT TO:\r\nBDAT 88 LAST\r\nTo: Susan@random.com\r\nFrom: Sam@random.com\r\nSubject: This is a bodyless test message\r\n ??? 250 <<< 250 OK ??? 250 diff --git a/test/stdout/2190 b/test/stdout/2190 index 9d386bdf7..e09556e69 100644 --- a/test/stdout/2190 +++ b/test/stdout/2190 @@ -91,13 +91,7 @@ Succeeded in starting TLS <<< 250-CHUNKING ??? 250 HELP <<< 250 HELP ->>> MAIL FROM: ->>> RCPT TO: ->>> BDAT 88 LAST ->>> To: Susan@random.com ->>> From: Sam@random.com ->>> Subject: This is a bodyless test message ->>> +>>> MAIL FROM:\r\nRCPT TO:\r\nBDAT 88 LAST\r\nTo: Susan@random.com\r\nFrom: Sam@random.com\r\nSubject: This is a bodyless test message\r\n ??? 250 <<< 250 OK ??? 250 -- cgit v1.2.3 From 014de25aba58181f2a3e6e3dfc21fb2acd82b6ce Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Tue, 30 May 2017 23:43:09 +0100 Subject: Testsuite: stabilize output sequence for case 0906 --- test/confs/0906 | 8 +++++++- test/log/0906 | 13 ++++++++----- test/mail/0906.a | 4 ++-- test/scripts/0000-Basic/0906 | 2 ++ test/stdout/0906 | 2 +- 5 files changed, 20 insertions(+), 9 deletions(-) (limited to 'test/stdout') diff --git a/test/confs/0906 b/test/confs/0906 index 2e8d35a91..2c47966ea 100644 --- a/test/confs/0906 +++ b/test/confs/0906 @@ -18,7 +18,7 @@ spool_wireformat = true primary_hostname = testhost.test.ex domainlist local_domains = @ : test.ex -acl_smtp_rcpt = accept +acl_smtp_rcpt = acl_r log_selector = +received_recipients .ifdef _OPT_MAIN_TLS_CERTIFICATE @@ -28,6 +28,12 @@ tls_privatekey = DIR/aux-fixed/cert1 # ----- ACL ----- +begin acl +acl_r: + accept condition = ${if != {$received_port}{PORT_S}} + control = queue_only + accept + # ----- Routers ----- begin routers diff --git a/test/log/0906 b/test/log/0906 index 3034f16fc..12e227677 100644 --- a/test/log/0906 +++ b/test/log/0906 @@ -1,9 +1,12 @@ +1999-03-02 09:44:33 Start queue run: pid=pppp +1999-03-02 09:44:33 10HmaX-0005vi-00 => a R=localuser T=local_delivery +1999-03-02 09:44:33 10HmaX-0005vi-00 Completed +1999-03-02 09:44:33 End queue run: pid=pppp ******** SERVER ******** 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225 port 1224 -1999-03-02 09:44:33 10HmaX-0005vi-00 <= sender@dom H=(test.com) [127.0.0.1] P=esmtp K S=sss for a@test.ex -1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for a@test.ex -1999-03-02 09:44:33 10HmaY-0005vi-00 => a R=localuser T=local_delivery +1999-03-02 09:44:33 10HmaY-0005vi-00 <= sender@dom H=(test.com) [127.0.0.1] P=esmtp K S=sss for a@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for a@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 no immediate delivery: queued by ACL +1999-03-02 09:44:33 10HmaY-0005vi-00 => a@test.ex R=to_server T=remote_smtp H=127.0.0.1 [127.0.0.1] K C="250- 8nn byte chunk, total 8nn\\n250 OK id=10HmaX-0005vi-00" 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed -1999-03-02 09:44:33 10HmaX-0005vi-00 => a@test.ex R=to_server T=remote_smtp H=127.0.0.1 [127.0.0.1] K C="250- 8nn byte chunk, total 8nn\\n250 OK id=10HmaY-0005vi-00" -1999-03-02 09:44:33 10HmaX-0005vi-00 Completed diff --git a/test/mail/0906.a b/test/mail/0906.a index e5a07fc2d..a55f68d7f 100644 --- a/test/mail/0906.a +++ b/test/mail/0906.a @@ -2,12 +2,12 @@ From MAILER-DAEMON Tue Mar 02 09:44:33 1999 Return-path: <> Received: from localhost ([127.0.0.1] helo=testhost.test.ex) by testhost.test.ex with esmtp (Exim x.yz) - id 10HmaY-0005vi-00 + id 10HmaX-0005vi-00 for a@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 Received: from [127.0.0.1] (helo=test.com) by testhost.test.ex with esmtp (Exim x.yz) (envelope-from ) - id 10HmaX-0005vi-00 + id 10HmaY-0005vi-00 for a@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 Subject: foo X-body-linecount: 0 diff --git a/test/scripts/0000-Basic/0906 b/test/scripts/0000-Basic/0906 index f80899fee..18ab5bb54 100644 --- a/test/scripts/0000-Basic/0906 +++ b/test/scripts/0000-Basic/0906 @@ -119,4 +119,6 @@ QUIT # sleep 1 killdaemon +exim -q +**** no_msglog_check diff --git a/test/stdout/0906 b/test/stdout/0906 index 57be5fe3d..0eb13cc3f 100644 --- a/test/stdout/0906 +++ b/test/stdout/0906 @@ -118,6 +118,6 @@ Connecting to 127.0.0.1 port 1224 ... connected ??? 250- <<< 250- 8408 byte chunk, total 8408 ??? 250 -<<< 250 OK id=10HmaX-0005vi-00 +<<< 250 OK id=10HmaY-0005vi-00 >>> QUIT End of script -- cgit v1.2.3 From 1c3c17d1739290508bbc7644454027f8617ce59a Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Wed, 31 May 2017 17:31:37 +0100 Subject: Testsuite: stabilize output sequence for case 4530 --- test/confs/0906 | 2 ++ test/log/4530 | 28 ++++++++++++++++------------ test/mail/4530.y | 4 ++-- test/mail/4530.z | 4 ++-- test/scripts/4500-DKIM/4530 | 2 ++ test/stdout/4530 | 4 ++-- 6 files changed, 26 insertions(+), 18 deletions(-) (limited to 'test/stdout') diff --git a/test/confs/0906 b/test/confs/0906 index 2c47966ea..0c7b83d5e 100644 --- a/test/confs/0906 +++ b/test/confs/0906 @@ -26,6 +26,8 @@ tls_certificate = DIR/aux-fixed/cert1 tls_privatekey = DIR/aux-fixed/cert1 .endif +queue_run_in_order = true + # ----- ACL ----- begin acl diff --git a/test/log/4530 b/test/log/4530 index 24d4253aa..f4ff0117c 100644 --- a/test/log/4530 +++ b/test/log/4530 @@ -1,17 +1,21 @@ +1999-03-02 09:44:33 Start queue run: pid=pppp +1999-03-02 09:44:33 10HmaX-0005vi-00 => z R=localuser T=local_delivery +1999-03-02 09:44:33 10HmaX-0005vi-00 Completed +1999-03-02 09:44:33 10HmaY-0005vi-00 => y R=localuser T=local_delivery +1999-03-02 09:44:33 10HmaY-0005vi-00 Completed +1999-03-02 09:44:33 End queue run: pid=pppp ******** SERVER ******** 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1224 port 1225 -1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@bloggs.com H=(xxx) [127.0.0.1] P=esmtp S=sss for z@test.ex +1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@bloggs.com H=(xxx) [127.0.0.1] P=esmtp S=sss for z@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 DKIM: d=test.ex s=sel c=relaxed/relaxed a=rsa-sha256 b=1024 [verification succeeded] +1999-03-02 09:44:33 10HmaX-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for z@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 no immediate delivery: queued by ACL +1999-03-02 09:44:33 10HmaZ-0005vi-00 => z@test.ex R=to_server T=remote_smtp_dkim H=127.0.0.1 [127.0.0.1] K C="250- 6nn byte chunk, total 6nn\\n250 OK id=10HmaX-0005vi-00" +1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed +1999-03-02 09:44:33 10HmbA-0005vi-00 <= CALLER@bloggs.com H=(xxx) [127.0.0.1] P=esmtp K S=sss for y@test.ex 1999-03-02 09:44:33 10HmaY-0005vi-00 DKIM: d=test.ex s=sel c=relaxed/relaxed a=rsa-sha256 b=1024 [verification succeeded] -1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for z@test.ex -1999-03-02 09:44:33 10HmaY-0005vi-00 => z R=localuser T=local_delivery -1999-03-02 09:44:33 10HmaY-0005vi-00 Completed -1999-03-02 09:44:33 10HmaX-0005vi-00 => z@test.ex R=to_server T=remote_smtp_dkim H=127.0.0.1 [127.0.0.1] K C="250- 6nn byte chunk, total 6nn\\n250 OK id=10HmaY-0005vi-00" -1999-03-02 09:44:33 10HmaX-0005vi-00 Completed -1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@bloggs.com H=(xxx) [127.0.0.1] P=esmtp K S=sss for y@test.ex -1999-03-02 09:44:33 10HmbA-0005vi-00 DKIM: d=test.ex s=sel c=relaxed/relaxed a=rsa-sha256 b=1024 [verification succeeded] -1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for y@test.ex -1999-03-02 09:44:33 10HmbA-0005vi-00 => y R=localuser T=local_delivery +1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> H=localhost (testhost.test.ex) [127.0.0.1] P=esmtp K S=sss for y@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 no immediate delivery: queued by ACL +1999-03-02 09:44:33 10HmbA-0005vi-00 => y@test.ex R=to_server T=remote_smtp_dkim H=127.0.0.1 [127.0.0.1] K C="250- 6nn byte chunk, total 6nn\\n250 OK id=10HmaY-0005vi-00" 1999-03-02 09:44:33 10HmbA-0005vi-00 Completed -1999-03-02 09:44:33 10HmaZ-0005vi-00 => y@test.ex R=to_server T=remote_smtp_dkim H=127.0.0.1 [127.0.0.1] K C="250- 6nn byte chunk, total 6nn\\n250 OK id=10HmbA-0005vi-00" -1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed diff --git a/test/mail/4530.y b/test/mail/4530.y index 580e21dc7..445e41a95 100644 --- a/test/mail/4530.y +++ b/test/mail/4530.y @@ -2,7 +2,7 @@ From MAILER-DAEMON Tue Mar 02 09:44:33 1999 Return-path: <> Received: from localhost ([127.0.0.1] helo=testhost.test.ex) by testhost.test.ex with esmtp (Exim x.yz) - id 10HmbA-0005vi-00 + id 10HmaY-0005vi-00 for y@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=test.ex; s=sel; h=LIST; bh=CVpkzY75tV/NCKk5pPx4GnM3NX83xwCiT0xVwo0G1Rs=; b=TIqPqpKM5qf @@ -12,7 +12,7 @@ DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=test.ex; Received: from [127.0.0.1] (helo=xxx) by testhost.test.ex with esmtp (Exim x.yz) (envelope-from ) - id 10HmaZ-0005vi-00 + id 10HmbA-0005vi-00 for y@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 Subject: simple test X-body-linecount: 0 diff --git a/test/mail/4530.z b/test/mail/4530.z index 2e74291f5..1b4735507 100644 --- a/test/mail/4530.z +++ b/test/mail/4530.z @@ -2,7 +2,7 @@ From MAILER-DAEMON Tue Mar 02 09:44:33 1999 Return-path: <> Received: from localhost ([127.0.0.1] helo=testhost.test.ex) by testhost.test.ex with esmtp (Exim x.yz) - id 10HmaY-0005vi-00 + id 10HmaX-0005vi-00 for z@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=test.ex; s=sel; h=LIST; bh=CVpkzY75tV/NCKk5pPx4GnM3NX83xwCiT0xVwo0G1Rs=; b=TIqPqpKM5qf @@ -12,7 +12,7 @@ DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=test.ex; Received: from [127.0.0.1] (helo=xxx) by testhost.test.ex with esmtp (Exim x.yz) (envelope-from ) - id 10HmaX-0005vi-00 + id 10HmaZ-0005vi-00 for z@test.ex; Tue, 2 Mar 1999 09:44:33 +0000 Subject: simple test X-body-linecount: 0 diff --git a/test/scripts/4500-DKIM/4530 b/test/scripts/4500-DKIM/4530 index 822f2fe02..1465d5896 100644 --- a/test/scripts/4500-DKIM/4530 +++ b/test/scripts/4500-DKIM/4530 @@ -63,4 +63,6 @@ QUIT sleep 1 # killdaemon +exim -q +**** no_msglog_check diff --git a/test/stdout/4530 b/test/stdout/4530 index a1f5cb471..9d64ae5f5 100644 --- a/test/stdout/4530 +++ b/test/stdout/4530 @@ -31,7 +31,7 @@ Connecting to 127.0.0.1 port 1224 ... connected >>> last line: 4 >>> . ??? 250 -<<< 250 OK id=10HmaX-0005vi-00 +<<< 250 OK id=10HmaZ-0005vi-00 >>> QUIT ??? 221 <<< 221 testhost.test.ex closing connection @@ -68,7 +68,7 @@ Connecting to 127.0.0.1 port 1224 ... connected ??? 250- <<< 250- 129 byte chunk, total 129 ??? 250 -<<< 250 OK id=10HmaZ-0005vi-00 +<<< 250 OK id=10HmbA-0005vi-00 >>> QUIT ??? 221 <<< 221 testhost.test.ex closing connection -- cgit v1.2.3 From e69636bc9ddf3617be688b07941d7d659d50eaa7 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 3 Jun 2017 13:39:18 +0100 Subject: Fix crash in transport, on second smtp-connect fail for a list of target hosts Reported as the sequence: 1MX: 554 on connect (banner) 2MX: TCP conn timeout --- doc/doc-txt/ChangeLog | 4 ++++ src/src/transports/smtp.c | 29 +++++++++++++++++++---------- test/confs/0211 | 7 ++++++- test/log/0211 | 8 ++++++++ test/log/0429 | 1 + test/log/4203 | 1 + test/log/4213 | 1 + test/log/4223 | 1 + test/log/4700 | 2 ++ test/scripts/0000-Basic/0211 | 19 +++++++++++++++++++ test/scripts/4700-dsn-info/4700 | 6 ++++++ test/stdout/0211 | 10 ++++++++++ 12 files changed, 78 insertions(+), 11 deletions(-) (limited to 'test/stdout') diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index 4eb0d3b1a..7f2e9d097 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -102,6 +102,10 @@ JH/14 For the "sock" variant of the malware scanner interface, accept an empty cmdline element to get the documented default one. Previously it was inaccessible. +JH/15 Fix a crash in the smtp transport caused when two hosts in succession + are unsuable for non-message-specific reasons - eg. connection timeout, + banner-time rejection. + Exim version 4.89 ----------------- diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 454c0f74d..dc9e03be1 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -2177,25 +2177,34 @@ return OK; /* The failure happened while setting up the call; see if the failure was a 5xx response (this will either be on connection, or following HELO - a 5xx - after EHLO causes it to try HELO). If so, fail all addresses, as this host is - never going to accept them. For other errors during setting up (timeouts or - whatever), defer all addresses, and yield DEFER, so that the host is not - tried again for a while. */ + after EHLO causes it to try HELO). If so, and there are no more hosts to try, + fail all addresses, as this host is never going to accept them. For other + errors during setting up (timeouts or whatever), defer all addresses, and + yield DEFER, so that the host is not tried again for a while. + + XXX This peeking for another host feels like a layering violation. We want + to note the host as unusable, but down here we shouldn't know if this was + the last host to try for the addr(list). Perhaps the upper layer should be + the one to do set_errno() ? The problem is that currently the addr is where + errno etc. are stashed, but until we run out of hosts to try the errors are + host-specific. Maybe we should enhance the host_item definition? */ FAILED: sx->ok = FALSE; /* For when reached by GOTO */ - - yield = code == '5' + set_errno(sx->addrlist, errno, message, + sx->host->next + ? DEFER + : code == '5' #ifdef SUPPORT_I18N - || errno == ERRNO_UTF8_FWD + || errno == ERRNO_UTF8_FWD #endif - ? FAIL : DEFER; - - set_errno(sx->addrlist, errno, message, yield, pass_message, sx->host + ? FAIL : DEFER, + pass_message, sx->host #ifdef EXPERIMENTAL_DSN_INFO , sx->smtp_greeting, sx->helo_response #endif ); + yield = DEFER; } diff --git a/test/confs/0211 b/test/confs/0211 index 96f3beac1..7856481df 100644 --- a/test/confs/0211 +++ b/test/confs/0211 @@ -30,7 +30,11 @@ begin routers others: driver = manualroute domains = ! +local_domains - route_list = * localhost4.test.ex byname +.ifdef LIST + route_data = 127.0.0.1:HOSTIPV4 +.else + route_data = localhost4.test.ex byname +.endif self = send transport = smtp no_more @@ -48,6 +52,7 @@ begin transports smtp: driver = smtp port = PORT_S + command_timeout = 1s local_delivery: driver = appendfile diff --git a/test/log/0211 b/test/log/0211 index 74acec3a3..ac647a67b 100644 --- a/test/log/0211 +++ b/test/log/0211 @@ -14,6 +14,7 @@ 1999-03-02 09:44:33 End queue run: pid=pppp -qf 1999-03-02 09:44:33 Test: reject connect 1999-03-02 09:44:33 Start queue run: pid=pppp -qf +1999-03-02 09:44:33 10HmaX-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away (A) 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userx@domain1 F= R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away (A) 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss 1999-03-02 09:44:33 10HmaY-0005vi-00 => CALLER F=<> R=all T=local_delivery @@ -22,6 +23,7 @@ 1999-03-02 09:44:33 End queue run: pid=pppp -qf 1999-03-02 09:44:33 Test: reject helo 1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss +1999-03-02 09:44:33 10HmaZ-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away (C) 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** userx@domain1 F= R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away (C) 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** usery@domain2 F= R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away (C) 1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> R=10HmaZ-0005vi-00 U=EXIMUSER P=local S=sss @@ -32,3 +34,9 @@ 1999-03-02 09:44:33 10HmbB-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss 1999-03-02 09:44:33 10HmbB-0005vi-00 H=localhost4.test.ex [127.0.0.1]: Remote host closed connection in response to HELO the.local.host.name (EHLO response was: 550 You are banned) 1999-03-02 09:44:33 10HmbB-0005vi-00 == userx@domain1 R=others T=smtp defer (-18) H=localhost4.test.ex [127.0.0.1]: Remote host closed connection in response to HELO the.local.host.name (EHLO response was: 550 You are banned) +1999-03-02 09:44:33 Test: smtp-reject conn on 1MX, timeout TCP conn on 2MX +1999-03-02 09:44:33 Start queue run: pid=pppp -qf +1999-03-02 09:44:33 10HmbB-0005vi-00 H=127.0.0.1 [127.0.0.1]: SMTP error from remote mail server after initial connection: 554 no smtp service here +1999-03-02 09:44:33 10HmbB-0005vi-00 H=ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4]: SMTP timeout after initial connection: Connection timed out +1999-03-02 09:44:33 10HmbB-0005vi-00 == userx@domain1 R=others T=smtp defer (dd): Connection timed out H=ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4]: SMTP timeout after initial connection +1999-03-02 09:44:33 End queue run: pid=pppp -qf diff --git a/test/log/0429 b/test/log/0429 index cb75f6cf4..d477a6b9b 100644 --- a/test/log/0429 +++ b/test/log/0429 @@ -1,5 +1,6 @@ 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss 1999-03-02 09:44:33 Start queue run: pid=pppp +1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: SMTP error from remote mail server after HELO myhost.test.ex: 550 No 1999-03-02 09:44:33 10HmaX-0005vi-00 ** x@y R=r1 T=t1 H=127.0.0.1 [127.0.0.1]: SMTP error from remote mail server after HELO myhost.test.ex: 550 No 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed diff --git a/test/log/4203 b/test/log/4203 index a1acf570e..1c2d52830 100644 --- a/test/log/4203 +++ b/test/log/4203 @@ -1,4 +1,5 @@ 1999-03-02 09:44:33 10HmaX-0005vi-00 <= यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local U=CALLER P=utf8local-esmtp S=sss for userz@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userz@test.ex F=<यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss for यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local 1999-03-02 09:44:33 10HmaY-0005vi-00 no immediate delivery: queued by ACL diff --git a/test/log/4213 b/test/log/4213 index e0c0a43d9..1a9aa5eae 100644 --- a/test/log/4213 +++ b/test/log/4213 @@ -1,4 +1,5 @@ 1999-03-02 09:44:33 10HmaX-0005vi-00 <= यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local U=CALLER P=utf8local-esmtp S=sss for userz@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userz@test.ex F=<यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] X=TLS1.x:xxxxRSA_AES_256_CBC_SHAnnn:256 CV=no: utf8 support required but not offered for forwarding 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss for यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local 1999-03-02 09:44:33 10HmaY-0005vi-00 no immediate delivery: queued by ACL diff --git a/test/log/4223 b/test/log/4223 index fb5a1defe..f4d4efa3a 100644 --- a/test/log/4223 +++ b/test/log/4223 @@ -1,4 +1,5 @@ 1999-03-02 09:44:33 10HmaX-0005vi-00 <= यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local U=CALLER P=utf8local-esmtp S=sss for userz@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userz@test.ex F=<यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] X=TLSv1:AES256-SHA:256 CV=no: utf8 support required but not offered for forwarding 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss for यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local 1999-03-02 09:44:33 10HmaY-0005vi-00 no immediate delivery: queued by ACL diff --git a/test/log/4700 b/test/log/4700 index 35da72e04..8a1a7a4f4 100644 --- a/test/log/4700 +++ b/test/log/4700 @@ -6,6 +6,7 @@ 1999-03-02 09:44:33 10HmaX-0005vi-00 == userx@domain1 R=others T=smtp defer (0) H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 450 I'm busy 1999-03-02 09:44:33 End queue run: pid=pppp -qf 1999-03-02 09:44:33 Start queue run: pid=pppp -qf +1999-03-02 09:44:33 10HmaX-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userx@domain1 F= R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss 1999-03-02 09:44:33 10HmaY-0005vi-00 => CALLER F=<> R=all T=local_delivery @@ -13,6 +14,7 @@ 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed 1999-03-02 09:44:33 End queue run: pid=pppp -qf 1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss +1999-03-02 09:44:33 10HmaZ-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** userx@domain1 F= R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** usery@domain2 F= R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away 1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> R=10HmaZ-0005vi-00 U=EXIMUSER P=local S=sss diff --git a/test/scripts/0000-Basic/0211 b/test/scripts/0000-Basic/0211 index 655b4dd96..c27a7ebf3 100644 --- a/test/scripts/0000-Basic/0211 +++ b/test/scripts/0000-Basic/0211 @@ -13,6 +13,7 @@ exim -odi userx@domain1 Test message 1 **** # +sudo rm DIR/spool/db/* exim -z "Test: temp-rej helo" **** server PORT_S @@ -27,6 +28,7 @@ QUIT exim -qf **** # +sudo rm DIR/spool/db/* exim -z "Test: drop conn after banner" **** server PORT_S @@ -35,6 +37,7 @@ server PORT_S exim -qf **** # +sudo rm DIR/spool/db/* exim -z "Test: reject connect" **** server PORT_S @@ -45,6 +48,7 @@ QUIT exim -qf **** # +sudo rm DIR/spool/db/* exim -z "Test: reject helo" **** server PORT_S @@ -74,4 +78,19 @@ Test message 4 **** # # +# +sudo rm DIR/spool/db/* +exim -z "Test: smtp-reject conn on 1MX, timeout TCP conn on 2MX" +**** +server PORT_S 2 +554 no smtp service here +QUIT +220 bye +*eof +*sleep 2 +**** +exim -DLIST -qf +**** +# +# no_msglog_check diff --git a/test/scripts/4700-dsn-info/4700 b/test/scripts/4700-dsn-info/4700 index d0dd4f363..29b2ce8b4 100644 --- a/test/scripts/4700-dsn-info/4700 +++ b/test/scripts/4700-dsn-info/4700 @@ -9,6 +9,8 @@ QUIT exim -odi userx@domain1 Test message 1 **** +sudo rm DIR/spool/db/* +# server PORT_S 220 Connected OK EHLO @@ -20,6 +22,8 @@ QUIT **** exim -qf **** +sudo rm DIR/spool/db/* +# server PORT_S 550 Go away QUIT @@ -27,6 +31,8 @@ QUIT **** exim -qf **** +sudo rm DIR/spool/db/* +# server PORT_S 220 Connected OK EHLO diff --git a/test/stdout/0211 b/test/stdout/0211 index 2d484bda5..708632ded 100644 --- a/test/stdout/0211 +++ b/test/stdout/0211 @@ -43,3 +43,13 @@ EHLO the.local.host.name 550 You are banned HELO the.local.host.name End of script +Listening on port 1224 ... +Connection request from [127.0.0.1] +554 no smtp service here +QUIT +220 bye +Expected EOF read from client +Listening on port 1224 ... +Connection request from [ip4.ip4.ip4.ip4] +*sleep 2 +End of script -- cgit v1.2.3 From fae8970db0e012deb28c139744583aad49ab9fa4 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Tue, 18 Jul 2017 23:10:33 +0100 Subject: Fix cache-cold random callout verify. Bug 2147 --- doc/doc-txt/ChangeLog | 4 ++++ src/src/verify.c | 4 +++- test/log/0376 | 4 ---- test/log/0578 | 4 ---- test/rejectlog/0376 | 4 ---- test/rejectlog/0578 | 4 ---- test/stderr/0376 | 8 -------- test/stderr/0578 | 8 -------- test/stdout/0376 | 4 ++-- test/stdout/0578 | 4 ++-- 10 files changed, 11 insertions(+), 37 deletions(-) (limited to 'test/stdout') diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index de8151f65..c5de41961 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -128,6 +128,10 @@ JH/19 Bug 2141: Use the full-complex API for Berkeley DB rather than the legacy- file in the working directory redirecting the DB files, possibly correpting some existing file. +JH/20 Bug 2147: Do not defer for a verify-with-callout-and-random which is not + cache-hot. Previously, although the result was properly cached, the + initial verify call returned a defer. + Exim version 4.89 ----------------- diff --git a/src/src/verify.c b/src/src/verify.c index 3a8f9b6bc..cda0b72e2 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -795,7 +795,9 @@ tls_retry_connection: { case PENDING_OK: new_domain_record.random_result = ccache_accept; - break; + yield = OK; /* Only usable result we can return */ + done = TRUE; + goto no_conn; case FAIL: new_domain_record.random_result = ccache_reject; diff --git a/test/log/0376 b/test/log/0376 index 0e2f661ac..fa191bcd5 100644 --- a/test/log/0376 +++ b/test/log/0376 @@ -12,10 +12,6 @@ 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root sender verify fail for 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root F= temporarily rejected RCPT : Could not complete sender verify callout -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root F= temporarily rejected RCPT : Could not complete sender verify callout 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : SMTP timeout after RCPT TO: 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= temporarily rejected RCPT : Could not complete sender verify callout 1999-03-02 09:44:33 10HmaX-0005vi-00 <= ok7@otherhost53 H=[V4NET.0.0.7] U=root P=smtp S=sss diff --git a/test/log/0578 b/test/log/0578 index 7c38e638c..60ddf3c13 100644 --- a/test/log/0578 +++ b/test/log/0578 @@ -12,10 +12,6 @@ 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root sender verify fail for 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK accepting that random recipient -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root F= temporarily rejected RCPT : Could not complete sender verify callout -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root F= temporarily rejected RCPT : Could not complete sender verify callout 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : SMTP timeout after RCPT TO: 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= temporarily rejected RCPT : Could not complete sender verify callout 1999-03-02 09:44:33 10HmaX-0005vi-00 <= ok7@otherhost53 H=[V4NET.0.0.7] U=root P=smtp S=sss diff --git a/test/rejectlog/0376 b/test/rejectlog/0376 index 47ad1638d..1702cec04 100644 --- a/test/rejectlog/0376 +++ b/test/rejectlog/0376 @@ -12,9 +12,5 @@ 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root sender verify fail for 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root F= temporarily rejected RCPT : Could not complete sender verify callout -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root F= temporarily rejected RCPT : Could not complete sender verify callout 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : SMTP timeout after RCPT TO: 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= temporarily rejected RCPT : Could not complete sender verify callout diff --git a/test/rejectlog/0578 b/test/rejectlog/0578 index db94a4d6e..bf20a240d 100644 --- a/test/rejectlog/0578 +++ b/test/rejectlog/0578 @@ -12,9 +12,5 @@ 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root sender verify fail for 1999-03-02 09:44:33 H=[V4NET.0.0.2] U=root F= rejected RCPT : Sender verify failed -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK accepting that random recipient -1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root F= temporarily rejected RCPT : Could not complete sender verify callout -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -1999-03-02 09:44:33 H=[V4NET.0.0.4] U=root F= temporarily rejected RCPT : Could not complete sender verify callout 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : SMTP timeout after RCPT TO: 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= temporarily rejected RCPT : Could not complete sender verify callout diff --git a/test/stderr/0376 b/test/stderr/0376 index bcddefdfa..abd462b01 100644 --- a/test/stderr/0376 +++ b/test/stderr/0376 @@ -323,10 +323,6 @@ Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected SMTP(close)>> wrote callout cache domain record for otherhost3: result=1 postmaster=0 random=1 -LOG: MAIN REJECT - H=[V4NET.0.0.3] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -LOG: MAIN REJECT - H=[V4NET.0.0.3] U=root F= temporarily rejected RCPT : Could not complete sender verify callout LOG: smtp_connection MAIN SMTP connection from root closed by QUIT >>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>> @@ -373,10 +369,6 @@ Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected SMTP(close)>> wrote callout cache domain record for otherhost4: result=1 postmaster=0 random=1 -LOG: MAIN REJECT - H=[V4NET.0.0.4] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -LOG: MAIN REJECT - H=[V4NET.0.0.4] U=root F= temporarily rejected RCPT : Could not complete sender verify callout LOG: smtp_connection MAIN SMTP connection from root closed by QUIT >>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>> diff --git a/test/stderr/0578 b/test/stderr/0578 index 74581fd9c..9f634c8c4 100644 --- a/test/stderr/0578 +++ b/test/stderr/0578 @@ -337,10 +337,6 @@ Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected SMTP(close)>> wrote callout cache domain record for otherhost3: result=1 postmaster=0 random=1 -LOG: MAIN REJECT - H=[V4NET.0.0.3] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK accepting that random recipient -LOG: MAIN REJECT - H=[V4NET.0.0.3] U=root F= temporarily rejected RCPT : Could not complete sender verify callout LOG: smtp_connection MAIN SMTP connection from root closed by QUIT >>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>> @@ -389,10 +385,6 @@ Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected SMTP(close)>> wrote callout cache domain record for otherhost4: result=1 postmaster=0 random=1 -LOG: MAIN REJECT - H=[V4NET.0.0.4] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : response to "RCPT TO:" was: 250 OK -LOG: MAIN REJECT - H=[V4NET.0.0.4] U=root F= temporarily rejected RCPT : Could not complete sender verify callout LOG: smtp_connection MAIN SMTP connection from root closed by QUIT >>>>>>>>>>>>>>>> Exim pid=pppp terminating with rc=0 >>>>>>>>>>>>>>>> diff --git a/test/stdout/0376 b/test/stdout/0376 index 8c2b9685c..a0af2cb92 100644 --- a/test/stdout/0376 +++ b/test/stdout/0376 @@ -78,7 +78,7 @@ 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK -451 Could not complete sender verify callout +250 Accepted 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK @@ -86,7 +86,7 @@ 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK -451 Could not complete sender verify callout +250 Accepted 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK diff --git a/test/stdout/0578 b/test/stdout/0578 index 40b265041..d3089e7e4 100644 --- a/test/stdout/0578 +++ b/test/stdout/0578 @@ -78,7 +78,7 @@ 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK -451 Could not complete sender verify callout +250 Accepted 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK @@ -86,7 +86,7 @@ 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK -451 Could not complete sender verify callout +250 Accepted 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK -- cgit v1.2.3 From 14de8063d82edc5bf003ed50abdea55ac542679b Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 22 Jul 2017 16:38:08 +0100 Subject: On callout avoid SIZE every time but noncacheable rcpt main-verify. Bug 2151 --- doc/doc-txt/ChangeLog | 4 ++ src/src/deliver.c | 2 +- src/src/exim.c | 10 ++-- src/src/macros.h | 16 +++--- src/src/transport.c | 10 ++-- src/src/transports/smtp.c | 126 ++++++++++++++++++++++--------------------- src/src/transports/smtp.h | 1 + src/src/verify.c | 20 ++++--- test/confs/0227 | 2 + test/log/0227 | 3 ++ test/rejectlog/0227 | 3 ++ test/scripts/0000-Basic/0227 | 84 ++++++++++++++++++++++++++++- test/stderr/0227 | 67 ++++++++++++++++++++++- test/stderr/2035 | 2 +- test/stderr/2135 | 2 +- test/stderr/5410 | 6 +-- test/stderr/5420 | 6 +-- test/stderr/5840 | 4 +- test/stdout/0227 | 79 ++++++++++++++++++++++++++- 19 files changed, 344 insertions(+), 103 deletions(-) (limited to 'test/stdout') diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index c5de41961..cc0b90134 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -132,6 +132,10 @@ JH/20 Bug 2147: Do not defer for a verify-with-callout-and-random which is not cache-hot. Previously, although the result was properly cached, the initial verify call returned a defer. +JH/21 Bug 2151 (partial): + Avoid using SIZE on the MAIL for a callout verify, on any but + the main verify for receipient in uncached-mode. + Exim version 4.89 ----------------- diff --git a/src/src/deliver.c b/src/src/deliver.c index bf465c977..49281f93d 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -8474,7 +8474,7 @@ if (cutthrough.fd >= 0 && cutthrough.callout_hold_only) #ifdef SUPPORT_TLS if (cutthrough.is_tls) { - smtp_peer_options |= PEER_OFFERED_TLS; + smtp_peer_options |= OPTION_TLS; sending_ip_address = cutthrough.snd_ip; sending_port = cutthrough.snd_port; diff --git a/src/src/exim.c b/src/src/exim.c index 0285e162a..37cf8bc6e 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -2738,7 +2738,7 @@ for (i = 1; i < argc; i++) /* -MCD: set the smtp_use_dsn flag; this indicates that the host that exim is connected to supports the esmtp extension DSN */ - case 'D': smtp_peer_options |= PEER_OFFERED_DSN; break; + case 'D': smtp_peer_options |= OPTION_DSN; break; /* -MCG: set the queue name, to a non-default value */ @@ -2748,12 +2748,12 @@ for (i = 1; i < argc; i++) /* -MCK: the peer offered CHUNKING. Must precede -MC */ - case 'K': smtp_peer_options |= PEER_OFFERED_CHUNKING; break; + case 'K': smtp_peer_options |= OPTION_CHUNKING; break; /* -MCP: set the smtp_use_pipelining flag; this is useful only when it preceded -MC (see above) */ - case 'P': smtp_peer_options |= PEER_OFFERED_PIPE; break; + case 'P': smtp_peer_options |= OPTION_PIPE; break; /* -MCQ: pass on the pid of the queue-running process that started this chain of deliveries and the fd of its synchronizing pipe; this @@ -2768,7 +2768,7 @@ for (i = 1; i < argc; i++) /* -MCS: set the smtp_use_size flag; this is useful only when it precedes -MC (see above) */ - case 'S': smtp_peer_options |= PEER_OFFERED_SIZE; break; + case 'S': smtp_peer_options |= OPTION_SIZE; break; #ifdef SUPPORT_TLS /* -MCt: similar to -MCT below but the connection is still open @@ -2788,7 +2788,7 @@ for (i = 1; i < argc; i++) precedes -MC (see above). The flag indicates that the host to which Exim is connected has offered TLS support. */ - case 'T': smtp_peer_options |= PEER_OFFERED_TLS; break; + case 'T': smtp_peer_options |= OPTION_TLS; break; #endif default: badarg = TRUE; break; diff --git a/src/src/macros.h b/src/src/macros.h index 8b608f7f8..a8022adf6 100644 --- a/src/src/macros.h +++ b/src/src/macros.h @@ -972,14 +972,14 @@ enum { FILTER_UNSET, FILTER_FORWARD, FILTER_EXIM, FILTER_SIEVE }; /* Codes for ESMTP facilities offered by peer */ -#define PEER_OFFERED_TLS BIT(0) -#define PEER_OFFERED_IGNQ BIT(1) -#define PEER_OFFERED_PRDR BIT(2) -#define PEER_OFFERED_UTF8 BIT(3) -#define PEER_OFFERED_DSN BIT(4) -#define PEER_OFFERED_PIPE BIT(5) -#define PEER_OFFERED_SIZE BIT(6) -#define PEER_OFFERED_CHUNKING BIT(7) +#define OPTION_TLS BIT(0) +#define OPTION_IGNQ BIT(1) +#define OPTION_PRDR BIT(2) +#define OPTION_UTF8 BIT(3) +#define OPTION_DSN BIT(4) +#define OPTION_PIPE BIT(5) +#define OPTION_SIZE BIT(6) +#define OPTION_CHUNKING BIT(7) /* Argument for *_getc */ diff --git a/src/src/transport.c b/src/src/transport.c index 6590fa7fb..b26251c88 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -1866,12 +1866,12 @@ but we have a number of extras that may be added. */ argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0); if (smtp_authenticated) argv[i++] = US"-MCA"; -if (smtp_peer_options & PEER_OFFERED_CHUNKING) argv[i++] = US"-MCK"; -if (smtp_peer_options & PEER_OFFERED_DSN) argv[i++] = US"-MCD"; -if (smtp_peer_options & PEER_OFFERED_PIPE) argv[i++] = US"-MCP"; -if (smtp_peer_options & PEER_OFFERED_SIZE) argv[i++] = US"-MCS"; +if (smtp_peer_options & OPTION_CHUNKING) argv[i++] = US"-MCK"; +if (smtp_peer_options & OPTION_DSN) argv[i++] = US"-MCD"; +if (smtp_peer_options & OPTION_PIPE) argv[i++] = US"-MCP"; +if (smtp_peer_options & OPTION_SIZE) argv[i++] = US"-MCS"; #ifdef SUPPORT_TLS -if (smtp_peer_options & PEER_OFFERED_TLS) +if (smtp_peer_options & OPTION_TLS) if (tls_out.active >= 0 || continue_proxy_cipher) { argv[i++] = US"-MCt"; diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 477cdac4d..11a996c65 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -1316,44 +1316,44 @@ ehlo_response(uschar * buf, uschar checks) size_t bsize = Ustrlen(buf); #ifdef SUPPORT_TLS -if ( checks & PEER_OFFERED_TLS +if ( checks & OPTION_TLS && pcre_exec(regex_STARTTLS, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_TLS; + checks &= ~OPTION_TLS; #endif -if ( checks & PEER_OFFERED_IGNQ +if ( checks & OPTION_IGNQ && pcre_exec(regex_IGNOREQUOTA, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_IGNQ; + checks &= ~OPTION_IGNQ; -if ( checks & PEER_OFFERED_CHUNKING +if ( checks & OPTION_CHUNKING && pcre_exec(regex_CHUNKING, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_CHUNKING; + checks &= ~OPTION_CHUNKING; #ifndef DISABLE_PRDR -if ( checks & PEER_OFFERED_PRDR +if ( checks & OPTION_PRDR && pcre_exec(regex_PRDR, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_PRDR; + checks &= ~OPTION_PRDR; #endif #ifdef SUPPORT_I18N -if ( checks & PEER_OFFERED_UTF8 +if ( checks & OPTION_UTF8 && pcre_exec(regex_UTF8, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_UTF8; + checks &= ~OPTION_UTF8; #endif -if ( checks & PEER_OFFERED_DSN +if ( checks & OPTION_DSN && pcre_exec(regex_DSN, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_DSN; + checks &= ~OPTION_DSN; -if ( checks & PEER_OFFERED_PIPE +if ( checks & OPTION_PIPE && pcre_exec(regex_PIPELINING, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_PIPE; + checks &= ~OPTION_PIPE; -if ( checks & PEER_OFFERED_SIZE +if ( checks & OPTION_SIZE && pcre_exec(regex_SIZE, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) - checks &= ~PEER_OFFERED_SIZE; + checks &= ~OPTION_SIZE; return checks; } @@ -1512,6 +1512,7 @@ sx->dane_required = verify_check_given_host(&sx->ob->hosts_require_dane, sx->hos if ((sx->max_rcpt = sx->tblock->max_addresses) == 0) sx->max_rcpt = 999999; sx->peer_offered = 0; +sx->avoid_option = 0; sx->igquotstr = US""; if (!sx->helo_data) sx->helo_data = sx->ob->helo_data; #ifdef EXPERIMENTAL_DSN_INFO @@ -1753,7 +1754,7 @@ goto SEND_QUIT; #ifdef SUPPORT_TLS if (sx->smtps) { - smtp_peer_options |= PEER_OFFERED_TLS; + smtp_peer_options |= OPTION_TLS; suppress_tls = FALSE; sx->ob->tls_tempfail_tryclear = FALSE; smtp_command = US"SSL-on-connect"; @@ -1818,18 +1819,18 @@ goto SEND_QUIT; } } - sx->peer_offered = smtp_peer_options = 0; + sx->avoid_option = sx->peer_offered = smtp_peer_options = 0; if (sx->esmtp || sx->lmtp) { sx->peer_offered = ehlo_response(sx->buffer, - PEER_OFFERED_TLS /* others checked later */ + OPTION_TLS /* others checked later */ ); /* Set tls_offered if the response to EHLO specifies support for STARTTLS. */ #ifdef SUPPORT_TLS - smtp_peer_options |= sx->peer_offered & PEER_OFFERED_TLS; + smtp_peer_options |= sx->peer_offered & OPTION_TLS; #endif } } @@ -1871,7 +1872,7 @@ else ) { sx->peer_offered = smtp_peer_options; - pipelining_active = !!(smtp_peer_options & PEER_OFFERED_PIPE); + pipelining_active = !!(smtp_peer_options & OPTION_PIPE); HDEBUG(D_transport) debug_printf("continued connection, %s TLS\n", continue_proxy_cipher ? "proxied" : "verify conn with"); return OK; @@ -1888,7 +1889,7 @@ the client not be required to use TLS. If the response is bad, copy the buffer for error analysis. */ #ifdef SUPPORT_TLS -if ( smtp_peer_options & PEER_OFFERED_TLS +if ( smtp_peer_options & OPTION_TLS && !suppress_tls && verify_check_given_host(&sx->ob->hosts_avoid_tls, sx->host) != OK && ( !sx->verify @@ -2034,7 +2035,7 @@ else if ( sx->smtps { errno = ERRNO_TLSREQUIRED; message = string_sprintf("a TLS session is required, but %s", - smtp_peer_options & PEER_OFFERED_TLS + smtp_peer_options & OPTION_TLS ? "an attempt to start TLS failed" : "the server did not offer TLS support"); goto TLS_FAILED; } @@ -2055,60 +2056,60 @@ if (continue_hostname == NULL { sx->peer_offered = ehlo_response(sx->buffer, 0 /* no TLS */ - | (sx->lmtp && sx->ob->lmtp_ignore_quota ? PEER_OFFERED_IGNQ : 0) - | PEER_OFFERED_CHUNKING - | PEER_OFFERED_PRDR + | (sx->lmtp && sx->ob->lmtp_ignore_quota ? OPTION_IGNQ : 0) + | OPTION_CHUNKING + | OPTION_PRDR #ifdef SUPPORT_I18N - | (sx->addrlist->prop.utf8_msg ? PEER_OFFERED_UTF8 : 0) + | (sx->addrlist->prop.utf8_msg ? OPTION_UTF8 : 0) /*XXX if we hand peercaps on to continued-conn processes, must not depend on this addr */ #endif - | PEER_OFFERED_DSN - | PEER_OFFERED_PIPE - | (sx->ob->size_addition >= 0 ? PEER_OFFERED_SIZE : 0) + | OPTION_DSN + | OPTION_PIPE + | (sx->ob->size_addition >= 0 ? OPTION_SIZE : 0) ); /* Set for IGNOREQUOTA if the response to LHLO specifies support and the lmtp_ignore_quota option was set. */ - sx->igquotstr = sx->peer_offered & PEER_OFFERED_IGNQ ? US" IGNOREQUOTA" : US""; + sx->igquotstr = sx->peer_offered & OPTION_IGNQ ? US" IGNOREQUOTA" : US""; /* If the response to EHLO specified support for the SIZE parameter, note this, provided size_addition is non-negative. */ - smtp_peer_options |= sx->peer_offered & PEER_OFFERED_SIZE; + smtp_peer_options |= sx->peer_offered & OPTION_SIZE; /* Note whether the server supports PIPELINING. If hosts_avoid_esmtp matched the current host, esmtp will be false, so PIPELINING can never be used. If the current host matches hosts_avoid_pipelining, don't do it. */ - if ( sx->peer_offered & PEER_OFFERED_PIPE + if ( sx->peer_offered & OPTION_PIPE && verify_check_given_host(&sx->ob->hosts_avoid_pipelining, sx->host) != OK) - smtp_peer_options |= PEER_OFFERED_PIPE; + smtp_peer_options |= OPTION_PIPE; DEBUG(D_transport) debug_printf("%susing PIPELINING\n", - smtp_peer_options & PEER_OFFERED_PIPE ? "" : "not "); + smtp_peer_options & OPTION_PIPE ? "" : "not "); - if ( sx->peer_offered & PEER_OFFERED_CHUNKING + if ( sx->peer_offered & OPTION_CHUNKING && verify_check_given_host(&sx->ob->hosts_try_chunking, sx->host) != OK) - sx->peer_offered &= ~PEER_OFFERED_CHUNKING; + sx->peer_offered &= ~OPTION_CHUNKING; - if (sx->peer_offered & PEER_OFFERED_CHUNKING) + if (sx->peer_offered & OPTION_CHUNKING) {DEBUG(D_transport) debug_printf("CHUNKING usable\n");} #ifndef DISABLE_PRDR - if ( sx->peer_offered & PEER_OFFERED_PRDR + if ( sx->peer_offered & OPTION_PRDR && verify_check_given_host(&sx->ob->hosts_try_prdr, sx->host) != OK) - sx->peer_offered &= ~PEER_OFFERED_PRDR; + sx->peer_offered &= ~OPTION_PRDR; - if (sx->peer_offered & PEER_OFFERED_PRDR) + if (sx->peer_offered & OPTION_PRDR) {DEBUG(D_transport) debug_printf("PRDR usable\n");} #endif /* Note if the server supports DSN */ - smtp_peer_options |= sx->peer_offered & PEER_OFFERED_DSN; + smtp_peer_options |= sx->peer_offered & OPTION_DSN; DEBUG(D_transport) debug_printf("%susing DSN\n", - sx->peer_offered & PEER_OFFERED_DSN ? "" : "not "); + sx->peer_offered & OPTION_DSN ? "" : "not "); /* Note if the response to EHLO specifies support for the AUTH extension. If it has, check that this host is one we want to authenticate to, and do @@ -2125,7 +2126,7 @@ if (continue_hostname == NULL } } } -pipelining_active = !!(smtp_peer_options & PEER_OFFERED_PIPE); +pipelining_active = !!(smtp_peer_options & OPTION_PIPE); /* The setting up of the SMTP call is now complete. Any subsequent errors are message-specific. */ @@ -2143,7 +2144,7 @@ if (sx->addrlist->prop.utf8_msg) } /* If this is an international message we need the host to speak SMTPUTF8 */ -if (sx->utf8_needed && !(sx->peer_offered & PEER_OFFERED_UTF8)) +if (sx->utf8_needed && !(sx->peer_offered & OPTION_UTF8)) { errno = ERRNO_UTF8_FWD; goto RESPONSE_FAILED; @@ -2270,14 +2271,15 @@ int address_count; *p = 0; -/* If we know the receiving MTA supports the SIZE qualification, +/* If we know the receiving MTA supports the SIZE qualification, and we know it, send it, adding something to the message size to allow for imprecision and things that get added en route. Exim keeps the number of lines in a message, so we can give an accurate value for the original message, but we need some additional to handle added headers. (Double "." characters don't get included in the count.) */ -if (sx->peer_offered & PEER_OFFERED_SIZE) +if ( message_size > 0 + && sx->peer_offered & OPTION_SIZE && !(sx->avoid_option & OPTION_SIZE)) { /*XXX problem here under spool_files_wireformat? Or just forget about lines? Or inflate by a fixed proportion? */ @@ -2291,7 +2293,7 @@ Or just forget about lines? Or inflate by a fixed proportion? */ request that */ sx->prdr_active = FALSE; -if (sx->peer_offered & PEER_OFFERED_PRDR) +if (sx->peer_offered & OPTION_PRDR) for (addr = addrlist; addr; addr = addr->next) if (addr->transport_return == PENDING_DEFER) { @@ -2310,7 +2312,7 @@ if (sx->peer_offered & PEER_OFFERED_PRDR) /* If it supports internationalised messages, and this meesage need that, request it */ -if ( sx->peer_offered & PEER_OFFERED_UTF8 +if ( sx->peer_offered & OPTION_UTF8 && addrlist->prop.utf8_msg && !addrlist->prop.utf8_downcvt ) @@ -2332,7 +2334,7 @@ for (sx->dsn_all_lasthop = TRUE, addr = addrlist, address_count = 0; /* Add any DSN flags to the mail command */ -if (sx->peer_offered & PEER_OFFERED_DSN && !sx->dsn_all_lasthop) +if (sx->peer_offered & OPTION_DSN && !sx->dsn_all_lasthop) { if (dsn_ret == dsn_ret_hdrs) { Ustrcpy(p, " RET=HDRS"); p += 9; } @@ -2368,7 +2370,7 @@ uschar * p = sx->buffer; /* Add any DSN flags to the rcpt command */ -if (sx->peer_offered & PEER_OFFERED_DSN && !(addr->dsn_flags & rf_dsnlasthop)) +if (sx->peer_offered & OPTION_DSN && !(addr->dsn_flags & rf_dsnlasthop)) { if (addr->dsn_flags & rf_dsnflags) { @@ -2438,7 +2440,7 @@ sx->pending_MAIL = TRUE; /* The block starts with MAIL */ the delivery log line. */ if ( sx->addrlist->prop.utf8_msg - && (sx->addrlist->prop.utf8_downcvt || !(sx->peer_offered & PEER_OFFERED_UTF8)) + && (sx->addrlist->prop.utf8_downcvt || !(sx->peer_offered & OPTION_UTF8)) ) { if (s = string_address_utf8_to_alabel(s, &errstr), errstr) @@ -2502,7 +2504,7 @@ for (addr = sx->first_addr, address_count = 0; BOOL no_flush; uschar * rcpt_addr; - addr->dsn_aware = sx->peer_offered & PEER_OFFERED_DSN + addr->dsn_aware = sx->peer_offered & OPTION_DSN ? dsn_support_yes : dsn_support_no; address_count++; @@ -2762,10 +2764,10 @@ if (tblock->filter_command) if ( transport_filter_argv && *transport_filter_argv && **transport_filter_argv - && sx.peer_offered & PEER_OFFERED_CHUNKING + && sx.peer_offered & OPTION_CHUNKING ) { - sx.peer_offered &= ~PEER_OFFERED_CHUNKING; + sx.peer_offered &= ~OPTION_CHUNKING; DEBUG(D_transport) debug_printf("CHUNKING not usable due to transport filter\n"); } } @@ -2847,7 +2849,7 @@ are pipelining. The responses are all handled by sync_responses(). If using CHUNKING, do not send a BDAT until we know how big a chunk we want to send is. */ -if ( !(sx.peer_offered & PEER_OFFERED_CHUNKING) +if ( !(sx.peer_offered & OPTION_CHUNKING) && (sx.ok || (pipelining_active && !mua_wrapper))) { int count = smtp_write_command(&sx.outblock, SCMD_FLUSH, "DATA\r\n"); @@ -2877,7 +2879,7 @@ for handling the SMTP dot-handling protocol, flagging to apply to headers as well as body. Set the appropriate timeout value to be used for each chunk. (Haven't been able to make it work using select() for writing yet.) */ -if (!(sx.peer_offered & PEER_OFFERED_CHUNKING) && !sx.ok) +if (!(sx.peer_offered & OPTION_CHUNKING) && !sx.ok) { /* Save the first address of the next batch. */ sx.first_addr = sx.next_addr; @@ -2904,7 +2906,7 @@ else of responses. The callback needs a whole bunch of state so set up a transport-context structure to be passed around. */ - if (sx.peer_offered & PEER_OFFERED_CHUNKING) + if (sx.peer_offered & OPTION_CHUNKING) { tctx.check_string = tctx.escape_string = NULL; tctx.options |= topt_use_bdat; @@ -2929,7 +2931,7 @@ else transport_write_timeout = sx.ob->data_timeout; smtp_command = US"sending data block"; /* For error messages */ DEBUG(D_transport|D_v) - if (sx.peer_offered & PEER_OFFERED_CHUNKING) + if (sx.peer_offered & OPTION_CHUNKING) debug_printf(" will write message using CHUNKING\n"); else debug_printf(" SMTP>> writing message and terminating \".\"\n"); @@ -2964,7 +2966,7 @@ else smtp_command = US"end of data"; - if (sx.peer_offered & PEER_OFFERED_CHUNKING && sx.cmd_count > 1) + if (sx.peer_offered & OPTION_CHUNKING && sx.cmd_count > 1) { /* Reap any outstanding MAIL & RCPT commands, but not a DATA-go-ahead */ switch(sync_responses(&sx, sx.cmd_count-1, 0)) @@ -3119,7 +3121,7 @@ else #ifndef DISABLE_PRDR if (sx.prdr_active) addr->flags |= af_prdr_used; #endif - if (sx.peer_offered & PEER_OFFERED_CHUNKING) addr->flags |= af_chunking_used; + if (sx.peer_offered & OPTION_CHUNKING) addr->flags |= af_chunking_used; flag = '-'; #ifndef DISABLE_PRDR @@ -3441,7 +3443,7 @@ if (sx.completed_addr && sx.ok && sx.send_quit) { /* Set up a pipe for proxying TLS for the new transport process */ - smtp_peer_options |= PEER_OFFERED_TLS; + smtp_peer_options |= OPTION_TLS; if (sx.ok = (socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) == 0)) socket_fd = pfd[1]; else diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h index 88b608bcc..c965a72a7 100644 --- a/src/src/transports/smtp.h +++ b/src/src/transports/smtp.h @@ -130,6 +130,7 @@ typedef struct { int cmd_count; uschar peer_offered; + uschar avoid_option; uschar * igquotstr; uschar * helo_data; #ifdef EXPERIMENTAL_DSN_INFO diff --git a/src/src/verify.c b/src/src/verify.c index cda0b72e2..6c71d7feb 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -785,8 +785,12 @@ tls_retry_connection: postmaster-verify. The sync_responses() would need to be taught about it and we'd need another return code filtering out to here. + + Avoid using a SIZE option on the MAIL for all randon-rcpt checks. */ + sx.avoid_option = OPTION_SIZE; + /* Remember when we last did a random test */ new_domain_record.random_stamp = time(NULL); @@ -798,8 +802,9 @@ tls_retry_connection: yield = OK; /* Only usable result we can return */ done = TRUE; goto no_conn; - case FAIL: + case FAIL: /* the preferred result */ new_domain_record.random_result = ccache_reject; + sx.avoid_option = 0; /* Between each check, issue RSET, because some servers accept only one recipient after MAIL FROM:<>. @@ -844,12 +849,14 @@ tls_retry_connection: else done = TRUE; - /* Main verify. If the host is accepting all local parts, as determined - by the "random" check, we don't need to waste time doing any further - checking. */ + /* Main verify. For rcpt-verify use SIZE if we know it and we're not cacheing; + for sndr-verify never use it. */ if (done) { + if (!(options & vopt_is_recipient && options & vopt_callout_no_cache)) + sx.avoid_option = OPTION_SIZE; + done = FALSE; switch(smtp_write_mail_and_rcpt_cmds(&sx, &yield)) { @@ -858,12 +865,12 @@ tls_retry_connection: case PENDING_OK: done = TRUE; new_address_record.result = ccache_accept; break; - case FAIL: done = TRUE; + case FAIL: done = TRUE; yield = FAIL; *failure_ptr = US"recipient"; new_address_record.result = ccache_reject; break; - default: break; + default: break; } break; @@ -916,6 +923,7 @@ tls_retry_connection: sx.ok = FALSE; sx.send_rset = TRUE; sx.completed_addr = FALSE; + sx.avoid_option = OPTION_SIZE; if( smtp_write_mail_and_rcpt_cmds(&sx, &yield) == 0 && addr->transport_return == PENDING_OK diff --git a/test/confs/0227 b/test/confs/0227 index ed988ac85..fea66e16f 100644 --- a/test/confs/0227 +++ b/test/confs/0227 @@ -24,6 +24,8 @@ check_recipient: !verify = sender/callout=no_cache deny hosts = V4NET.0.0.3 !verify = recipient/callout=no_cache + deny hosts = V4NET.0.0.7 + !verify = recipient/callout=no_cache,use_sender,random deny hosts = V4NET.0.0.5 !verify = sender/callout=no_cache/check_postmaster deny hosts = V4NET.0.0.6 diff --git a/test/log/0227 b/test/log/0227 index f1e64701c..d91075fb6 100644 --- a/test/log/0227 +++ b/test/log/0227 @@ -14,6 +14,9 @@ 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= rejected RCPT : relay not permitted 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root sender verify fail for : 127.0.0.1 [127.0.0.1] : SMTP error from remote mail server after RCPT TO:: 550 Don't like postmaster 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= rejected RCPT : Sender verify failed +1999-03-02 09:44:33 H=(me) [V4NET.0.0.3] U=root F= rejected RCPT : relay not permitted +1999-03-02 09:44:33 H=(me) [V4NET.0.0.3] U=root F= rejected RCPT : relay not permitted +1999-03-02 09:44:33 H=(me) [V4NET.0.0.7] U=root F= rejected RCPT : relay not permitted 1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root F= rejected RCPT : 127.0.0.1 [127.0.0.1] : SMTP error from remote mail server after RCPT TO:: 550 Recipient not liked 1999-03-02 09:44:33 H=[V4NET.0.0.1] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : Remote host closed connection in response to initial connection 1999-03-02 09:44:33 H=[V4NET.0.0.1] U=root F= temporarily rejected RCPT : Could not complete sender verify callout diff --git a/test/rejectlog/0227 b/test/rejectlog/0227 index 8f5c0ad15..e44ddafbc 100644 --- a/test/rejectlog/0227 +++ b/test/rejectlog/0227 @@ -30,6 +30,9 @@ F From: abcd@x.y.z 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= rejected RCPT : relay not permitted 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root sender verify fail for : 127.0.0.1 [127.0.0.1] : SMTP error from remote mail server after RCPT TO:: 550 Don't like postmaster 1999-03-02 09:44:33 H=[V4NET.0.0.5] U=root F= rejected RCPT : Sender verify failed +1999-03-02 09:44:33 H=(me) [V4NET.0.0.3] U=root F= rejected RCPT : relay not permitted +1999-03-02 09:44:33 H=(me) [V4NET.0.0.3] U=root F= rejected RCPT : relay not permitted +1999-03-02 09:44:33 H=(me) [V4NET.0.0.7] U=root F= rejected RCPT : relay not permitted 1999-03-02 09:44:33 H=[V4NET.0.0.3] U=root F= rejected RCPT : 127.0.0.1 [127.0.0.1] : SMTP error from remote mail server after RCPT TO:: 550 Recipient not liked 1999-03-02 09:44:33 H=[V4NET.0.0.1] U=root sender verify defer for : Could not complete sender verify callout: 127.0.0.1 [127.0.0.1] : Remote host closed connection in response to initial connection 1999-03-02 09:44:33 H=[V4NET.0.0.1] U=root F= temporarily rejected RCPT : Could not complete sender verify callout diff --git a/test/scripts/0000-Basic/0227 b/test/scripts/0000-Basic/0227 index a67dab2eb..e53c1da7f 100644 --- a/test/scripts/0000-Basic/0227 +++ b/test/scripts/0000-Basic/0227 @@ -12,11 +12,13 @@ RCPT TO QUIT 250 OK **** +# sender sudo exim -v -bs -oMa V4NET.0.0.1 MAIL FROM: RCPT TO: QUIT **** +# sender, no callout sudo exim -v -bs -oMa V4NET.0.0.2 MAIL FROM: RCPT TO: @@ -33,6 +35,7 @@ RCPT TO QUIT 250 OK **** +# sender, refused sudo exim -v -bs -oMa V4NET.0.0.1 MAIL FROM: RCPT TO: @@ -49,6 +52,7 @@ RCPT TO QUIT 250 OK **** +# sender, tmperr sudo exim -v -bs -oMa V4NET.0.0.1 MAIL FROM: RCPT TO: @@ -63,6 +67,7 @@ MAIL FROM QUIT 250 OK **** +# sender, err on mailfrom sudo exim -v -bs -oMa V4NET.0.0.1 MAIL FROM: RCPT TO: @@ -78,6 +83,7 @@ MAIL FROM QUIT 250 OK **** +# sender, err on mailfrom sudo exim -v -bs -oMa V4NET.0.0.1 MAIL FROM: RCPT TO: @@ -94,6 +100,7 @@ RCPT TO QUIT 250 OK **** +# recipient, refused sudo exim -v -bs -oMa V4NET.0.0.3 MAIL FROM: RCPT TO: @@ -111,11 +118,13 @@ RCPT TO QUIT 250 OK **** +# recipient, refused sudo exim -v -bs -oMa V4NET.0.0.3 MAIL FROM: RCPT TO: QUIT **** +# recipient, no conneect sudo exim -v -bs -oMa V4NET.0.0.3 MAIL FROM: RCPT TO: @@ -207,16 +216,17 @@ QUIT server PORT_S 220 Server ready EHLO -250- wotcher +250- wotcher sverifier 250-SIZE 250 OK -MAIL FROM +MAIL FROM:<> 250 OK RCPT TO 250 OK QUIT 250 OK **** +# sender, size known, tgt supports sixe sudo exim -v -bs -oMa V4NET.0.0.6 EHLO me MAIL FROM: SIZE=420000 @@ -226,6 +236,76 @@ QUIT # server PORT_S 220 Server ready +EHLO +250- wotcher rverifier +250-SIZE +250 OK +MAIL FROM:<> SIZE= +250 OK +RCPT TO +250 OK +QUIT +250 OK +**** +# receipient, size known, tgt supports size +sudo exim -v -bs -oMa V4NET.0.0.3 +EHLO me +MAIL FROM: SIZE=420000 +RCPT TO: +QUIT +**** +# +server PORT_S +220 Server ready +EHLO +250- wotcher rverifier +250-SIZE +250 OK +MAIL FROM:<> +250 OK +RCPT TO +250 OK +QUIT +250 OK +**** +# receipient, size unknown, tgt supports size +sudo exim -v -bs -oMa V4NET.0.0.3 +EHLO me +MAIL FROM: +RCPT TO: +QUIT +**** +# +server PORT_S +220 Server ready +EHLO +250- wotcher rverifier +250-SIZE +250 OK +MAIL FROM: +250 OK +RCPT TO: SIZE= +250 OK +RCPT TO: +250 OK +QUIT +250 OK +**** +# receipient, size known, tgt supports size, use_sender, w/random +# stdout should see SIZE on the main but not on the random receipient's MAIL FROM +sudo exim -v -bs -oMa V4NET.0.0.7 +EHLO me +MAIL FROM: SIZE=420000 +RCPT TO: +QUIT +**** +# +server PORT_S +220 Server ready LHLO 250 OK MAIL FROM diff --git a/test/stderr/0227 b/test/stderr/0227 index 6a8ee1f7e..e3282cc6f 100644 --- a/test/stderr/0227 +++ b/test/stderr/0227 @@ -222,10 +222,10 @@ LOG: smtp_connection MAIN Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected SMTP<< 220 Server ready SMTP>> EHLO myhost.test.ex - SMTP<< 250- wotcher + SMTP<< 250- wotcher sverifier 250-SIZE 250 OK - SMTP>> MAIL FROM:<> SIZE=ssss + SMTP>> MAIL FROM:<> SMTP<< 250 OK SMTP>> RCPT TO: SMTP<< 250 OK @@ -236,6 +236,69 @@ LOG: smtp_connection MAIN SMTP connection from root closed by QUIT LOG: smtp_connection MAIN SMTP connection from root +Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected + SMTP<< 220 Server ready + SMTP>> EHLO myhost.test.ex + SMTP<< 250- wotcher rverifier + 250-SIZE + 250 OK + SMTP>> MAIL FROM:<> SIZE=ssss + SMTP<< 250 OK + SMTP>> RCPT TO: + SMTP<< 250 OK + SMTP>> QUIT + SMTP<< 250 OK + SMTP(close)>> +LOG: MAIN REJECT + H=(me) [V4NET.0.0.3] U=root F= rejected RCPT : relay not permitted +LOG: smtp_connection MAIN + SMTP connection from root closed by QUIT +LOG: smtp_connection MAIN + SMTP connection from root +Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected + SMTP<< 220 Server ready + SMTP>> EHLO myhost.test.ex + SMTP<< 250- wotcher rverifier + 250-SIZE + 250 OK + SMTP>> MAIL FROM:<> + SMTP<< 250 OK + SMTP>> RCPT TO: + SMTP<< 250 OK + SMTP>> QUIT + SMTP<< 250 OK + SMTP(close)>> +LOG: MAIN REJECT + H=(me) [V4NET.0.0.3] U=root F= rejected RCPT : relay not permitted +LOG: smtp_connection MAIN + SMTP connection from root closed by QUIT +LOG: smtp_connection MAIN + SMTP connection from root +Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected + SMTP<< 220 Server ready + SMTP>> EHLO myhost.test.ex + SMTP<< 250- wotcher rverifier + 250-SIZE + 250 OK + SMTP>> MAIL FROM: + SMTP<< 250 OK + SMTP>> RCPT TO: + SMTP<< 550 RANDOM NOT OK + SMTP>> RSET + SMTP<< 250 OK + SMTP>> MAIL FROM: SIZE=ssss + SMTP<< 250 OK + SMTP>> RCPT TO: + SMTP<< 250 OK + SMTP>> QUIT + SMTP<< 250 OK + SMTP(close)>> +LOG: MAIN REJECT + H=(me) [V4NET.0.0.7] U=root F= rejected RCPT : relay not permitted +LOG: smtp_connection MAIN + SMTP connection from root closed by QUIT +LOG: smtp_connection MAIN + SMTP connection from root Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected SMTP<< 220 Server ready SMTP>> LHLO myhost.test.ex diff --git a/test/stderr/2035 b/test/stderr/2035 index e37495092..dedf4ad48 100644 --- a/test/stderr/2035 +++ b/test/stderr/2035 @@ -26,7 +26,7 @@ cmd buf flush ddd bytes 250 HELP using PIPELINING not using DSN - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK diff --git a/test/stderr/2135 b/test/stderr/2135 index 5a4de14ec..ac524d2af 100644 --- a/test/stderr/2135 +++ b/test/stderr/2135 @@ -26,7 +26,7 @@ cmd buf flush ddd bytes 250 HELP using PIPELINING not using DSN - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK diff --git a/test/stderr/5410 b/test/stderr/5410 index 6a37deb6e..b4db690ab 100644 --- a/test/stderr/5410 +++ b/test/stderr/5410 @@ -119,7 +119,7 @@ cmd buf flush ddd bytes using PIPELINING not using DSN 127.0.0.1 in hosts_require_auth? no (option unset) - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK @@ -345,7 +345,7 @@ cmd buf flush ddd bytes using PIPELINING not using DSN 127.0.0.1 in hosts_require_auth? no (option unset) - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK @@ -571,7 +571,7 @@ cmd buf flush ddd bytes using PIPELINING not using DSN 127.0.0.1 in hosts_require_auth? no (option unset) - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK diff --git a/test/stderr/5420 b/test/stderr/5420 index eaa40bce8..b34440046 100644 --- a/test/stderr/5420 +++ b/test/stderr/5420 @@ -118,7 +118,7 @@ cmd buf flush ddd bytes using PIPELINING not using DSN 127.0.0.1 in hosts_require_auth? no (option unset) - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK @@ -344,7 +344,7 @@ cmd buf flush ddd bytes using PIPELINING not using DSN 127.0.0.1 in hosts_require_auth? no (option unset) - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK @@ -570,7 +570,7 @@ cmd buf flush ddd bytes using PIPELINING not using DSN 127.0.0.1 in hosts_require_auth? no (option unset) - SMTP>> MAIL FROM: SIZE=ssss + SMTP>> MAIL FROM: SMTP>> RCPT TO: cmd buf flush ddd bytes SMTP<< 250 OK diff --git a/test/stderr/5840 b/test/stderr/5840 index 5ccf7cda0..bc8aeca30 100644 --- a/test/stderr/5840 +++ b/test/stderr/5840 @@ -50,9 +50,9 @@ >>> 250 HELP >>> ip4.ip4.ip4.ip4 in hosts_avoid_pipelining? no (option unset) >>> ip4.ip4.ip4.ip4 in hosts_require_auth? no (option unset) ->>> SMTP>> MAIL FROM:<> SIZE=ssss +>>> SMTP>> MAIL FROM:<> >>> SMTP>> RCPT TO: ->>> cmd buf flush 62 bytes +>>> cmd buf flush 52 bytes >>> SMTP<< 250 OK >>> SMTP<< 250 Accepted >>> SMTP>> QUIT diff --git a/test/stdout/0227 b/test/stdout/0227 index ea8520ef1..9ee7e9f54 100644 --- a/test/stdout/0227 +++ b/test/stdout/0227 @@ -88,6 +88,33 @@ 250 Accepted 221 myhost.test.ex closing connection 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +250-myhost.test.ex Hello root at me [V4NET.0.0.3] +250-SIZE 52428800 +250-8BITMIME +250-PIPELINING +250 HELP +250 OK +550 relay not permitted +221 myhost.test.ex closing connection +220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +250-myhost.test.ex Hello root at me [V4NET.0.0.3] +250-SIZE 52428800 +250-8BITMIME +250-PIPELINING +250 HELP +250 OK +550 relay not permitted +221 myhost.test.ex closing connection +220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +250-myhost.test.ex Hello root at me [V4NET.0.0.7] +250-SIZE 52428800 +250-8BITMIME +250-PIPELINING +250 HELP +250 OK +550 relay not permitted +221 myhost.test.ex closing connection +220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK 550-Callout verification failed: 550 550 Recipient not liked @@ -248,10 +275,10 @@ Listening on port 1224 ... Connection request from [127.0.0.1] 220 Server ready EHLO myhost.test.ex -250- wotcher +250- wotcher sverifier 250-SIZE 250 OK -MAIL FROM:<> SIZE=ssss +MAIL FROM:<> 250 OK RCPT TO: 250 OK @@ -261,6 +288,54 @@ End of script Listening on port 1224 ... Connection request from [127.0.0.1] 220 Server ready +EHLO myhost.test.ex +250- wotcher rverifier +250-SIZE +250 OK +MAIL FROM:<> SIZE=ssss +250 OK +RCPT TO: +250 OK +QUIT +250 OK +End of script +Listening on port 1224 ... +Connection request from [127.0.0.1] +220 Server ready +EHLO myhost.test.ex +250- wotcher rverifier +250-SIZE +250 OK +MAIL FROM:<> +250 OK +RCPT TO: +250 OK +QUIT +250 OK +End of script +Listening on port 1224 ... +Connection request from [127.0.0.1] +220 Server ready +EHLO myhost.test.ex +250- wotcher rverifier +250-SIZE +250 OK +MAIL FROM: +250 OK +RCPT TO: +550 RANDOM NOT OK +RSET +250 OK +MAIL FROM: SIZE=ssss +250 OK +RCPT TO: +250 OK +QUIT +250 OK +End of script +Listening on port 1224 ... +Connection request from [127.0.0.1] +220 Server ready LHLO myhost.test.ex 250 OK MAIL FROM:<> -- cgit v1.2.3 From 571b27158aa468eda77b1a20ed80bed1f342e15d Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 30 Jul 2017 20:51:10 +0100 Subject: Logging: millisecond timestamps. Bug 2102 No change to QT logging yet as that will need a spool format update --- doc/doc-docbook/spec.xfpt | 9 ++ doc/doc-txt/NewStuff | 2 + src/OS/Makefile-Base | 6 +- src/src/exigrep.src | 6 +- src/src/eximstats.src | 36 ++++-- src/src/globals.c | 1 + src/src/log.c | 54 +++------ src/src/macros.h | 1 + src/src/tod.c | 254 ++++++++++++++++++++++--------------------- test/log/0551 | 6 +- test/runtest | 3 +- test/scripts/0000-Basic/0551 | 4 +- test/stdout/0551 | 6 +- 13 files changed, 207 insertions(+), 181 deletions(-) (limited to 'test/stdout') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 84540508f..a4b1926ec 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -36118,6 +36118,7 @@ selection marked by asterisks: &` incoming_interface `& local interface on <= and => lines &` incoming_port `& remote port on <= lines &`*lost_incoming_connection `& as it says (includes timeouts) +&` millisec `& millisecond timestamps &` outgoing_interface `& local interface on => lines &` outgoing_port `& add remote port to => lines &`*queue_run `& start and end queue runs @@ -36282,6 +36283,14 @@ important with the widening use of NAT (see RFC 2505). &%lost_incoming_connection%&: A log line is written when an incoming SMTP connection is unexpectedly dropped. .next +.new +.cindex "log" "millisecond timestamps" +.cindex millisecond logging +.cindex timstamps "millisecond, in logs" +&%millisec%&: Timestamps have a period and three decimal places of finer granularity +appended to the seconds value. +.wen +.next .cindex "log" "outgoing interface" .cindex "log" "local interface" .cindex "log" "local address and port" diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index f3050a950..ea757f059 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -44,6 +44,8 @@ Version 4.90 10. Variable $smtp_command_history returning a comma-sep list of recent SMTP commands. +11. Millisecond timetamps in logs, on log_selector "millisec". + Version 4.89 ------------ diff --git a/src/OS/Makefile-Base b/src/OS/Makefile-Base index 11f4dad0c..b4345e86f 100644 --- a/src/OS/Makefile-Base +++ b/src/OS/Makefile-Base @@ -487,7 +487,7 @@ OBJ_MONBIN = util-spool_in.o \ util-store.o \ util-string.o \ util-queue.o \ - tod.o \ + util-tod.o \ tree.o \ $(MONBIN) @@ -614,6 +614,10 @@ util-queue.o: $(HDRS) queue.c @echo "$(CC) -DCOMPILE_UTILITY queue.c" $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-queue.o queue.c +util-tod.o: $(HDRS) tod.c + @echo "$(CC) -DCOMPILE_UTILITY tod.c" + $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-tod.o tod.c + util-os.o: $(HDRS) os.c @echo "$(CC) -DCOMPILE_UTILITY os.c" $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) \ diff --git a/src/src/exigrep.src b/src/src/exigrep.src index 851ca63bb..1899267be 100644 --- a/src/src/exigrep.src +++ b/src/src/exigrep.src @@ -45,7 +45,7 @@ use POSIX qw(mktime); sub seconds { my($year,$month,$day,$hour,$min,$sec,$tzs,$tzh,$tzm) = - $_[0] =~ /^(\d{4})-(\d\d)-(\d\d)\s(\d\d):(\d\d):(\d\d)(?>\s([+-])(\d\d)(\d\d))?/o; + $_[0] =~ /^(\d{4})-(\d\d)-(\d\d)\s(\d\d):(\d\d):(\d\d)(?:.\d+)?(?>\s([+-])(\d\d)(\d\d))?/o; my $seconds = mktime $sec, $min, $hour, $day, $month - 1, $year - 1900; @@ -83,7 +83,7 @@ sub do_line { if (!/^\d{4}-/o) { $_ =~ s/^.*? exim\b.*?: //o; } return unless - my($date,$id) = /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d (?:[+-]\d{4} )?)(?:\[\d+\] )?(\w{6}\-\w{6}\-\w{2})?/o; + my($date,$id) = /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:\.\d+)? (?:[+-]\d{4} )?)(?:\[\d+\] )?(\w{6}\-\w{6}\-\w{2})?/o; # Handle the case when the log line belongs to a specific message. We save # lines for specific messages until the message is complete. Then either print @@ -124,7 +124,7 @@ if (defined $id) if (index($_, 'Completed') != -1 || index($_, 'SMTP data timeout') != -1 || (index($_, 'rejected') != -1 && - /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d (?:[+-]\d{4} )?)(?:\[\d+\] )?\w{6}\-\w{6}\-\w{2} rejected/o)) + /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:\.\d+)? (?:[+-]\d{4} )?)(?:\[\d+\] )?\w{6}\-\w{6}\-\w{2} rejected/o)) { if ($queue_time != -1 && $saved{$id} =~ /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d ([+-]\d{4} )?)/o) diff --git a/src/src/eximstats.src b/src/src/eximstats.src index a2113f106..727ee44b9 100644 --- a/src/src/eximstats.src +++ b/src/src/eximstats.src @@ -896,6 +896,7 @@ sub unformat_time { # POSIX::mktime. We expect the timestamp to be of the form # "$year-$mon-$day $hour:$min:$sec", with month going from 1 to 12, # and the year to be absolute (we do the necessary conversions). The +# seconds value can be followed by decimals, which we ignore. The # timestamp may be followed with an offset from UTC like "+$hh$mm"; if the # offset is not present, and we have not been told that the log is in UTC # (with the -utc option), then we adjust the time by the current local @@ -919,7 +920,7 @@ sub seconds { # Is the timestamp the same as the last one? return $last_time if ($last_timestamp eq $timestamp); - return 0 unless ($timestamp =~ /^((\d{4})\-(\d\d)-(\d\d))\s(\d\d):(\d\d):(\d\d)( ([+-])(\d\d)(\d\d))?/o); + return 0 unless ($timestamp =~ /^((\d{4})\-(\d\d)-(\d\d))\s(\d\d):(\d\d):(\d\d)(?:\.\d+)?( ([+-])(\d\d)(\d\d))?/o); unless ($last_date eq $1) { $last_date = $1; @@ -931,7 +932,7 @@ sub seconds { my $time = $date_seconds + ($5 * 3600) + ($6 * 60) + $7; # SC. Use caching. Also note we want seconds not minutes. - #my($this_offset) = ($10 * 60 + $11) * ($9 . "1") if defined $8; + #my($this_offset) = ($10 * 60 + $12) * ($9 . "1") if defined $8; if (defined $8 && ($8 ne $last_offset)) { $last_offset = $8; $offset_seconds = ($10 * 60 + $11) * 60; @@ -939,7 +940,7 @@ sub seconds { } - if (defined $7) { + if (defined $8) { #$time -= $this_offset; $time -= $offset_seconds; } elsif (defined $localtime_offset) { @@ -1853,12 +1854,23 @@ sub generate_parser { $length = length($_); next if ($length < 38); - next unless /^(\\d{4}\\-\\d\\d-\\d\\d\\s(\\d\\d):(\\d\\d):\\d\\d( [-+]\\d\\d\\d\\d)?)( \\[\\d+\\])?/o; - - ($tod,$m_hour,$m_min) = ($1,$2,$3); + next unless /^ + (\\d{4}\\-\\d\\d-\\d\\d\\s # 1: YYYYMMDD HHMMSS + (\\d\\d) # 2: HH + : + (\\d\\d) # 3: MM + :\\d\\d + ) + (\\.\\d+)? # 4: subseconds + (\s[-+]\\d\\d\\d\\d)? # 5: tz-offset + (\s\\[\\d+\\])? # 6: pid + /ox; + + $tod = defined($5) ? $1 . $5 : $1; + ($m_hour,$m_min) = ($2,$3); # PH - watch for GMT offsets in the timestamp. - if (defined($4)) { + if (defined($5)) { $extra = 6; next if ($length < 44); } @@ -1866,9 +1878,15 @@ sub generate_parser { $extra = 0; } + # watch for subsecond precision + if (defined($4)) { + $extra += length($4); + next if ($length < 38 + $extra); + } + # PH - watch for PID added after the timestamp. - if (defined($5)) { - $extra += length($5); + if (defined($6)) { + $extra += length($6); next if ($length < 38 + $extra); } diff --git a/src/src/globals.c b/src/src/globals.c index bfb1c2166..8c35c6fe2 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -893,6 +893,7 @@ bit_table log_options[] = { /* must be in alphabetical order */ BIT_TABLE(L, incoming_interface), BIT_TABLE(L, incoming_port), BIT_TABLE(L, lost_incoming_connection), + BIT_TABLE(L, millisec), BIT_TABLE(L, outgoing_interface), BIT_TABLE(L, outgoing_port), BIT_TABLE(L, pid), diff --git a/src/src/log.c b/src/src/log.c index 62ee629b0..d12989b6f 100644 --- a/src/src/log.c +++ b/src/src/log.c @@ -147,7 +147,7 @@ int linecount = 0; if (running_in_test_harness) return; -if (!syslog_timestamp) s += log_timezone? 26 : 20; +if (!syslog_timestamp) s += log_timezone ? 26 : 20; if (!syslog_pid && LOGGING(pid)) memmove(s + pid_position[0], s + pid_position[1], pid_position[1] - pid_position[0]); @@ -223,10 +223,10 @@ Returns: The function does not return static void die(uschar *s1, uschar *s2) { -if (s1 != NULL) +if (s1) { write_syslog(LOG_CRIT, s1); - if (debug_file != NULL) debug_printf("%s\n", s1); + if (debug_file) debug_printf("%s\n", s1); if (log_stderr != NULL && log_stderr != debug_file) fprintf(log_stderr, "%s\n", s1); } @@ -272,7 +272,7 @@ if (fd < 0 && errno == ENOENT) *lastslash = 0; created = directory_make(NULL, name, LOG_DIRECTORY_MODE, FALSE); DEBUG(D_any) debug_printf("%s log directory %s\n", - created? "created" : "failed to create", name); + created ? "created" : "failed to create", name); *lastslash = '/'; if (created) fd = Uopen(name, #ifdef O_CLOEXEC @@ -446,10 +446,8 @@ else if (string_datestamp_offset >= 0) /* If the file name is too long, it is an unrecoverable disaster */ if (!ok) - { die(US"exim: log file path too long: aborting", US"Logging failure; please try later"); - } /* We now have the file name. Try to open an existing file. After a successful open, arrange for automatic closure on exec(), and then return. */ @@ -559,10 +557,7 @@ if ((flags & (LOG_CONFIG_FOR & ~LOG_CONFIG)) != 0) } if ((flags & (LOG_CONFIG_IN & ~LOG_CONFIG)) != 0) - { - sprintf(CS ptr, " in line %d of %s", config_lineno, config_filename); - while (*ptr) ptr++; - } + ptr += sprintf(CS ptr, " in line %d of %s", config_lineno, config_filename); Ustrcpy(ptr, ":\n "); return ptr + 4; @@ -736,7 +731,7 @@ Returns: nothing void log_write(unsigned int selector, int flags, const char *format, ...) { -uschar *ptr; +uschar * ptr; int length; int paniclogfd; ssize_t written_len; @@ -903,22 +898,17 @@ if (!write_rejectlog) flags &= ~LOG_REJECT; when called by a utility. */ ptr = log_buffer; -sprintf(CS ptr, "%s ", tod_stamp(tod_log)); -while(*ptr) ptr++; +ptr += sprintf(CS ptr, "%s ", tod_stamp(tod_log)); if (LOGGING(pid)) { - sprintf(CS ptr, "[%d] ", (int)getpid()); if (!syslog_pid) pid_position[0] = ptr - log_buffer; /* remember begin … */ - while (*ptr) ptr++; + ptr += sprintf(CS ptr, "[%d] ", (int)getpid()); if (!syslog_pid) pid_position[1] = ptr - log_buffer; /* … and end+1 of the PID */ } if (really_exim && message_id[0] != 0) - { - sprintf(CS ptr, "%s ", message_id); - while(*ptr) ptr++; - } + ptr += sprintf(CS ptr, "%s ", message_id); if ((flags & LOG_CONFIG) != 0) ptr = log_config_info(ptr, flags); @@ -933,10 +923,7 @@ this way because it kind of fits with LOG_RECIPIENTS. */ if ((flags & LOG_SENDER) != 0 && ptr < log_buffer + LOG_BUFFER_SIZE - 10 - Ustrlen(raw_sender)) - { - sprintf(CS ptr, " from <%s>", raw_sender); - while (*ptr) ptr++; - } + ptr += sprintf(CS ptr, " from <%s>", raw_sender); /* Add list of recipients to the message if required; the raw list, before rewriting, was saved in raw_recipients. There may be none, if an ACL @@ -946,19 +933,16 @@ if ((flags & LOG_RECIPIENTS) != 0 && ptr < log_buffer + LOG_BUFFER_SIZE - 6 && raw_recipients_count > 0) { int i; - sprintf(CS ptr, " for"); - while (*ptr) ptr++; + ptr += sprintf(CS ptr, " for"); for (i = 0; i < raw_recipients_count; i++) { - uschar *s = raw_recipients[i]; + uschar * s = raw_recipients[i]; if (log_buffer + LOG_BUFFER_SIZE - ptr < Ustrlen(s) + 3) break; - sprintf(CS ptr, " %s", s); - while (*ptr) ptr++; + ptr += sprintf(CS ptr, " %s", s); } } -sprintf(CS ptr, "\n"); -while(*ptr) ptr++; +ptr += sprintf(CS ptr, "\n"); length = ptr - log_buffer; /* Handle loggable errors when running a utility, or when address testing. @@ -1084,11 +1068,9 @@ if ((flags & LOG_REJECT) != 0) /* A header with a NULL text is an unfilled in Received: header */ - for (h = header_list; h != NULL; h = h->next) + for (h = header_list; h; h = h->next) if (h->text) { - BOOL fitted; - if (h->text == NULL) continue; - fitted = string_format(ptr, LOG_BUFFER_SIZE - (ptr-log_buffer), + BOOL fitted = string_format(ptr, LOG_BUFFER_SIZE - (ptr-log_buffer), "%c %s", h->type, h->text); while(*ptr) ptr++; if (!fitted) /* Buffer is full; truncate */ @@ -1118,7 +1100,7 @@ if ((flags & LOG_REJECT) != 0) { struct stat statbuf; - if (rejectlog_datestamp != NULL) + if (rejectlog_datestamp) { uschar *nowstamp = tod_stamp(string_datestamp_type); if (Ustrncmp (rejectlog_datestamp, nowstamp, Ustrlen(nowstamp)) != 0) @@ -1175,9 +1157,7 @@ if ((flags & LOG_PANIC) != 0) fprintf(log_stderr, "%s", CS log_buffer); if ((logging_mode & LOG_MODE_SYSLOG) != 0) - { write_syslog(LOG_ALERT, log_buffer); - } /* If this panic logging was caused by a failure to open the main log, the original log line is in panic_save_buffer. Make an attempt to write it. */ diff --git a/src/src/macros.h b/src/src/macros.h index a8022adf6..937722494 100644 --- a/src/src/macros.h +++ b/src/src/macros.h @@ -467,6 +467,7 @@ enum { Li_ident_timeout, Li_incoming_interface, Li_incoming_port, + Li_millisec, Li_outgoing_interface, Li_outgoing_port, Li_pid, diff --git a/src/src/tod.c b/src/src/tod.c index 5f451ba96..dac8b27f3 100644 --- a/src/src/tod.c +++ b/src/src/tod.c @@ -13,7 +13,7 @@ /* #define TESTING_LOG_DATESTAMP */ -static uschar timebuf[sizeof("www, dd-mmm-yyyy hh:mm:ss +zzzz")]; +static uschar timebuf[sizeof("www, dd-mmm-yyyy hh:mm:ss.ddd +zzzz")]; /************************************************* @@ -52,159 +52,169 @@ Returns: pointer to fixed buffer containing the timestamp uschar * tod_stamp(int type) { -time_t now; -struct tm *t; +struct timeval now; +struct tm * t; +int off = 0; -if (type == tod_epoch_l) - { - struct timeval tv; - gettimeofday(&tv, NULL); - /* Unix epoch/usec format */ - (void) sprintf(CS timebuf, TIME_T_FMT "%06ld", tv.tv_sec, (long) tv.tv_usec ); - return timebuf; - } - -now = time(NULL); - -/* Vary log type according to timezone requirement */ - -if (type == tod_log) type = log_timezone? tod_log_zone : tod_log_bare; +gettimeofday(&now, NULL); /* Styles that don't need local time */ -else if (type == tod_epoch) +switch(type) { - (void) sprintf(CS timebuf, TIME_T_FMT, now); /* Unix epoch format */ - return timebuf; /* NB the above will be wrong if time_t is FP */ + case tod_epoch: + (void) sprintf(CS timebuf, TIME_T_FMT, now.tv_sec); /* Unix epoch format */ + return timebuf; /* NB the above will be wrong if time_t is FP */ + + case tod_epoch_l: + /* Unix epoch/usec format */ + (void) sprintf(CS timebuf, TIME_T_FMT "%06ld", now.tv_sec, (long) now.tv_usec ); + return timebuf; + + case tod_zulu: + t = gmtime(&now.tv_sec); + (void) sprintf(CS timebuf, "%04d%02d%02d%02d%02d%02dZ", + 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday, t->tm_hour, t->tm_min, + t->tm_sec); + return timebuf; } -else if (type == tod_zulu) - { - t = gmtime(&now); - (void) sprintf(CS timebuf, "%04d%02d%02d%02d%02d%02dZ", - 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday, t->tm_hour, t->tm_min, - t->tm_sec); - return timebuf; - } +/* Vary log type according to timezone requirement */ + +if (type == tod_log) type = log_timezone ? tod_log_zone : tod_log_bare; /* Convert to local time or UTC */ -t = timestamps_utc? gmtime(&now) : localtime(&now); +t = timestamps_utc ? gmtime(&now.tv_sec) : localtime(&now.tv_sec); switch(type) { case tod_log_bare: /* Format used in logging without timezone */ - (void) sprintf(CS timebuf, "%04d-%02d-%02d %02d:%02d:%02d", - 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday, - t->tm_hour, t->tm_min, t->tm_sec); - break; + off = sprintf(CS timebuf, "%04d-%02d-%02d %02d:%02d:%02d", + 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday, + t->tm_hour, t->tm_min, t->tm_sec); + break; - /* Format used as suffix of log file name when 'log_datestamp' is active. For - testing purposes, it changes the file every second. */ + /* Format used as suffix of log file name when 'log_datestamp' is active. For + testing purposes, it changes the file every second. */ - #ifdef TESTING_LOG_DATESTAMP +#ifdef TESTING_LOG_DATESTAMP case tod_log_datestamp_daily: case tod_log_datestamp_monthly: - (void) sprintf(CS timebuf, "%04d%02d%02d%02d%02d", - 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday, t->tm_hour, t->tm_min); - break; + off = sprintf(CS timebuf, "%04d%02d%02d%02d%02d", + 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday, t->tm_hour, t->tm_min); + break; - #else +#else case tod_log_datestamp_daily: - (void) sprintf(CS timebuf, "%04d%02d%02d", - 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday); - break; + off = sprintf(CS timebuf, "%04d%02d%02d", + 1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday); + break; case tod_log_datestamp_monthly: - (void) sprintf(CS timebuf, "%04d%02d", - 1900 + t->tm_year, 1 + t->tm_mon); - break; - #endif + off = sprintf(CS timebuf, "%04d%02d", + 1900 + t->tm_year, 1 + t->tm_mon); + break; +#endif - /* Format used in BSD inbox separator lines. Sort-of documented in RFC 976 - ("UUCP Mail Interchange Format Standard") but only by example, not by - explicit definition. The examples show no timezone offsets, and some MUAs - appear to be sensitive to this, so Exim has been changed to remove the - timezone offsets that originally appeared. */ + /* Format used in BSD inbox separator lines. Sort-of documented in RFC 976 + ("UUCP Mail Interchange Format Standard") but only by example, not by + explicit definition. The examples show no timezone offsets, and some MUAs + appear to be sensitive to this, so Exim has been changed to remove the + timezone offsets that originally appeared. */ case tod_bsdin: - { - int len = Ustrftime(timebuf, sizeof(timebuf), "%a %b %d %H:%M:%S", t); - Ustrftime(timebuf + len, sizeof(timebuf) - len, " %Y", t); - } - break; - - /* Other types require the GMT offset to be calculated, or just set up in the - case of UTC timestamping. We need to take a copy of the local time first. */ - - default: - { - int diff_hour, diff_min; - struct tm local; - memcpy(&local, t, sizeof(struct tm)); - - if (timestamps_utc) { - diff_hour = diff_min = 0; - } - else - { - struct tm *gmt = gmtime(&now); - diff_min = 60*(local.tm_hour - gmt->tm_hour) + local.tm_min - gmt->tm_min; - if (local.tm_year != gmt->tm_year) - diff_min += (local.tm_year > gmt->tm_year)? 1440 : -1440; - else if (local.tm_yday != gmt->tm_yday) - diff_min += (local.tm_yday > gmt->tm_yday)? 1440 : -1440; - diff_hour = diff_min/60; - diff_min = abs(diff_min - diff_hour*60); + int len = Ustrftime(timebuf, sizeof(timebuf), "%a %b %d %H:%M:%S", t); + Ustrftime(timebuf + len, sizeof(timebuf) - len, " %Y", t); } + break; + + /* Other types require the GMT offset to be calculated, or just set up in the + case of UTC timestamping. We need to take a copy of the local time first. */ - switch(type) + default: { - case tod_log_zone: /* Format used in logging with timezone */ - (void) sprintf(CS timebuf, "%04d-%02d-%02d %02d:%02d:%02d %+03d%02d", - 1900 + local.tm_year, 1 + local.tm_mon, local.tm_mday, - local.tm_hour, local.tm_min, local.tm_sec, - diff_hour, diff_min); - break; - - case tod_zone: /* Just the timezone offset */ - (void) sprintf(CS timebuf, "%+03d%02d", diff_hour, diff_min); - break; - - /* tod_mbx: format used in MBX mailboxes - subtly different to tod_full */ - - #ifdef SUPPORT_MBX - case tod_mbx: - { - int len; - (void) sprintf(CS timebuf, "%02d-", local.tm_mday); - len = Ustrlen(timebuf); - len += Ustrftime(timebuf + len, sizeof(timebuf) - len, "%b-%Y %H:%M:%S", - &local); - (void) sprintf(CS timebuf + len, " %+03d%02d", diff_hour, diff_min); - } - break; - #endif - - /* tod_full: format used in Received: headers (use as default just in case - called with a junk type value) */ - - default: - { - int len = Ustrftime(timebuf, sizeof(timebuf), "%a, ", &local); - (void) sprintf(CS timebuf + len, "%02d ", local.tm_mday); - len += Ustrlen(timebuf + len); - len += Ustrftime(timebuf + len, sizeof(timebuf) - len, "%b %Y %H:%M:%S", - &local); - (void) sprintf(CS timebuf + len, " %+03d%02d", diff_hour, diff_min); - } - break; + int diff_hour, diff_min; + struct tm local; + memcpy(&local, t, sizeof(struct tm)); + + if (timestamps_utc) + diff_hour = diff_min = 0; + else + { + struct tm * gmt = gmtime(&now.tv_sec); + + diff_min = 60*(local.tm_hour - gmt->tm_hour) + local.tm_min - gmt->tm_min; + if (local.tm_year != gmt->tm_year) + diff_min += (local.tm_year > gmt->tm_year)? 1440 : -1440; + else if (local.tm_yday != gmt->tm_yday) + diff_min += (local.tm_yday > gmt->tm_yday)? 1440 : -1440; + diff_hour = diff_min/60; + diff_min = abs(diff_min - diff_hour*60); + } + + switch(type) + { + case tod_log_zone: /* Format used in logging with timezone */ +#ifndef COMPILE_UTILITY + if (LOGGING(millisec)) + (void) sprintf(CS timebuf, + "%04d-%02d-%02d %02d:%02d:%02d.%03d %+03d%02d", + 1900 + local.tm_year, 1 + local.tm_mon, local.tm_mday, + local.tm_hour, local.tm_min, local.tm_sec, now.tv_usec/1000, + diff_hour, diff_min); + else +#endif + (void) sprintf(CS timebuf, + "%04d-%02d-%02d %02d:%02d:%02d %+03d%02d", + 1900 + local.tm_year, 1 + local.tm_mon, local.tm_mday, + local.tm_hour, local.tm_min, local.tm_sec, + diff_hour, diff_min); + break; + + case tod_zone: /* Just the timezone offset */ + (void) sprintf(CS timebuf, "%+03d%02d", diff_hour, diff_min); + break; + + /* tod_mbx: format used in MBX mailboxes - subtly different to tod_full */ + + #ifdef SUPPORT_MBX + case tod_mbx: + { + int len; + (void) sprintf(CS timebuf, "%02d-", local.tm_mday); + len = Ustrlen(timebuf); + len += Ustrftime(timebuf + len, sizeof(timebuf) - len, "%b-%Y %H:%M:%S", + &local); + (void) sprintf(CS timebuf + len, " %+03d%02d", diff_hour, diff_min); + } + break; + #endif + + /* tod_full: format used in Received: headers (use as default just in case + called with a junk type value) */ + + default: + { + int len = Ustrftime(timebuf, sizeof(timebuf), "%a, ", &local); + (void) sprintf(CS timebuf + len, "%02d ", local.tm_mday); + len += Ustrlen(timebuf + len); + len += Ustrftime(timebuf + len, sizeof(timebuf) - len, "%b %Y %H:%M:%S", + &local); + (void) sprintf(CS timebuf + len, " %+03d%02d", diff_hour, diff_min); + } + break; + } } - } - break; + break; } +#ifndef COMPILE_UTILITY +if (LOGGING(millisec) && off > 0) + (void) sprintf(CS timebuf + off, ".%03d", now.tv_usec/1000); +#endif + return timebuf; } diff --git a/test/log/0551 b/test/log/0551 index 831e11a89..189bd380d 100644 --- a/test/log/0551 +++ b/test/log/0551 @@ -6,6 +6,6 @@ 1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 => userx R=r1 T=t1 1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 => userz R=r1 T=t1 1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 Completed -1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss -1999-03-02 09:44:33 10HmaZ-0005vi-00 => userx R=r1 T=t1 -1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 => userx R=r1 T=t1 +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 Completed diff --git a/test/runtest b/test/runtest index 0e7e8ee0b..b7339f5d0 100755 --- a/test/runtest +++ b/test/runtest @@ -487,7 +487,8 @@ RESET_AFTER_EXTRA_LINE_READ: /Tue, 2 Mar 1999 09:44:33 +0000/gx; # Date/time in logs and in one instance of a filter test - s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d(\s[+-]\d\d\d\d)?/1999-03-02 09:44:33/gx; + s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d(\s[+-]\d\d\d\d)?\s/1999-03-02 09:44:33 /gx; + s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\.\d{3}(\s[+-]\d\d\d\d)?\s/2017-07-30 18:51:05.712 /gx; s/^Logwrite\s"\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Logwrite "1999-03-02 09:44:33/gx; # Date/time in message separators diff --git a/test/scripts/0000-Basic/0551 b/test/scripts/0000-Basic/0551 index d11064c00..b91f7e206 100644 --- a/test/scripts/0000-Basic/0551 +++ b/test/scripts/0000-Basic/0551 @@ -1,4 +1,4 @@ -# log_selector = +pid +# log_selector = +pid , +millisec # exim -odi userx@test.ex usery@test.ex Message 1 @@ -6,7 +6,7 @@ Message 1 exim -odi userx@test.ex userz@test.ex Message 2 **** -exim -DLOG_SELECTOR= -odi userx@test.ex +exim -DLOG_SELECTOR=+millisec -odi userx@test.ex Message 3 **** exigrep userx diff --git a/test/stdout/0551 b/test/stdout/0551 index 8af281557..9212d1410 100644 --- a/test/stdout/0551 +++ b/test/stdout/0551 @@ -8,9 +8,9 @@ 1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 => userz R=r1 T=t1 1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 Completed -1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss -1999-03-02 09:44:33 10HmaZ-0005vi-00 => userx R=r1 T=t1 -1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 => userx R=r1 T=t1 +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 Completed exigrep exit code = 0 -- cgit v1.2.3 From 32dfdf8baa8ccf091a0d5d4d75e8627424898756 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Fri, 4 Aug 2017 15:13:17 +0100 Subject: Logging: millisecond QT and DT. Bug 2102 --- doc/doc-docbook/spec.xfpt | 6 ++- src/exim_monitor/em_globals.c | 2 +- src/exim_monitor/em_queue.c | 2 +- src/src/deliver.c | 102 +++++++++++++++++++++++++++++++----------- src/src/exim.c | 9 ++-- src/src/expand.c | 4 +- src/src/functions.h | 4 +- src/src/globals.c | 3 +- src/src/globals.h | 2 +- src/src/queue.c | 2 +- src/src/readconf.c | 8 ++-- src/src/receive.c | 4 +- src/src/retry.c | 8 ++-- src/src/spool_in.c | 14 ++++-- src/src/spool_out.c | 26 ++++++----- src/src/structs.h | 1 + src/src/transport.c | 5 ++- src/src/transports/smtp.c | 15 ++++--- test/log/0506 | 2 +- test/log/0551 | 8 ++-- test/runtest | 9 ++-- test/scripts/0000-Basic/0551 | 4 +- test/stdout/0035 | 7 +++ test/stdout/0245 | 1 + test/stdout/0250 | 3 ++ test/stdout/0254 | 3 ++ test/stdout/0311 | 2 + test/stdout/0338 | 1 + test/stdout/0377 | 1 + test/stdout/0378 | 1 + test/stdout/0379 | 1 + test/stdout/0389 | 1 + test/stdout/0488 | 1 + test/stdout/0490 | 1 + test/stdout/0514 | 1 + test/stdout/0551 | 8 ++-- test/stdout/3415 | 5 +++ 37 files changed, 191 insertions(+), 86 deletions(-) (limited to 'test/stdout') diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index a4b1926ec..0d03c2f9c 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -36118,7 +36118,7 @@ selection marked by asterisks: &` incoming_interface `& local interface on <= and => lines &` incoming_port `& remote port on <= lines &`*lost_incoming_connection `& as it says (includes timeouts) -&` millisec `& millisecond timestamps +&` millisec `& millisecond timestamps and QT/DT times &` outgoing_interface `& local interface on => lines &` outgoing_port `& add remote port to => lines &`*queue_run `& start and end queue runs @@ -36208,6 +36208,8 @@ process is started because &%queue_only%& is set or &%-odq%& was used. .cindex "log" "delivery duration" &%deliver_time%&: For each delivery, the amount of real time it has taken to perform the actual delivery is logged as DT=<&'time'&>, for example, &`DT=1s`&. +If millisecond logging is enabled, short times will be shown with greater +precision, eg. &`DT=0.304`&. .next .cindex "log" "message size on delivery" .cindex "size" "of message" @@ -36329,6 +36331,8 @@ includes reception time as well as the delivery time for the current address. This means that it may be longer than the difference between the arrival and delivery log line times, because the arrival log line is not written until the message has been successfully received. +If millisecond logging is enabled, short times will be shown with greater +precision, eg. &`QT=1.578s`&. .next &%queue_time_overall%&: The amount of time the message has been in the queue on the local host is logged as QT=<&'time'&> on &"Completed"& lines, for diff --git a/src/exim_monitor/em_globals.c b/src/exim_monitor/em_globals.c index 54032362c..46ad834da 100644 --- a/src/exim_monitor/em_globals.c +++ b/src/exim_monitor/em_globals.c @@ -191,7 +191,7 @@ uschar *queue_name = US""; int received_count = 0; uschar *received_protocol = NULL; -int received_time = 0; +struct timeval received_time = { 0, 0 }; int recipients_count = 0; recipient_item *recipients_list = NULL; int recipients_list_max = 0; diff --git a/src/exim_monitor/em_queue.c b/src/exim_monitor/em_queue.c index cb284eb48..e6b1e91d9 100644 --- a/src/exim_monitor/em_queue.c +++ b/src/exim_monitor/em_queue.c @@ -204,7 +204,7 @@ if it's there. */ else { - q->update_time = q->input_time = received_time; + q->update_time = q->input_time = received_time.tv_sec; if ((p = strstric(sender_address+1, qualify_domain, FALSE)) != NULL && *(--p) == '@') *p = 0; } diff --git a/src/src/deliver.c b/src/src/deliver.c index 49281f93d..2713cc56f 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -382,6 +382,7 @@ for (addr2 = addr->next; addr2; addr2 = addr2->next) addr2->transport_return = addr->transport_return; addr2->basic_errno = addr->basic_errno; addr2->more_errno = addr->more_errno; + addr2->delivery_usec = addr->delivery_usec; addr2->special_action = addr->special_action; addr2->message = addr->message; addr2->user_message = addr->user_message; @@ -1029,6 +1030,43 @@ return str; } + +void +timesince(struct timeval * diff, struct timeval * then) +{ +gettimeofday(diff, NULL); +diff->tv_sec -= then->tv_sec; +if ((diff->tv_usec -= then->tv_usec) < 0) + { + diff->tv_sec--; + diff->tv_usec += 1000*1000; + } +} + + + +static uschar * +string_timediff(struct timeval * diff) +{ +static uschar buf[sizeof("0.000s")]; + +if (diff->tv_sec >= 5 || !LOGGING(millisec)) + return readconf_printtime((int)diff->tv_sec); + +sprintf(CS buf, "%d.%03ds", (int)diff->tv_sec, (int)diff->tv_usec/1000); +return buf; +} + + +static uschar * +string_timesince(struct timeval * then) +{ +struct timeval diff; + +timesince(&diff, then); +return string_timediff(&diff); +} + /******************************************************************************/ @@ -1191,11 +1229,13 @@ if ( LOGGING(smtp_confirmation) if (LOGGING(queue_time)) s = string_append(s, &size, &ptr, 2, US" QT=", - readconf_printtime( (int) ((long)time(NULL) - (long)received_time)) ); + string_timesince(&received_time)); if (LOGGING(deliver_time)) - s = string_append(s, &size, &ptr, 2, US" DT=", - readconf_printtime(addr->more_errno)); + { + struct timeval diff = {addr->more_errno, addr->delivery_usec}; + s = string_append(s, &size, &ptr, 2, US" DT=", string_timediff(&diff)); + } /* string_cat() always leaves room for the terminator. Release the store we used to build the line after writing it. */ @@ -2347,6 +2387,7 @@ if ((pid = fork()) == 0) || (ret = write(pfd[pipe_write], &addr2->flags, sizeof(addr2->flags))) != sizeof(addr2->flags) || (ret = write(pfd[pipe_write], &addr2->basic_errno, sizeof(int))) != sizeof(int) || (ret = write(pfd[pipe_write], &addr2->more_errno, sizeof(int))) != sizeof(int) + || (ret = write(pfd[pipe_write], &addr2->delivery_usec, sizeof(int))) != sizeof(int) || (ret = write(pfd[pipe_write], &addr2->special_action, sizeof(int))) != sizeof(int) || (ret = write(pfd[pipe_write], &addr2->transport, sizeof(transport_instance *))) != sizeof(transport_instance *) @@ -2414,6 +2455,7 @@ for (addr2 = addr; addr2; addr2 = addr2->next) len = read(pfd[pipe_read], &addr2->flags, sizeof(addr2->flags)); len = read(pfd[pipe_read], &addr2->basic_errno, sizeof(int)); len = read(pfd[pipe_read], &addr2->more_errno, sizeof(int)); + len = read(pfd[pipe_read], &addr2->delivery_usec, sizeof(int)); len = read(pfd[pipe_read], &addr2->special_action, sizeof(int)); len = read(pfd[pipe_read], &addr2->transport, sizeof(transport_instance *)); @@ -2639,8 +2681,8 @@ time_t now = time(NULL); while (addr_local) { - time_t delivery_start; - int deliver_time; + struct timeval delivery_start; + struct timeval deliver_time; address_item *addr2, *addr3, *nextaddr; int logflags = LOG_MAIN; int logchar = dont_deliver? '*' : '='; @@ -2936,9 +2978,10 @@ while (addr_local) single delivery. */ deliver_set_expansions(addr); - delivery_start = time(NULL); + + gettimeofday(&delivery_start, NULL); deliver_local(addr, FALSE); - deliver_time = (int)(time(NULL) - delivery_start); + timesince(&deliver_time, &delivery_start); /* If a shadow transport (which must perforce be another local transport), is defined, and its condition is met, we must pass the message to the shadow @@ -3075,7 +3118,11 @@ while (addr_local) /* Done with this address */ - if (result == OK) addr2->more_errno = deliver_time; + if (result == OK) + { + addr2->more_errno = deliver_time.tv_sec; + addr2->delivery_usec = deliver_time.tv_usec; + } post_process_one(addr2, result, logflags, DTYPE_TRANSPORT, logchar); /* If a pipe delivery generated text to be sent back, the result may be @@ -3417,9 +3464,9 @@ while (!done) r->flags = *ptr++; r->key = string_copy(ptr); while (*ptr++); - memcpy(&(r->basic_errno), ptr, sizeof(r->basic_errno)); + memcpy(&r->basic_errno, ptr, sizeof(r->basic_errno)); ptr += sizeof(r->basic_errno); - memcpy(&(r->more_errno), ptr, sizeof(r->more_errno)); + memcpy(&r->more_errno, ptr, sizeof(r->more_errno)); ptr += sizeof(r->more_errno); r->message = *ptr ? string_copy(ptr) : NULL; DEBUG(D_deliver|D_retry) debug_printf(" added %s item\n", @@ -3561,11 +3608,13 @@ while (!done) DEBUG(D_deliver) debug_printf("A0 %s tret %d\n", addr->address, *ptr); addr->transport_return = *ptr++; addr->special_action = *ptr++; - memcpy(&(addr->basic_errno), ptr, sizeof(addr->basic_errno)); + memcpy(&addr->basic_errno, ptr, sizeof(addr->basic_errno)); ptr += sizeof(addr->basic_errno); - memcpy(&(addr->more_errno), ptr, sizeof(addr->more_errno)); + memcpy(&addr->more_errno, ptr, sizeof(addr->more_errno)); ptr += sizeof(addr->more_errno); - memcpy(&(addr->flags), ptr, sizeof(addr->flags)); + memcpy(&addr->delivery_usec, ptr, sizeof(addr->delivery_usec)); + ptr += sizeof(addr->delivery_usec); + memcpy(&addr->flags, ptr, sizeof(addr->flags)); ptr += sizeof(addr->flags); addr->message = *ptr ? string_copy(ptr) : NULL; while(*ptr++); @@ -4815,9 +4864,9 @@ for (delivery_count = 0; addr_remote; delivery_count++) { sprintf(CS big_buffer, "%c%.500s", r->flags, r->key); ptr = big_buffer + Ustrlen(big_buffer+2) + 3; - memcpy(ptr, &(r->basic_errno), sizeof(r->basic_errno)); + memcpy(ptr, &r->basic_errno, sizeof(r->basic_errno)); ptr += sizeof(r->basic_errno); - memcpy(ptr, &(r->more_errno), sizeof(r->more_errno)); + memcpy(ptr, &r->more_errno, sizeof(r->more_errno)); ptr += sizeof(r->more_errno); if (!r->message) *ptr++ = 0; else { @@ -4866,11 +4915,13 @@ for (delivery_count = 0; addr_remote; delivery_count++) sprintf(CS big_buffer, "%c%c", addr->transport_return, addr->special_action); ptr = big_buffer + 2; - memcpy(ptr, &(addr->basic_errno), sizeof(addr->basic_errno)); + memcpy(ptr, &addr->basic_errno, sizeof(addr->basic_errno)); ptr += sizeof(addr->basic_errno); - memcpy(ptr, &(addr->more_errno), sizeof(addr->more_errno)); + memcpy(ptr, &addr->more_errno, sizeof(addr->more_errno)); ptr += sizeof(addr->more_errno); - memcpy(ptr, &(addr->flags), sizeof(addr->flags)); + memcpy(ptr, &addr->delivery_usec, sizeof(addr->delivery_usec)); + ptr += sizeof(addr->delivery_usec); + memcpy(ptr, &addr->flags, sizeof(addr->flags)); ptr += sizeof(addr->flags); if (!addr->message) *ptr++ = 0; else @@ -4883,7 +4934,7 @@ for (delivery_count = 0; addr_remote; delivery_count++) { ptr += sprintf(CS ptr, "%.256s", addr->host_used->name) + 1; ptr += sprintf(CS ptr, "%.64s", addr->host_used->address) + 1; - memcpy(ptr, &(addr->host_used->port), sizeof(addr->host_used->port)); + memcpy(ptr, &addr->host_used->port, sizeof(addr->host_used->port)); ptr += sizeof(addr->host_used->port); /* DNS lookup status */ @@ -5559,14 +5610,15 @@ give up; if the message has been around for sufficiently long, remove it. */ if (rc != spool_read_hdrerror) { - received_time = 0; + received_time.tv_sec = received_time.tv_usec = 0; + /*XXX subsec precision?*/ for (i = 0; i < 6; i++) - received_time = received_time * BASE_62 + tab62[id[i] - '0']; + received_time.tv_sec = received_time.tv_sec * BASE_62 + tab62[id[i] - '0']; } /* If we've had this malformed message too long, sling it. */ - if (now - received_time > keep_malformed) + if (now - received_time.tv_sec > keep_malformed) { Uunlink(spool_fname(US"msglog", message_subdir, id, US"")); Uunlink(spool_fname(US"input", message_subdir, id, US"-D")); @@ -7792,7 +7844,7 @@ wording. */ if (rc != 0) { uschar *s = US""; - if (now - received_time < retry_maximum_timeout && !addr_defer) + if (now - received_time.tv_sec < retry_maximum_timeout && !addr_defer) { addr_defer = (address_item *)(+1); deliver_freeze = TRUE; @@ -7878,7 +7930,7 @@ if (!addr_defer) if (LOGGING(queue_time_overall)) log_write(0, LOG_MAIN, "Completed QT=%s", - readconf_printtime( (int) ((long)time(NULL) - (long)received_time)) ); + string_timesince(&received_time)); else log_write(0, LOG_MAIN, "Completed"); @@ -8016,7 +8068,7 @@ else if (addr_defer != (address_item *)(+1)) { int count; int show_time; - int queue_time = time(NULL) - received_time; + int queue_time = time(NULL) - received_time.tv_sec; /* When running in the test harness, there's an option that allows us to fudge this time so as to get repeatability of the tests. Take the first diff --git a/src/src/exim.c b/src/src/exim.c index 581b16190..c990bc08c 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -345,7 +345,7 @@ Arguments: Returns: -1, 0, or +1 */ -int +static int exim_tvcmp(struct timeval *t1, struct timeval *t2) { if (t1->tv_sec > t2->tv_sec) return +1; @@ -4537,8 +4537,9 @@ if (test_retry_arg >= 0) } } - yield = retry_find_config(s1, s2, basic_errno, more_errno); - if (yield == NULL) printf("No retry information found\n"); else + if (!(yield = retry_find_config(s1, s2, basic_errno, more_errno))) + printf("No retry information found\n"); + else { retry_rule *r; more_errno = yield->more_errno; @@ -4570,7 +4571,7 @@ if (test_retry_arg >= 0) printf("auth_failed "); else printf("* "); - for (r = yield->rules; r != NULL; r = r->next) + for (r = yield->rules; r; r = r->next) { printf("%c,%s", r->rule, readconf_printtime(r->timeout)); /* Do not */ printf(",%s", readconf_printtime(r->p1)); /* amalgamate */ diff --git a/src/src/expand.c b/src/src/expand.c index 4eb1818f1..d8d36a3b5 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -642,7 +642,7 @@ static var_entry var_table[] = { { "received_ip_address", vtype_stringptr, &interface_address }, { "received_port", vtype_int, &interface_port }, { "received_protocol", vtype_stringptr, &received_protocol }, - { "received_time", vtype_int, &received_time }, + { "received_time", vtype_int, &received_time.tv_sec }, { "recipient_data", vtype_stringptr, &recipient_data }, { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure }, { "recipients", vtype_string_func, &fn_recipients }, @@ -1484,9 +1484,7 @@ while (*s != 0) /* If value2 is unset, just compute one number */ if (value2 < 0) - { s = string_sprintf("%d", total % value1); - } /* Otherwise do a div/mod hash */ diff --git a/src/src/functions.h b/src/src/functions.h index 9c9caaf97..5b2a683b8 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -184,7 +184,6 @@ extern const uschar * exim_errstr(int); extern void exim_exit(int); extern void exim_nullstd(void); extern void exim_setugid(uid_t, gid_t, BOOL, uschar *); -extern int exim_tvcmp(struct timeval *, struct timeval *); extern void exim_wait_tick(struct timeval *, int); extern int exp_bool(address_item *addr, uschar *mtype, uschar *mname, unsigned dgb_opt, uschar *oname, BOOL bvalue, @@ -474,9 +473,10 @@ extern int strcmpic(const uschar *, const uschar *); extern int strncmpic(const uschar *, const uschar *, int); extern uschar *strstric(uschar *, uschar *, BOOL); +extern void timesince(struct timeval * diff, struct timeval * then); +extern void tls_modify_variables(tls_support *); extern uschar *tod_stamp(int); -extern void tls_modify_variables(tls_support *); extern BOOL transport_check_waiting(const uschar *, const uschar *, int, uschar *, BOOL *, oicf, void*); extern void transport_init(void); diff --git a/src/src/globals.c b/src/src/globals.c index 8c35c6fe2..894b59967 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -383,6 +383,7 @@ address_item address_defaults = { { 0 }, /* localpart_cache - ditto */ -1, /* mode */ 0, /* more_errno */ + 0, /* delivery_usec */ ERRNO_UNKNOWNERROR, /* basic_errno */ 0, /* child_count */ -1, /* return_file */ @@ -1107,7 +1108,7 @@ uschar *received_header_text = US int received_headers_max = 30; uschar *received_protocol = NULL; -int received_time = 0; +struct timeval received_time = { 0, 0 }; uschar *recipient_data = NULL; uschar *recipient_unqualified_hosts = NULL; uschar *recipient_verify_failure = NULL; diff --git a/src/src/globals.h b/src/src/globals.h index 4a54c3c73..2d26bd0d7 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -721,7 +721,7 @@ extern int received_count; /* Count of Received: headers */ extern uschar *received_for; /* For "for" field */ extern uschar *received_header_text; /* Definition of Received: header */ extern int received_headers_max; /* Max count of Received: headers */ -extern int received_time; /* Time the message was received */ +extern struct timeval received_time; /* Time the message was received */ extern uschar *recipient_data; /* lookup data for recipients */ extern uschar *recipient_unqualified_hosts; /* Permitted unqualified recipients */ extern uschar *recipient_verify_failure; /* What went wrong */ diff --git a/src/src/queue.c b/src/src/queue.c index 5d8d610c6..60bf2ce77 100644 --- a/src/src/queue.c +++ b/src/src/queue.c @@ -880,7 +880,7 @@ for (reset_point = store_get(0); f; f = f->next) if (Ustat(fname, &statbuf) == 0) size = message_size + statbuf.st_size - SPOOL_DATA_START_OFFSET + 1; - i = (now - received_time)/60; /* minutes on queue */ + i = (now - received_time.tv_sec)/60; /* minutes on queue */ if (i > 90) { i = (i + 30)/60; diff --git a/src/src/readconf.c b/src/src/readconf.c index df3fe823c..1ed93eb64 100644 --- a/src/src/readconf.c +++ b/src/src/readconf.c @@ -2298,10 +2298,10 @@ t /= 24; d = t % 7; w = t/7; -if (w > 0) { sprintf(CS p, "%dw", w); while (*p) p++; } -if (d > 0) { sprintf(CS p, "%dd", d); while (*p) p++; } -if (h > 0) { sprintf(CS p, "%dh", h); while (*p) p++; } -if (m > 0) { sprintf(CS p, "%dm", m); while (*p) p++; } +if (w > 0) p += sprintf(CS p, "%dw", w); +if (d > 0) p += sprintf(CS p, "%dd", d); +if (h > 0) p += sprintf(CS p, "%dh", h); +if (m > 0) p += sprintf(CS p, "%dm", m); if (s > 0 || p == time_buffer) sprintf(CS p, "%ds", s); return time_buffer; diff --git a/src/src/receive.c b/src/src/receive.c index 9561a4baf..71026ff4a 100644 --- a/src/src/receive.c +++ b/src/src/receive.c @@ -1743,9 +1743,9 @@ message id creation below. */ /* For other uses of the received time we can operate with granularity of one second, and for that we use the global variable received_time. This is for -things like ultimate message timeouts. */ +things like ultimate message timeouts.XXX */ -received_time = message_id_tv.tv_sec; +received_time = message_id_tv; /* If SMTP input, set the special handler for timeouts. The alarm() calls happen in the smtp_getc() function when it refills its buffer. */ diff --git a/src/src/retry.c b/src/src/retry.c index abfeeb4fb..0bb33a053 100644 --- a/src/src/retry.c +++ b/src/src/retry.c @@ -54,8 +54,8 @@ if (retry != NULL && retry->rules != NULL) last_rule = last_rule->next); DEBUG(D_retry) debug_printf(" received_time=%d diff=%d timeout=%d\n", - received_time, (int)(now - received_time), last_rule->timeout); - address_timeout = (now - received_time > last_rule->timeout); + received_time.tv_sec, (int)(now - received_time.tv_sec), last_rule->timeout); + address_timeout = (now - received_time.tv_sec > last_rule->timeout); } else { @@ -754,7 +754,7 @@ for (i = 0; i < 3; i++) this is a small bit of code, and it does no harm to leave it in place, just in case. */ - if ( received_time <= retry_record->first_failed + if ( received_time.tv_sec <= retry_record->first_failed && addr == endaddr && !retry_record->expired && rule) @@ -762,7 +762,7 @@ for (i = 0; i < 3; i++) retry_rule *last_rule; for (last_rule = rule; last_rule->next; last_rule = last_rule->next) ; - if (now - received_time > last_rule->timeout) + if (now - received_time.tv_sec > last_rule->timeout) { DEBUG(D_retry) debug_printf("on queue longer than maximum retry\n"); timedout_count++; diff --git a/src/src/spool_in.c b/src/src/spool_in.c index 0bdf92e3b..a5b14959f 100644 --- a/src/src/spool_in.c +++ b/src/src/spool_in.c @@ -397,10 +397,11 @@ sender_address[n-3] = 0; /* time */ if (Ufgets(big_buffer, big_buffer_size, f) == NULL) goto SPOOL_READ_ERROR; -if (sscanf(CS big_buffer, "%d %d", &received_time, &warning_count) != 2) +if (sscanf(CS big_buffer, TIME_T_FMT " %d", &received_time.tv_sec, &warning_count) != 2) goto SPOOL_FORMAT_ERROR; +received_time.tv_usec = 0; -message_age = time(NULL) - received_time; +message_age = time(NULL) - received_time.tv_sec; #ifndef COMPILE_UTILITY DEBUG(D_deliver) debug_printf("user=%s uid=%ld gid=%ld sender=%s\n", @@ -573,7 +574,8 @@ for (;;) break; case 'l': - if (Ustrcmp(p, "ocal") == 0) sender_local = TRUE; + if (Ustrcmp(p, "ocal") == 0) + sender_local = TRUE; else if (Ustrcmp(big_buffer, "-localerror") == 0) local_error_message = TRUE; else if (Ustrncmp(p, "ocal_scan ", 10) == 0) @@ -593,6 +595,12 @@ for (;;) case 'r': if (Ustrncmp(p, "eceived_protocol", 16) == 0) received_protocol = string_copy(big_buffer + 19); + else if (Ustrncmp(p, "eceived_time_usec", 17) == 0) + { + unsigned usec; + if (sscanf(CS big_buffer + 21, "%u", &usec) == 1) + received_time.tv_usec = usec; + } break; case 's': diff --git a/src/src/spool_out.c b/src/src/spool_out.c index ebe089d4f..ac3927910 100644 --- a/src/src/spool_out.c +++ b/src/src/spool_out.c @@ -154,26 +154,28 @@ fprintf(f, "%s-H\n", message_id); fprintf(f, "%.63s %ld %ld\n", originator_login, (long int)originator_uid, (long int)originator_gid); fprintf(f, "<%s>\n", sender_address); -fprintf(f, "%d %d\n", received_time, warning_count); +fprintf(f, "%d %d\n", received_time.tv_sec, warning_count); + +fprintf(f, "-received_time_usec .%06d\n", received_time.tv_usec); /* If there is information about a sending host, remember it. The HELO data can be set for local SMTP as well as remote. */ -if (sender_helo_name != NULL) +if (sender_helo_name) fprintf(f, "-helo_name %s\n", sender_helo_name); -if (sender_host_address != NULL) +if (sender_host_address) { fprintf(f, "-host_address %s.%d\n", sender_host_address, sender_host_port); - if (sender_host_name != NULL) + if (sender_host_name) fprintf(f, "-host_name %s\n", sender_host_name); - if (sender_host_authenticated != NULL) + if (sender_host_authenticated) fprintf(f, "-host_auth %s\n", sender_host_authenticated); } /* Also about the interface a message came in on */ -if (interface_address != NULL) +if (interface_address) fprintf(f, "-interface_address %s.%d\n", interface_address, interface_port); if (smtp_active_hostname != primary_hostname) @@ -183,11 +185,11 @@ if (smtp_active_hostname != primary_hostname) likely to be the same as originator_login, but will be different if the originator was root, forcing a different ident. */ -if (sender_ident != NULL) fprintf(f, "-ident %s\n", sender_ident); +if (sender_ident) fprintf(f, "-ident %s\n", sender_ident); /* Ditto for the received protocol */ -if (received_protocol != NULL) +if (received_protocol) fprintf(f, "-received_protocol %s\n", received_protocol); /* Preserve any ACL variables that are set. */ @@ -205,9 +207,9 @@ fprintf(f, "-max_received_linelength %d\n", max_received_linelength); if (body_zerocount > 0) fprintf(f, "-body_zerocount %d\n", body_zerocount); -if (authenticated_id != NULL) +if (authenticated_id) fprintf(f, "-auth_id %s\n", authenticated_id); -if (authenticated_sender != NULL) +if (authenticated_sender) fprintf(f, "-auth_sender %s\n", authenticated_sender); if (allow_unqualified_recipient) fprintf(f, "-allow_unqualified_recipient\n"); @@ -261,7 +263,7 @@ if (message_smtputf8) /* Write the dsn flags to the spool header file */ DEBUG(D_deliver) debug_printf("DSN: Write SPOOL :-dsn_envid %s\n", dsn_envid); -if (dsn_envid != NULL) fprintf(f, "-dsn_envid %s\n", dsn_envid); +if (dsn_envid) fprintf(f, "-dsn_envid %s\n", dsn_envid); DEBUG(D_deliver) debug_printf("DSN: Write SPOOL :-dsn_ret %d\n", dsn_ret); if (dsn_ret != 0) fprintf(f, "-dsn_ret %d\n", dsn_ret); @@ -316,7 +318,7 @@ various other headers, or an asterisk for old headers that have been rewritten. These are saved as a record for debugging. Don't included them in the message's size. */ -for (h = header_list; h != NULL; h = h->next) +for (h = header_list; h; h = h->next) { fprintf(f, "%03d%c %s", h->slen, h->type, h->text); size_correction += 5; diff --git a/src/src/structs.h b/src/src/structs.h index 885c1b500..3de8f3d41 100644 --- a/src/src/structs.h +++ b/src/src/structs.h @@ -631,6 +631,7 @@ typedef struct address_item { int mode; /* mode for local transporting to a file */ int more_errno; /* additional error information */ /* (may need to hold a timestamp) */ + unsigned int delivery_usec; /* subsecond part of delivery time */ short int basic_errno; /* status after failure */ unsigned short child_count; /* number of child addresses */ diff --git a/src/src/transport.c b/src/src/transport.c index b26251c88..c6b4c77ef 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -1253,6 +1253,8 @@ if ((write_pid = fork()) == 0) != sizeof(int) || write(pfd[pipe_write], (void *)&tctx->addr->more_errno, sizeof(int)) != sizeof(int) + || write(pfd[pipe_write], (void *)&tctx->addr->delivery_usec, sizeof(int)) + != sizeof(int) ) rc = FALSE; /* compiler quietening */ _exit(0); @@ -1377,7 +1379,8 @@ if (write_pid > 0) else if (!ok) { int dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int)); - dummy = read(pfd[pipe_read], (void *)&(tctx->addr->more_errno), sizeof(int)); + dummy = read(pfd[pipe_read], (void *)&tctx->addr->more_errno, sizeof(int)); + dummy = read(pfd[pipe_read], (void *)&tctx->addr->delivery_usec, sizeof(int)); yield = FALSE; } } diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 11a996c65..d424c182c 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -2714,7 +2714,7 @@ address_item *addr; int yield = OK; int save_errno; int rc; -time_t start_delivery_time = time(NULL); +struct timeval start_delivery_time; BOOL pass_message = FALSE; uschar *message = NULL; @@ -2723,6 +2723,7 @@ uschar *p; smtp_context sx; +gettimeofday(&start_delivery_time, NULL); suppress_tls = suppress_tls; /* stop compiler warning when no TLS support */ *message_defer = FALSE; @@ -3036,10 +3037,11 @@ else if (sx.ok) { int flag = '='; - int delivery_time = (int)(time(NULL) - start_delivery_time); + struct timeval delivery_time; int len; - uschar *conf = NULL; + uschar * conf = NULL; + timesince(&delivery_time, &start_delivery_time); sx.send_rset = FALSE; pipelining_active = FALSE; @@ -3114,7 +3116,8 @@ else actual host that was used. */ addr->transport_return = OK; - addr->more_errno = delivery_time; + addr->more_errno = delivery_time.tv_sec; + addr->delivery_usec = delivery_time.tv_usec; addr->host_used = host; addr->special_action = flag; addr->message = conf; @@ -4160,7 +4163,7 @@ for (cutoff_retry = 0; { if ( !host->address || host->status != hstatus_unusable_expired - || host->last_try > received_time) + || host->last_try > received_time.tv_sec) continue; DEBUG(D_transport) debug_printf("trying expired host %s [%s]%s\n", host->name, host->address, pistring); @@ -4480,7 +4483,7 @@ for (cutoff_retry = 0; for (last_rule = retry->rules; last_rule->next; last_rule = last_rule->next); - timedout = time(NULL) - received_time > last_rule->timeout; + timedout = time(NULL) - received_time.tv_sec > last_rule->timeout; } else timedout = TRUE; /* No rule => timed out */ diff --git a/test/log/0506 b/test/log/0506 index d74b21dab..e0140f58b 100644 --- a/test/log/0506 +++ b/test/log/0506 @@ -1,7 +1,7 @@ 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss 1999-03-02 09:44:33 10HmaX-0005vi-00 => discarded R=r1 1999-03-02 09:44:33 10HmaX-0005vi-00 => discarded R=r1 -1999-03-02 09:44:33 10HmaX-0005vi-00 Completed QT=0s +1999-03-02 09:44:33 10HmaX-0005vi-00 Completed QT=qqs 1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss 1999-03-02 09:44:33 10HmaY-0005vi-00 => discarded R=r1 1999-03-02 09:44:33 10HmaY-0005vi-00 == sieve-filter@test.ex R=r1 defer (-17): error in filter file: Sieve filtering not enabled diff --git a/test/log/0551 b/test/log/0551 index 189bd380d..5baae67cb 100644 --- a/test/log/0551 +++ b/test/log/0551 @@ -3,9 +3,9 @@ 1999-03-02 09:44:33 [1235] 10HmaX-0005vi-00 => usery R=r1 T=t1 1999-03-02 09:44:33 [1235] 10HmaX-0005vi-00 Completed 1999-03-02 09:44:33 [1236] 10HmaY-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss -1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 => userx R=r1 T=t1 -1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 => userz R=r1 T=t1 +1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 => userx R=r1 T=t1 QT=qqs +1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 => userz R=r1 T=t1 QT=qqs 1999-03-02 09:44:33 [1237] 10HmaY-0005vi-00 Completed 2017-07-30 18:51:05.712 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss -2017-07-30 18:51:05.712 10HmaZ-0005vi-00 => userx R=r1 T=t1 -2017-07-30 18:51:05.712 10HmaZ-0005vi-00 Completed +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 => userx R=r1 T=t1 QT=q.qqqs DT=q.qqqs +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 Completed QT=q.qqqs diff --git a/test/runtest b/test/runtest index b7339f5d0..90c1758ca 100755 --- a/test/runtest +++ b/test/runtest @@ -491,6 +491,9 @@ RESET_AFTER_EXTRA_LINE_READ: s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\.\d{3}(\s[+-]\d\d\d\d)?\s/2017-07-30 18:51:05.712 /gx; s/^Logwrite\s"\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Logwrite "1999-03-02 09:44:33/gx; + s/([QD]T=)\d+s/$1qqs/g; + s/([QD]T=)\d\.\d{3}s/$1q.qqqs/g; + # Date/time in message separators s/(?:[A-Z][a-z]{2}\s){2}\d\d\s\d\d:\d\d:\d\d\s\d\d\d\d /Tue Mar 02 09:44:33 1999/gx; @@ -518,9 +521,6 @@ RESET_AFTER_EXTRA_LINE_READ: # Date/time in exim -bV output s/\d\d-[A-Z][a-z]{2}-\d{4}\s\d\d:\d\d:\d\d/07-Mar-2000 12:21:52/g; - # Time on queue tolerance - s/(QT|D)=1s/$1=0s/; - # Eximstats heading s/Exim\sstatistics\sfrom\s\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\sto\s \d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Exim statistics from