Further TPDA events

 msg:complete
 msg:fail:internal
 msg:fail:delivery

10 files changed, 199 insertions, 43 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index 280c6a83e..837af5afc 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -11673,7 +11673,7 @@ This variable is like &$message_headers$& except that no processing of the
 contents of header lines is done.
 
 .vitem &$message_id$&
-This is an old name for &$message_exim_id$&, which is now deprecated.
+This is an old name for &$message_exim_id$&. It is now deprecated.
 
 .vitem &$message_linecount$&
 .vindex "&$message_linecount$&"
diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index b98ac7929..d8bd0bf46 100644
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -762,8 +762,10 @@ b. Configure, somewhere before the DATA ACL, the control option to
 Transport post-delivery actions
 --------------------------------------------------------------
 
-An arbitrary per-transport string can be expanded on successful delivery,
+An arbitrary per-transport string can be expanded upon various transport events
 and (for SMTP transports) a second string on deferrals caused by a host error.
+Additionally a main-section configuration option can be expanded on some
+per-message events.
 This feature may be used, for example, to write exim internal log information
 (not available otherwise) into a database.
 
@@ -773,18 +775,23 @@ EXPERIMENTAL_TPDA=yes
 
 in your Local/Makefile
 
-and define the tpda_event_action option in the transport, to
-be expanded when the event fires.
+and define one or both of
+- the tpda_event_action option in the transport
+- the delivery_event_action
+to be expanded when the event fires.
 
 A new variable, $tpda_event, is set to the event type when the
 expansion is done.  The current list of events is:
 
-	msg:delivery
-	msg:host:defer
-	tcp:connect
-	tcp:close
-	tls:cert
-	smtp:connect
+	msg:complete		main		per message
+	msg:delivery		transport	per recipient
+	msg:host:defer		transport	per attempt
+	msg:fail:delivery	main		per recipient
+	msg:fail:internal	main		per recipient
+	tcp:connect		transport	per connection
+	tcp:close		transport	per connection
+	tls:cert		transport	per certificate in verification chain
+	smtp:connect		transport	per connection
 
 The expansion is called for all event types, and should use the $tpda_event
 value to decide when to act.  The variable data is a colon-separated
@@ -800,7 +807,7 @@ content is event_dependent:
 
 The msg:host:defer event populates one extra variable, $tpda_defer_errno.
 
-The following variables are likely to be useful for most event types:
+The following variables are likely to be useful depending on the event type:
 
 	router_name, transport_name
 	local_part, domain
@@ -808,6 +815,7 @@ The following variables are likely to be useful for most event types:
 	tls_out_peercert
 	lookup_dnssec_authenticated, tls_out_dane
 	sending_ip_address, sending_port
+	message_exim_id
 
 
 An example might look like:
@@ -823,13 +831,10 @@ tpda_event_action = ${if = {msg:delivery}{$tpda_event} \
     '${quote_pgsql:$message_exim_id}')}} \
 } {}}
 
-The string is expanded after the delivery completes and any
+The string is expanded for each of the supported events and any
 side-effects will happen.  The result is then discarded.
 Note that for complex operations an ACL expansion can be used.
 
-During the expansion the tpda_event variable will contain the
-string-list "msg:delivery".
-
 
 The expansion of the tpda_event_action option should normally
 return an empty string.  Should it return anything else the
@@ -837,6 +842,7 @@ following will be forced:
 
 	msg:delivery	(ignored)
 	msg:host:defer	(ignored)
+	msg:fail:delivery (ignored)
 	tcp:connect	do not connect
 	tcp:close	(ignored)
 	tls:cert	refuse verification
diff --git a/src/src/deliver.c b/src/src/deliver.c
index 48d3fd7ec..b3a5a49b2 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -708,6 +708,8 @@ d_tlslog(uschar * s, int * sizep, int * ptrp, address_item * addr)
 #endif
 
 
+
+
 #ifdef EXPERIMENTAL_TPDA
 int
 tpda_raise_event(uschar * action, uschar * event, uschar * ev_data)
@@ -742,7 +744,32 @@ if (action)
   }
 return OK;
 }
-#endif
+
+static void
+tpda_msg_event(uschar * event, address_item * addr)
+{
+uschar * save_domain = deliver_domain;
+uschar * save_local =  deliver_localpart;
+
+if (!addr->transport)
+  return;
+
+router_name =    addr->router ? addr->router->name : NULL;
+transport_name = addr->transport->name;
+deliver_domain = addr->domain;
+deliver_localpart = addr->local_part;
+
+(void) tpda_raise_event(addr->transport->tpda_event_action, event,
+	  addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
+	  ? addr->message : NULL);
+
+deliver_localpart = save_local;
+deliver_domain =    save_domain;
+router_name = transport_name = NULL;
+}
+#endif	/*EXPERIMENTAL_TPDA*/
+
+
 
 /* If msg is NULL this is a delivery log and logchar is used. Otherwise
 this is a nonstandard call; no two-character delivery flag is written
@@ -902,24 +929,10 @@ s[ptr] = 0;
 log_write(0, flags, "%s", s);
 
 #ifdef EXPERIMENTAL_TPDA
-  {
-  uschar * save_domain = deliver_domain;
-  uschar * save_local =  deliver_localpart;
-
-  router_name =    addr->router ? addr->router->name : NULL;
-  transport_name = addr->transport ? addr->transport->name : NULL;
-  deliver_domain = addr->domain;
-  deliver_localpart = addr->local_part;
-
-  (void) tpda_raise_event(addr->transport->tpda_event_action, US"msg:delivery",
-	    addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
-	    ? addr->message : NULL);
-
-  deliver_localpart = save_local;
-  deliver_domain =    save_domain;
-  router_name = transport_name = NULL;
-  }
+/*XXX cutthrough calls this also for non-delivery...*/
+tpda_msg_event(US"msg:delivery", addr);
 #endif
+
 store_reset(reset_point);
 return;
 }
@@ -1347,6 +1360,11 @@ else
     deliver_msglog("%s %s\n", now, s);
 
   log_write(0, LOG_MAIN, "** %s", s);
+
+#ifdef EXPERIMENTAL_TPDA
+  tpda_msg_event(US"msg:fail:delivery", addr);
+#endif
+
   store_reset(reset_point);
   }
 
@@ -5462,6 +5480,25 @@ if (process_recipients != RECIP_IGNORE)
         addr_last = new;
         break;
         }
+
+#ifdef EXPERIMENTAL_TPDA
+      if (process_recipients != RECIP_ACCEPT)
+	{
+	uschar * save_local =  deliver_localpart;
+	uschar * save_domain = deliver_domain;
+
+	deliver_localpart = expand_string(
+		      string_sprintf("${local_part:%s}", new->address));
+	deliver_domain =    expand_string(
+		      string_sprintf("${domain:%s}", new->address));
+
+	(void) tpda_raise_event(delivery_event_action,
+		      US"msg:fail:internal", new->message);
+
+	deliver_localpart = save_local;
+	deliver_domain =    save_domain;
+	}
+#endif
       }
     }
   }
@@ -7217,7 +7254,11 @@ if (addr_defer == NULL)
 
   /* Unset deliver_freeze so that we won't try to move the spool files further down */
   deliver_freeze = FALSE;
-  }
+
+#ifdef EXPERIMENTAL_TPDA
+  (void) tpda_raise_event(delivery_event_action, US"msg:complete", NULL);
+#endif
+}
 
 /* If there are deferred addresses, we are keeping this message because it is
 not yet completed. Lose any temporary files that were catching output from
diff --git a/src/src/globals.c b/src/src/globals.c
index f1b771ad3..ef1c1fd02 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -1327,8 +1327,9 @@ BOOL    timestamps_utc         = FALSE;
 
 #ifdef EXPERIMENTAL_TPDA
 int     tpda_defer_errno        = 0;
-uschar *tpda_event              = NULL;
-uschar *tpda_data               = NULL;
+uschar *tpda_event              = NULL;	/* event name */
+uschar *tpda_data               = NULL;	/* auxilary data for event */
+uschar *delivery_event_action   = NULL;	/* expansion for delivery events */
 #endif
 
 transport_instance  *transports = NULL;
diff --git a/src/src/globals.h b/src/src/globals.h
index f0a3091df..73793aa2e 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -872,7 +872,8 @@ extern BOOL    timestamps_utc;         /* Use UTC for all times */
 #ifdef EXPERIMENTAL_TPDA
 extern int     tpda_defer_errno;        /* error number set when a remote delivery is deferred with a host error */
 extern uschar *tpda_event;		/* event classification */
-extern uschar *tpda_data;;		/* event data */
+extern uschar *tpda_data;		/* event data */
+extern uschar *delivery_event_action;   /* expansion for delivery events */
 #endif
 
 extern uschar *transport_name;         /* Name of transport last started */
diff --git a/src/src/readconf.c b/src/src/readconf.c
index adb538c59..2e18b670e 100644
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -205,6 +205,9 @@ static optionlist optionlist_config[] = {
   { "deliver_drop_privilege",   opt_bool,        &deliver_drop_privilege },
   { "deliver_queue_load_max",   opt_fixed,       &deliver_queue_load_max },
   { "delivery_date_remove",     opt_bool,        &delivery_date_remove },
+#ifdef EXPERIMENTAL_TPDA
+  { "delivery_event_action",    opt_stringptr,   &delivery_event_action },
+#endif
 #ifdef ENABLE_DISABLE_FSYNC
   { "disable_fsync",            opt_bool,        &disable_fsync },
 #endif
diff --git a/test/confs/5700 b/test/confs/5700
index 0856bb056..65a1b59ba 100644
--- a/test/confs/5700
+++ b/test/confs/5700
@@ -13,6 +13,8 @@ gecos_name = CALLER_NAME
 acl_smtp_rcpt = accept
 acl_smtp_data = accept
 
+delivery_event_action = ${acl {logger}}
+
 
 # ----- ACL -----
 
@@ -32,11 +34,29 @@ ev_smtp:
 			[$host_address]:$host_port
 	   logwrite = . banner <$tpda_data>
 
+ev_msg_fail:
+    accept condition = ${if eq {$tpda_event}{msg:fail:delivery}}
+	   logwrite = . \
+	    refused by fdqn <$host> \
+	    local_part <$local_part> \
+	    domain <$domain> \
+
+    accept logwrite = . \
+	    local_part <$local_part> \
+	    domain <$domain> \
+	    reason <$tpda_data>
+
 ev_msg:
-    accept condition = ${if !eq {$acl_arg2}{domain1}}
+    accept condition = ${if eq {fail} {${listextract{2}{$tpda_event}}}}
+	   acl = ev_msg_fail
+
+    accept condition = ${if eq {$tpda_event}{msg:complete}}
+	   logwrite = . finished: $message_exim_id
+
+    accept condition = ${if !eq {$domain}{domain1}}
 	   logwrite = $this_expansion_will_fail
 
-    accept condition = ${if eq {$acl_arg1}{msg:delivery}}
+    accept condition = ${if eq {$tpda_event}{msg:delivery}}
 	   logwrite = . \
 	    delivery \
 	    ip <$host_address> \
@@ -48,7 +68,7 @@ ev_msg:
 	    router <$router_name> \
 	    transport <$transport_name>
 
-    accept condition = ${if eq {$acl_arg1}{msg:host:defer}}
+    accept condition = ${if eq {$tpda_event}{msg:host:defer}}
 	   logwrite = . \
 	    deferral \
 	    ip <$host_address> \
@@ -68,7 +88,7 @@ logger:
     accept condition = ${if eq {smtp} {${listextract{1}{$tpda_event}}}}
 	   acl = ev_smtp
     accept condition = ${if eq {msg} {${listextract{1}{$tpda_event}}}}
-	   acl = ev_msg $tpda_event $acl_arg2
+	   acl = ev_msg
 
 
 # ----- Routers -----
@@ -97,6 +117,6 @@ smtp:
   port = PORT_S
   command_timeout = 1s
   final_timeout = 1s
-  tpda_event_action =   ${acl {logger} {$tpda_event} {$domain} }
+  tpda_event_action =   ${acl {logger}}
 
 # End
diff --git a/test/log/5700 b/test/log/5700
index 150aef773..2bebd8cff 100644
--- a/test/log/5700
+++ b/test/log/5700
@@ -11,6 +11,8 @@
 1999-03-02 09:44:33 10HmaY-0005vi-00 event msg:delivery
 1999-03-02 09:44:33 10HmaY-0005vi-00 . delivery ip <127.0.0.1> port <1224> fqdn <127.0.0.1> local_part <userx> domain <domain1> confirmation <250 OK> router <others> transport <smtp>
 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaY-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmaY-0005vi-00 . finished: 10HmaY-0005vi-00
 1999-03-02 09:44:33 End queue run: pid=pppp -qqf
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local S=sss
 1999-03-02 09:44:33 Start queue run: pid=pppp -qqf
@@ -26,6 +28,8 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 failed to expand tpda_event_action msg:delivery in smtp: error from acl "logger"
 
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaX-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmaX-0005vi-00 . finished: 10HmaX-0005vi-00
 1999-03-02 09:44:33 End queue run: pid=pppp -qqf
 1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local S=sss
 1999-03-02 09:44:33 10HmaZ-0005vi-00 event tcp:connect
@@ -43,4 +47,41 @@
 1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> R=10HmaZ-0005vi-00 U=EXIMUSER P=local S=sss
 1999-03-02 09:44:33 10HmbA-0005vi-00 => :blackhole: <CALLER@the.local.host.name> R=dump_bounces
 1999-03-02 09:44:33 10HmbA-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbA-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmbA-0005vi-00 . finished: 10HmbA-0005vi-00
 1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaZ-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmaZ-0005vi-00 . finished: 10HmaZ-0005vi-00
+1999-03-02 09:44:33 10HmbB-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local S=sss
+1999-03-02 09:44:33 Start queue run: pid=pppp -qqf
+1999-03-02 09:44:33 10HmbB-0005vi-00 event tcp:connect
+1999-03-02 09:44:33 10HmbB-0005vi-00 . [127.0.0.1]:-1
+1999-03-02 09:44:33 10HmbB-0005vi-00 event smtp:connect
+1999-03-02 09:44:33 10HmbB-0005vi-00 . [127.0.0.1] -> [127.0.0.1]:1224
+1999-03-02 09:44:33 10HmbB-0005vi-00 . banner <220 ESMTP>
+1999-03-02 09:44:33 10HmbB-0005vi-00 event tcp:close
+1999-03-02 09:44:33 10HmbB-0005vi-00 . [127.0.0.1] -> [127.0.0.1]:1224
+1999-03-02 09:44:33 10HmbB-0005vi-00 ** userx@domain1 R=others T=smtp: SMTP error from remote mail server after RCPT TO:<userx@domain1>: host 127.0.0.1 [127.0.0.1]: 550 GO AWAY
+1999-03-02 09:44:33 10HmbB-0005vi-00 event msg:fail:delivery
+1999-03-02 09:44:33 10HmbB-0005vi-00 . refused by fdqn <127.0.0.1> local_part <userx> domain <domain1>
+1999-03-02 09:44:33 10HmbC-0005vi-00 <= <> R=10HmbB-0005vi-00 U=EXIMUSER P=local S=sss
+1999-03-02 09:44:33 10HmbC-0005vi-00 => :blackhole: <CALLER@the.local.host.name> R=dump_bounces
+1999-03-02 09:44:33 10HmbC-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbC-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmbC-0005vi-00 . finished: 10HmbC-0005vi-00
+1999-03-02 09:44:33 10HmbB-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbB-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmbB-0005vi-00 . finished: 10HmbB-0005vi-00
+1999-03-02 09:44:33 End queue run: pid=pppp -qqf
+1999-03-02 09:44:33 10HmbD-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmbD-0005vi-00 cancelled by CALLER
+1999-03-02 09:44:33 10HmbD-0005vi-00 event msg:fail:internal
+1999-03-02 09:44:33 10HmbD-0005vi-00 . local_part <userx> domain <domain1> reason <delivery cancelled by administrator>
+1999-03-02 09:44:33 10HmbE-0005vi-00 <= <> R=10HmbD-0005vi-00 U=EXIMUSER P=local S=sss
+1999-03-02 09:44:33 10HmbE-0005vi-00 => :blackhole: <CALLER@the.local.host.name> R=dump_bounces
+1999-03-02 09:44:33 10HmbE-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbE-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmbE-0005vi-00 . finished: 10HmbE-0005vi-00
+1999-03-02 09:44:33 10HmbD-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbD-0005vi-00 event msg:complete
+1999-03-02 09:44:33 10HmbD-0005vi-00 . finished: 10HmbD-0005vi-00
diff --git a/test/scripts/5700-tpt-post-dlv-action/5700 b/test/scripts/5700-tpt-post-dlv-action/5700
index b85231b6e..f7cd8272c 100644
--- a/test/scripts/5700-tpt-post-dlv-action/5700
+++ b/test/scripts/5700-tpt-post-dlv-action/5700
@@ -1,4 +1,4 @@
-# Arbtirary expansion after transport
+# Arbitrary expansion after transport
 # (EXPERIMENTAL_TPDA)
 #
 need_ipv4
@@ -62,3 +62,33 @@ A message which will hit a timeout at the destination server
 .
 ****
 #
+#
+#
+#
+#
+exim -odq userx@domain1
+A message which will get refused
+****
+server PORT_S
+220 ESMTP
+EHLO
+250-OK
+250 HELP
+MAIL
+250 OK
+RCPT
+550 GO AWAY
+QUIT
+220 OK
+****
+exim -qqf
+****
+#
+#
+#
+#
+exim -odq userx@domain1
+A message we will cancel from the queue
+****
+exim -odi -Mg $msg1
+****
diff --git a/test/stdout/5700 b/test/stdout/5700
index a45b44c87..b185dca05 100644
--- a/test/stdout/5700
+++ b/test/stdout/5700
@@ -58,3 +58,16 @@ Connection request from [127.0.0.1]
 EHLO the.local.host.name
 *sleep 4
 End of script
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+220 ESMTP
+EHLO the.local.host.name
+250-OK
+250 HELP
+MAIL FROM:<CALLER@the.local.host.name>
+250 OK
+RCPT TO:<userx@domain1>
+550 GO AWAY
+QUIT
+220 OK
+End of script