Release packaging & scripting improvements.exim-4_89_RC3

* Make the .xz tarball variant too, and work harder on compressing our
  files for distribution.
  + The .xz files have gained more positive feedback than any other part
    of the 4.89 release.
* Drop usercodes from tarball
  + We shouldn't be embedding own-system-specifc ownership information
    into software release tarballs.  That's for local system backups,
    not distribution.
* Script for the size/checksums
  + We include checksums in the mail; this gets the format fixed and not
    including checksums-of-signatures, etc.  I've also experimented with
    including the size, so let's script that to be portably generated.
* Better tarball signing script
  + Automatically find the signing directory (if not already in it)
  + Sign all files, properly skipping existing .asc files
  + Find the signing key from git config, if available, else error out
    (Nigel is not on the hook as the default victim now)
  + Show what we're doing as we do it

All changes made on the original `release_4_89` branch with
`RELEASE EXPERIMENT` subject tags.

3 files changed, 97 insertions, 21 deletions

diff --git a/release-process/scripts/mk_exim_release b/release-process/scripts/mk_exim_release
index f6cd33c7e..34fe77c67 100755
--- a/release-process/scripts/mk_exim_release
+++ b/release-process/scripts/mk_exim_release
@@ -361,21 +361,25 @@ sub create_tar_files {
         }
     }
 
+    # See also environment variables set in main, tuning compression levels
+    my @COMPRESSIONS = (
+        # compressors-dict-key, file-extension, flags-as-string
+        [ "gzip", "gz", "--gzip" ],
+        [ "bzip2", "bz2", "--bzip2" ],
+        [ "lzip", "lz", "--lzip" ],
+        [ "xz", "xz", "--xz" ],
+    );
+
     foreach my $dir ( glob( File::Spec->catdir( $pkgdirs, ( 'exim*-' . $context->{release} ) ) ) ) {
         my $dirname = ( File::Spec->splitdir($dir) )[-1];
-        if ($context->{compressors}{gzip}) {
-            print "Creating: ${pkgs}/${dirname}.tar.gz\n" if ($verbose || $debug);
-            system("$tar cf  ${pkgs}/${dirname}.tar.gz  --gzip  -C ${pkgdirs} ${dirname}")
-        }
-        if ($context->{compressors}{bzip2}) {
-            print "Creating: ${pkgs}/${dirname}.tar.bz2\n" if ($verbose || $debug);
-            system("$tar cf  ${pkgs}/${dirname}.tar.bz2 --bzip2 -C ${pkgdirs} ${dirname}")
-        }
-        if ($context->{compressors}{lzip}) {
-            print "Creating: ${pkgs}/${dirname}.tar.lz\n" if ($verbose || $debug);
-            system("$tar cf  ${pkgs}/${dirname}.tar.lz  --lzip  -C ${pkgdirs} ${dirname}")
+        foreach my $comp (@COMPRESSIONS) {
+            my ($compkey, $extension, $flags) = @{$comp};
+            next unless $context->{compressors}{$compkey};
+            print "Creating: ${pkgs}/${dirname}.tar.${extension}\n" if ($verbose || $debug);
+            system("$tar cf  ${pkgs}/${dirname}.tar.${extension} ${flags} --numeric-owner -C ${pkgdirs} ${dirname}");
         }
     }
+
 }
 
 # ------------------------------------------------------------------
@@ -394,6 +398,7 @@ MAIN: {
         compressors => {
                 gzip    => 1,
                 bzip2   => 1,
+                xz      => 1,
                 lzip    => 0,
         },
         build_docs   => 1,
@@ -402,6 +407,13 @@ MAIN: {
     my $delete;
     my $cleanup = 1;
     ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};
+    # We are creating files for mass distribution, so work harder to make smaller files.
+    $ENV{'GZIP'} = '-9';
+    $ENV{'BZIP2'} = '-9';
+    # xz documents minimum file sizes for levels higher than -6 to be useful and each
+    # requires more RAM on the decompressing system.  Exim tarball currently 24MiB so
+    # using -8.
+    $ENV{'XZ_DEFAULTS'} = '-8';
 
     GetOptions(
         'directory=s'   => \$context->{directory},
diff --git a/release-process/scripts/sign_exim_packages b/release-process/scripts/sign_exim_packages
index 4ed614f19..bd02d1183 100755
--- a/release-process/scripts/sign_exim_packages
+++ b/release-process/scripts/sign_exim_packages
@@ -1,17 +1,53 @@
-#!/bin/sh
+#!/bin/sh -eu
+
+# gpg signs all *.tar.* files under the release directory.
+# Invoke from that dir, or let the script try to figure it out for you.
+
+# Key used is from env var EXIM_KEY; if git config finds user.signingkey, then
+# that is the default.  You can set this per-repo with:
+#    git config --local user.signingkey SOME_IDENTIFIER
 #
-# gpg signs all *.tar.* files under a given directory
-# key used set from env var EXIM_KEY, script defaults that to Nigel's.
+# If not set in git config then you _MUST_ set the env var.
+
 # woe betide the poor sod who does not use a gpg agent, so has
 # to enter their password for every file...
-#
 
-dir=${1:?start directory}
-
-: ${EXIM_KEY:=nigel@exim.org}
+if repo_signing_key="$(git config user.signingkey)"; then
+	: "${EXIM_KEY:=$repo_signing_key}"
+else
+	: "${EXIM_KEY:?Need a PGP key uid to sign with}"
+fi
 
+: "${GPG_COMMAND:=gpg}"
 umask 022
 
-find "$dir" \
-    -type f -name '*.tar.*' \
-    -exec gpg  --local-user ${EXIM_KEY} --detach-sig --armor {} \;
+cd_to() { echo "Working in: $1"; cd "$1"; }
+
+okay=false
+if [ -d ../../release-process ] && [ "${PWD##*/}" = "pkgs" ]; then
+	okay=true # we are in right dir
+elif [ -d release-process ]; then
+	b="$(find . -maxdepth 1 -name 'exim-packaging-*' | sort | tail -n 1)"
+	if [ ".$b" != "." ]; then
+		cd_to "$b/pkgs"
+		okay=true
+	fi
+fi
+if ! $okay; then
+	if [ -d "${1:?need a directory to look in}" ]; then
+		cd_to "$1"
+		shift
+	else
+		printf "%s: %s\n" >&2 "$(basename "$0")" "where should I be looking"
+		exit 1
+	fi
+fi
+
+# Assumes no whitespace (strictly, $IFS) in filenames, which we're okay with
+set $(find . -name '*.asc' -prune -o -type f -print | cut -c 3- | sort)
+
+for FILE
+do
+	echo "Signing: $FILE"
+	${GPG_COMMAND} --local-user "${EXIM_KEY}" --detach-sig --armor "$FILE"
+done
diff --git a/release-process/scripts/stats_for_email b/release-process/scripts/stats_for_email
new file mode 100755
index 000000000..0eb0c2981
--- /dev/null
+++ b/release-process/scripts/stats_for_email
@@ -0,0 +1,28 @@
+#!/bin/sh -eu
+
+okay=false
+if [ -d ../../release-process ] && [ "${PWD##*/}" = "pkgs" ]; then
+	okay=true # we are in right dir
+elif [ -d release-process ]; then
+	b="$(find . -maxdepth 1 -name 'exim-packaging-*' | sort | tail -n 1)"
+	if [ ".$b" != "." ]; then
+		cd "$b/pkgs"
+		okay=true
+	fi
+fi
+if ! $okay; then
+	if [ -d "${1:?need a directory to look in}" ]; then
+		cd "$1"
+		shift
+	else
+		printf "%s: %s\n" >&2 "$(basename "$0")" "where should I be looking"
+		exit 1
+	fi
+fi
+
+set $(find "${1:-.}" -name '*.asc' -prune -o -type f -print | cut -c 3- | sort)
+
+# stat(1) formats are non-portable BSD vs GNU
+perl -le 'print "SIZE($_)= @{[-s $_]}" foreach @ARGV' "$@"
+echo
+openssl dgst -sha256 "$@"