GnuTLS: fix use of SHA3 hashes

4 files changed, 40 insertions, 19 deletions

diff --git a/src/src/expand.c b/src/src/expand.c
index a1e351cbd..dfd62e50c 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -6440,7 +6440,11 @@ while (*s != 0)
 	  blob b;
 	  char st[3];
 
-	  exim_sha_init(&h, HASH_SHA256);
+	  if (!exim_sha_init(&h, HASH_SHA256))
+	    {
+	    expand_string_message = US"unrecognised sha256 variant";
+	    goto EXPAND_FAILED;
+	    }
 	  exim_sha_update(&h, sub, Ustrlen(sub));
 	  exim_sha_finish(&h, &b);
 	  while (b.len-- > 0)
@@ -6467,13 +6471,12 @@ while (*s != 0)
 	  : Ustrcmp(arg, "512") == 0 ? HASH_SHA3_512
 	  : HASH_BADTYPE;
 
-	if (m == HASH_BADTYPE)
+	if (m == HASH_BADTYPE || !exim_sha_init(&h, m))
 	  {
 	  expand_string_message = US"unrecognised sha3 variant";
 	  goto EXPAND_FAILED;
 	  }
 
-	exim_sha_init(&h, m);
 	exim_sha_update(&h, sub, Ustrlen(sub));
 	exim_sha_finish(&h, &b);
 	while (b.len-- > 0)
diff --git a/src/src/hash.c b/src/src/hash.c
index c2be85d17..059e6d9bb 100644
--- a/src/src/hash.c
+++ b/src/src/hash.c
@@ -30,15 +30,16 @@ sha1;
 /******************************************************************************/
 #ifdef SHA_OPENSSL
 
-void
+BOOL
 exim_sha_init(hctx * h, hashmethod m)
 {
 switch (h->method = m)
   {
   case HASH_SHA1:   h->hashlen = 20; SHA1_Init  (&h->u.sha1); break;
   case HASH_SHA256: h->hashlen = 32; SHA256_Init(&h->u.sha2); break;
-  default:	    h->hashlen = 0; break;
+  default:	    h->hashlen = 0; return FALSE;
   }
+return TRUE;
 }
 
 
@@ -69,7 +70,7 @@ switch (h->method)
 #elif defined(SHA_GNUTLS)
 /******************************************************************************/
 
-void
+BOOL
 exim_sha_init(hctx * h, hashmethod m)
 {
 switch (h->method = m)
@@ -79,8 +80,9 @@ switch (h->method = m)
 #ifdef EXIM_HAVE_SHA3
   case HASH_SHA3_256: h->hashlen = 32; gnutls_hash_init(&h->sha, GNUTLS_DIG_SHA3_256); break;
 #endif
-  default: h->hashlen = 0; break;
+  default: h->hashlen = 0; return FALSE;
   }
+return TRUE;
 }
 
 
@@ -103,15 +105,16 @@ gnutls_hash_output(h->sha, b->data);
 #elif defined(SHA_GCRYPT)
 /******************************************************************************/
 
-void
+BOOL
 exim_sha_init(hctx * h, hashmethod m)
 {
 switch (h->method = m)
   {
   case HASH_SHA1:   h->hashlen = 20; gcry_md_open(&h->sha, GCRY_MD_SHA1, 0); break;
   case HASH_SHA256: h->hashlen = 32; gcry_md_open(&h->sha, GCRY_MD_SHA256, 0); break;
-  default:	    h->hashlen = 0; break;
+  default:	    h->hashlen = 0; return FALSE;
   }
+return TRUE;
 }
 
 
@@ -135,15 +138,16 @@ memcpy(b->data, gcry_md_read(h->sha, 0), h->hashlen);
 #elif defined(SHA_POLARSSL)
 /******************************************************************************/
 
-void
+BOOL
 exim_sha_init(hctx * h, hashmethod m)
 {
 switch (h->method = m)
   {
   case HASH_SHA1:   h->hashlen = 20; sha1_starts(&h->u.sha1);    break;
   case HASH_SHA256: h->hashlen = 32; sha2_starts(&h->u.sha2, 0); break;
-  default:	    h->hashlen = 0; break;
+  default:	    h->hashlen = 0; return FALSE;
   }
+return TRUE;
 }
 
 
@@ -382,11 +386,12 @@ for (i = 0; i < 5; i++)
 
 
 # ifdef notdef
-void
+BOOL
 exim_sha_init(hctx * h, hashmethod m)
 {
 h->hashlen = 20;
 native_sha1_start(&h->sha1);
+return TRUE;
 }
 
 
@@ -452,7 +457,7 @@ native_sha1_end(&h->sha1, data, len, digest);
 void
 sha1_start(hctx * h)
 {
-exim_sha_init(h, HASH_SHA1);
+(void) exim_sha_init(h, HASH_SHA1);
 }
 
 void
diff --git a/src/src/hash.h b/src/src/hash.h
index 9e91f1aad..09b65944d 100644
--- a/src/src/hash.h
+++ b/src/src/hash.h
@@ -67,7 +67,7 @@ typedef struct {
 
 } hctx;
 
-extern void     exim_sha_init(hctx *, hashmethod);
+extern BOOL     exim_sha_init(hctx *, hashmethod);
 extern void     exim_sha_update(hctx *, const uschar *a, int);
 extern void     exim_sha_finish(hctx *, blob *);
 extern int      exim_sha_hashlen(hctx *);
diff --git a/src/src/pdkim/pdkim.c b/src/src/pdkim/pdkim.c
index 0ae075f71..178f8f6a5 100644
--- a/src/src/pdkim/pdkim.c
+++ b/src/src/pdkim/pdkim.c
@@ -582,8 +582,12 @@ DEBUG(D_acl)
 	  "PDKIM <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
   }
 
-exim_sha_init(&sig->body_hash_ctx,
-	       sig->algo == PDKIM_ALGO_RSA_SHA1 ? HASH_SHA1 : HASH_SHA256);
+if (!exim_sha_init(&sig->body_hash_ctx,
+	       sig->algo == PDKIM_ALGO_RSA_SHA1 ? HASH_SHA1 : HASH_SHA256))
+  {
+  DEBUG(D_acl) debug_printf("PDKIM: hash init internal error\n");
+  return NULL;
+  }
 return sig;
 }
 
@@ -1411,7 +1415,11 @@ while (sig)
   hdata.data = NULL;
   hdata.len = 0;
 
-  exim_sha_init(&hhash_ctx, is_sha1 ? HASH_SHA1 : HASH_SHA256);
+  if (!exim_sha_init(&hhash_ctx, is_sha1 ? HASH_SHA1 : HASH_SHA256))
+    {
+    DEBUG(D_acl) debug_printf("PDKIM: hask setup internal error\n");
+    break;
+    }
 
   DEBUG(D_acl) debug_printf(
       "PDKIM >> Header data for hash, canonicalized, in sequence >>>>>>>>>>>>>>\n");
@@ -1717,8 +1725,13 @@ sig->selector = string_copy(US selector);
 sig->rsa_privkey = string_copy(US rsa_privkey);
 sig->algo = algo;
 
-exim_sha_init(&sig->body_hash_ctx,
-	       algo == PDKIM_ALGO_RSA_SHA1 ? HASH_SHA1 : HASH_SHA256);
+if (!exim_sha_init(&sig->body_hash_ctx,
+	       algo == PDKIM_ALGO_RSA_SHA1 ? HASH_SHA1 : HASH_SHA256))
+  {
+  DEBUG(D_acl) debug_printf("PDKIM: hash setup internal error\n");
+  return NULL;
+  }
+
 DEBUG(D_acl)
   {
   pdkim_signature s = *sig;