m_ssl_gnutls, m_ssl_openssl Simplify Handshake() result handling

2 files changed, 25 insertions, 36 deletions

diff --git a/src/modules/extra/m_ssl_gnutls.cpp b/src/modules/extra/m_ssl_gnutls.cpp
index a684e5916..0b22788fd 100644
--- a/src/modules/extra/m_ssl_gnutls.cpp
+++ b/src/modules/extra/m_ssl_gnutls.cpp
@@ -628,7 +628,8 @@ class GnuTLSIOHook : public SSLIOHook
 		status = ISSL_NONE;
 	}
 
-	bool Handshake(StreamSocket* user)
+	// Returns 1 if handshake succeeded, 0 if it is still in progress, -1 if it failed
+	int Handshake(StreamSocket* user)
 	{
 		int ret = gnutls_handshake(this->sess);
 
@@ -649,15 +650,16 @@ class GnuTLSIOHook : public SSLIOHook
 					// gnutls_handshake() wants to write() again.
 					SocketEngine::ChangeEventMask(user, FD_WANT_NO_READ | FD_WANT_SINGLE_WRITE);
 				}
+
+				return 0;
 			}
 			else
 			{
 				user->SetError("Handshake Failed - " + std::string(gnutls_strerror(ret)));
 				CloseSession();
 				this->status = ISSL_CLOSING;
+				return -1;
 			}
-
-			return false;
 		}
 		else
 		{
@@ -669,7 +671,7 @@ class GnuTLSIOHook : public SSLIOHook
 			// Finish writing, if any left
 			SocketEngine::ChangeEventMask(user, FD_WANT_POLL_READ | FD_WANT_NO_WRITE | FD_ADD_TRIAL_WRITE);
 
-			return true;
+			return 1;
 		}
 	}
 
@@ -883,13 +885,9 @@ info_done_dealloc:
 		if (this->status == ISSL_HANDSHAKING)
 		{
 			// The handshake isn't finished, try to finish it.
-
-			if (!Handshake(user))
-			{
-				if (this->status != ISSL_CLOSING)
-					return 0;
-				return -1;
-			}
+			int ret = Handshake(user);
+			if (ret <= 0)
+				return ret;
 		}
 
 		// If we resumed the handshake then this->status will be ISSL_HANDSHAKEN.
@@ -938,10 +936,9 @@ info_done_dealloc:
 		if (this->status == ISSL_HANDSHAKING)
 		{
 			// The handshake isn't finished, try to finish it.
-			Handshake(user);
-			if (this->status != ISSL_CLOSING)
-				return 0;
-			return -1;
+			int ret = Handshake(user);
+			if (ret <= 0)
+				return ret;
 		}
 
 		int ret = 0;
diff --git a/src/modules/extra/m_ssl_openssl.cpp b/src/modules/extra/m_ssl_openssl.cpp
index c1a3bf41a..21227fe6d 100644
--- a/src/modules/extra/m_ssl_openssl.cpp
+++ b/src/modules/extra/m_ssl_openssl.cpp
@@ -354,7 +354,8 @@ class OpenSSLIOHook : public SSLIOHook
 	bool data_to_write;
 	reference<OpenSSL::Profile> profile;
 
-	bool Handshake(StreamSocket* user)
+	// Returns 1 if handshake succeeded, 0 if it is still in progress, -1 if it failed
+	int Handshake(StreamSocket* user)
 	{
 		int ret;
 
@@ -372,20 +373,19 @@ class OpenSSLIOHook : public SSLIOHook
 			{
 				SocketEngine::ChangeEventMask(user, FD_WANT_POLL_READ | FD_WANT_NO_WRITE);
 				this->status = ISSL_HANDSHAKING;
-				return true;
+				return 0;
 			}
 			else if (err == SSL_ERROR_WANT_WRITE)
 			{
 				SocketEngine::ChangeEventMask(user, FD_WANT_NO_READ | FD_WANT_SINGLE_WRITE);
 				this->status = ISSL_HANDSHAKING;
-				return true;
+				return 0;
 			}
 			else
 			{
 				CloseSession();
+				return -1;
 			}
-
-			return false;
 		}
 		else if (ret > 0)
 		{
@@ -396,13 +396,13 @@ class OpenSSLIOHook : public SSLIOHook
 
 			SocketEngine::ChangeEventMask(user, FD_WANT_POLL_READ | FD_WANT_NO_WRITE | FD_ADD_TRIAL_WRITE);
 
-			return true;
+			return 1;
 		}
 		else if (ret == 0)
 		{
 			CloseSession();
 		}
-		return false;
+		return -1;
 	}
 
 	void CloseSession()
@@ -540,13 +540,9 @@ class OpenSSLIOHook : public SSLIOHook
 		if (status == ISSL_HANDSHAKING)
 		{
 			// The handshake isn't finished and it wants to read, try to finish it.
-			if (!Handshake(user))
-			{
-				// Couldn't resume handshake.
-				if (status == ISSL_NONE)
-					return -1;
-				return 0;
-			}
+			int ret = Handshake(user);
+			if (ret <= 0)
+				return ret;
 		}
 
 		// If we resumed the handshake then this->status will be ISSL_OPEN
@@ -614,13 +610,9 @@ class OpenSSLIOHook : public SSLIOHook
 
 		if (status == ISSL_HANDSHAKING)
 		{
-			if (!Handshake(user))
-			{
-				// Couldn't resume handshake.
-				if (status == ISSL_NONE)
-					return -1;
-				return 0;
-			}
+			int ret = Handshake(user);
+			if (ret <= 0)
+				return ret;
 		}
 
 		if (status == ISSL_OPEN)