Refactored OpenSSL loop

src/dtlstransport.cpp

@@ -182,6 +182,7 @@ void DtlsTransport::runRecvLoop() {
 
 	// Receive loop
 	try {
+		PLOG_DEBUG << "DTLS handshake done";
 		changeState(State::Connected);
 
 		const size_t bufferSize = maxMtu;
@@ -463,54 +464,62 @@ void DtlsTransport::runRecvLoop() {
 	try {
 		changeState(State::Connecting);
 
+		// Initiate the handshake
 		int ret = SSL_do_handshake(mSsl);
 		check_openssl_ret(mSsl, ret, "Handshake failed");
 
 		const size_t bufferSize = maxMtu;
 		byte buffer[bufferSize];
 		while (true) {
-			std::optional<milliseconds> duration;
-			struct timeval timeout = {};
-			if (DTLSv1_get_timeout(mSsl, &timeout))
-				duration = milliseconds(timeout.tv_sec * 1000 + timeout.tv_usec / 1000);
-
-			if (!mIncomingQueue.wait(duration))
-				break; // queue is stopped
-
-			message_ptr decrypted;
-			if (!mIncomingQueue.empty()) {
+			// Process pending messages
+			while (!mIncomingQueue.empty()) {
 				auto message = *mIncomingQueue.pop();
 				BIO_write(mInBio, message->data(), message->size());
 
-				int ret = SSL_read(mSsl, buffer, bufferSize);
-				if (!check_openssl_ret(mSsl, ret))
-					break;
-
-				if (ret > 0)
-					decrypted = make_message(buffer, buffer + ret);
-			}
-
-			if (mState == State::Connecting) {
-				if (SSL_is_init_finished(mSsl)) {
-					changeState(State::Connected);
-
-					// RFC 8261: DTLS MUST support sending messages larger than the current path MTU
-					// See https://tools.ietf.org/html/rfc8261#section-5
-					SSL_set_mtu(mSsl, maxMtu + 1);
-				} else {
+				if (mState == State::Connecting) {
 					// Continue the handshake
 					int ret = SSL_do_handshake(mSsl);
 					if (!check_openssl_ret(mSsl, ret, "Handshake failed"))
 						break;
 
-					// Warning: This function breaks the usual return value convention
-					if (DTLSv1_handle_timeout(mSsl) < 0)
-						throw std::runtime_error("Handshake timeout"); // write BIO can't fail
+					if (SSL_is_init_finished(mSsl)) {
+						PLOG_DEBUG << "DTLS handshake done";
+						changeState(State::Connected);
+
+						// RFC 8261: DTLS MUST support sending messages larger than the current path
+						// MTU See https://tools.ietf.org/html/rfc8261#section-5
+						SSL_set_mtu(mSsl, maxMtu + 1);
+					}
+				} else {
+					int ret = SSL_read(mSsl, buffer, bufferSize);
+					if (!check_openssl_ret(mSsl, ret))
+						break;
+					if (ret > 0)
+						recv(make_message(buffer, buffer + ret));
+				}
+			}
+
+			// No more messages pending, retransmit and rearm timeout if connecting
+			std::optional<milliseconds> duration;
+			if (mState == State::Connecting) {
+				// Warning: This function breaks the usual return value convention
+				int ret = DTLSv1_handle_timeout(mSsl);
+				if (ret < 0) {
+					throw std::runtime_error("Handshake timeout"); // write BIO can't fail
+				} else if (ret > 0) {
+					LOG_VERBOSE << "OpenSSL did DTLS retransmit";
+				}
+
+				struct timeval timeout = {};
+				if (mState == State::Connecting && DTLSv1_get_timeout(mSsl, &timeout)) {
+					duration = milliseconds(timeout.tv_sec * 1000 + timeout.tv_usec / 1000);
+					LOG_VERBOSE << "OpenSSL DTLS retransmit timeout is " << duration->count()
+					            << "ms";
 				}
 			}
 
-			if (decrypted)
-				recv(decrypted);
+			if (!mIncomingQueue.wait(duration))
+				break; // queue is stopped
 		}
 	} catch (const std::exception &e) {
 		PLOG_ERROR << "DTLS recv: " << e.what();