Merge pull request #908 from paullouisageneau/fixes-openssl

Fix multiple issues with OpenSSL

src/impl/dtlstransport.cpp

@@ -393,8 +393,7 @@ DtlsTransport::DtlsTransport(shared_ptr<IceTransport> lower, certificate_ptr cer
 
 		SSL_CTX_set_min_proto_version(mCtx, DTLS1_VERSION);
 		SSL_CTX_set_read_ahead(mCtx, 1);
-		//sent the dtls close_notify alert
-		//SSL_CTX_set_quiet_shutdown(mCtx, 1);
+		SSL_CTX_set_quiet_shutdown(mCtx, 0); // send the close_notify alert
 		SSL_CTX_set_info_callback(mCtx, InfoCallback);
 
 		SSL_CTX_set_verify(mCtx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT,
@@ -480,7 +479,6 @@ void DtlsTransport::stop() {
 	unregisterIncoming();
 	mIncomingQueue.stop();
 	mRecvThread.join();
-	SSL_shutdown(mSsl);
 }
 
 bool DtlsTransport::send(message_ptr message) {
@@ -625,6 +623,10 @@ void DtlsTransport::runRecvLoop() {
 
 			mIncomingQueue.wait(duration);
 		}
+
+		std::lock_guard lock(mSslMutex);
+		SSL_shutdown(mSsl);
+
 	} catch (const std::exception &e) {
 		PLOG_ERROR << "DTLS recv: " << e.what();
 	}

src/impl/tls.cpp

@@ -85,24 +85,33 @@ void init() {
 	}
 }
 
-string error_string(unsigned long err) {
+string error_string(unsigned long error) {
 	const size_t bufferSize = 256;
 	char buffer[bufferSize];
-	ERR_error_string_n(err, buffer, bufferSize);
+	ERR_error_string_n(error, buffer, bufferSize);
 	return string(buffer);
 }
 
 bool check(int success, const string &message) {
-	if (success)
+	unsigned long last_error = ERR_peek_last_error();
+	ERR_clear_error();
+
+	if (success > 0)
 		return true;
 
-	string str = error_string(ERR_get_error());
-	PLOG_ERROR << message << ": " << str;
-	throw std::runtime_error(message + ": " + str);
+	string str = message;
+	if (last_error != 0)
+		str += ": " + error_string(last_error);
+
+	PLOG_ERROR << str;
+	throw std::runtime_error(str);
 }
 
 bool check(SSL *ssl, int ret, const string &message) {
-	unsigned long err = SSL_get_error(ssl, ret);
+	unsigned long last_error = ERR_peek_last_error();
+	ERR_clear_error();
+
+	int err = SSL_get_error(ssl, ret);
 	if (err == SSL_ERROR_NONE || err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
 		return true;
 	}
@@ -110,9 +119,16 @@ bool check(SSL *ssl, int ret, const string &message) {
 		PLOG_DEBUG << "OpenSSL connection cleanly closed";
 		return false;
 	}
-	string str = error_string(err);
+
+	string str = message;
+	if (err == SSL_ERROR_SYSCALL) {
+		str += ": fatal I/O error";
+	} else if (err == SSL_ERROR_SSL) {
+		if (last_error != 0)
+			str += ": " + error_string(last_error);
+	}
 	PLOG_ERROR << str;
-	throw std::runtime_error(message + ": " + str);
+	throw std::runtime_error(str);
 }
 
 BIO *BIO_new_from_file(const string &filename) {

src/impl/tls.hpp

@@ -64,7 +64,7 @@ gnutls_datum_t make_datum(char *data, size_t size);
 namespace rtc::openssl {
 
 void init();
-string error_string(unsigned long err);
+string error_string(unsigned long error);
 
 bool check(int success, const string &message = "OpenSSL error");
 bool check(SSL *ssl, int ret, const string &message = "OpenSSL error");

src/impl/tlstransport.cpp

@@ -322,10 +322,10 @@ TlsTransport::TlsTransport(shared_ptr<TcpTransport> lower, optional<string> host
 			}
 		}
 
-		SSL_CTX_set_options(mCtx, SSL_OP_NO_SSLv3);
+		SSL_CTX_set_options(mCtx, SSL_OP_NO_SSLv3 | SSL_OP_NO_RENEGOTIATION);
 		SSL_CTX_set_min_proto_version(mCtx, TLS1_VERSION);
 		SSL_CTX_set_read_ahead(mCtx, 1);
-		SSL_CTX_set_quiet_shutdown(mCtx, 1);
+		SSL_CTX_set_quiet_shutdown(mCtx, 0); // send the close_notify alert
 		SSL_CTX_set_info_callback(mCtx, InfoCallback);
 		SSL_CTX_set_verify(mCtx, SSL_VERIFY_NONE, NULL);
 
@@ -391,7 +391,6 @@ void TlsTransport::stop() {
 	unregisterIncoming();
 	mIncomingQueue.stop();
 	mRecvThread.join();
-	SSL_shutdown(mSsl);
 }
 
 bool TlsTransport::send(message_ptr message) {
@@ -483,6 +482,9 @@ void TlsTransport::runRecvLoop() {
 				recv(message); // Pass zero-sized messages through
 		}
 
+		std::lock_guard lock(mSslMutex);
+		SSL_shutdown(mSsl);
+
 	} catch (const std::exception &e) {
 		PLOG_ERROR << "TLS recv: " << e.what();
 	}