Some performance tuning for SCTP transport

README.md

@@ -8,7 +8,7 @@ Licensed under LGPLv2, see [LICENSE](https://github.com/paullouisageneau/libdata
 
 ## Compatibility
 
-The library aims at fully implementing SCTP DataChannels ([draft-ietf-rtcweb-data-channel-13](https://tools.ietf.org/html/draft-ietf-rtcweb-data-channel-13)) over DTLS/UDP ([RFC7350](https://tools.ietf.org/html/rfc7350)) and has been tested to be compatible with Firefox and Chromium. It supports IPv6 and Multicast DNS candidates resolution ([draft-ietf-rtcweb-mdns-ice-candidates-03](https://tools.ietf.org/html/draft-ietf-rtcweb-mdns-ice-candidates-03)) provided the operating system also supports it.
+The library aims at fully implementing SCTP DataChannels ([draft-ietf-rtcweb-data-channel-13](https://tools.ietf.org/html/draft-ietf-rtcweb-data-channel-13)) over DTLS/UDP ([RFC7350](https://tools.ietf.org/html/rfc7350) and [RFC8261](https://tools.ietf.org/html/rfc8261)) and has been tested to be compatible with Firefox and Chromium. It supports IPv6 and Multicast DNS candidates resolution ([draft-ietf-rtcweb-mdns-ice-candidates-03](https://tools.ietf.org/html/draft-ietf-rtcweb-mdns-ice-candidates-03)) provided the operating system also supports it.
 
 ## Dependencies
 

include/rtc/include.hpp

@@ -44,8 +44,8 @@ using std::uint8_t;
 const size_t MAX_NUMERICNODE_LEN = 48; // Max IPv6 string representation length
 const size_t MAX_NUMERICSERV_LEN = 6;  // Max port string representation length
 
-const size_t RECV_QUEUE_SIZE = 256; // DataChannel receive queue size in messages
-                                    // (0 means unlimited)
+const size_t RECV_QUEUE_SIZE = 16; // DataChannel receive queue size in messages
+                                   // (0 means unlimited)
 
 const uint16_t DEFAULT_SCTP_PORT = 5000; // SCTP port to use by default
 

src/dtlstransport.cpp

@@ -71,7 +71,7 @@ DtlsTransport::DtlsTransport(shared_ptr<IceTransport> lower, shared_ptr<Certific
 	check_gnutls(
 	    gnutls_credentials_set(mSession, GNUTLS_CRD_CERTIFICATE, mCertificate->credentials()));
 
-	gnutls_dtls_set_mtu(mSession, 1280 - 40 - 8); // min MTU over UDP/IPv6
+	gnutls_dtls_set_mtu(mSession, 1280 - 40 - 8); // min MTU over UDP/IPv6 (only for handshake)
 	gnutls_dtls_set_timeouts(mSession, 400, 60000);
 	gnutls_handshake_set_timeout(mSession, 60000);
 
@@ -119,7 +119,7 @@ void DtlsTransport::changeState(State state) {
 }
 
 void DtlsTransport::runRecvLoop() {
-	const size_t maxMtu = 2048;
+	const size_t maxMtu = 4096;
 
 	// Handshake loop
 	try {

src/sctptransport.cpp

@@ -55,12 +55,14 @@ SctpTransport::SctpTransport(std::shared_ptr<Transport> lower, uint16_t port, me
 	onRecv(recv);
 
 	GlobalInit();
+
 	usrsctp_register_address(this);
 	mSock = usrsctp_socket(AF_CONN, SOCK_STREAM, IPPROTO_SCTP, &SctpTransport::ReadCallback,
 	                       nullptr, 0, this);
 	if (!mSock)
-		throw std::runtime_error("Could not create usrsctp socket, errno=" + std::to_string(errno));
+		throw std::runtime_error("Could not create SCTP socket, errno=" + std::to_string(errno));
 
+	// SCTP must stop sending after the lower layer is shut down, so disable linger
 	struct linger sol = {};
 	sol.l_onoff = 1;
 	sol.l_linger = 0;
@@ -68,14 +70,6 @@ SctpTransport::SctpTransport(std::shared_ptr<Transport> lower, uint16_t port, me
 		throw std::runtime_error("Could not set socket option SO_LINGER, errno=" +
 		                         std::to_string(errno));
 
-	struct sctp_paddrparams spp = {};
-	spp.spp_flags = SPP_PMTUD_ENABLE;
-	spp.spp_pathmtu = 1200; // Max safe value recommended by RFC 8261
-	                        // See https://tools.ietf.org/html/rfc8261#section-5
-	if (usrsctp_setsockopt(mSock, IPPROTO_SCTP, SCTP_PEER_ADDR_PARAMS, &spp, sizeof(spp)))
-		throw std::runtime_error("Could not set socket option SCTP_PEER_ADDR_PARAMS, errno=" +
-		                         std::to_string(errno));
-
 	struct sctp_assoc_value av = {};
 	av.assoc_id = SCTP_ALL_ASSOC;
 	av.assoc_value = 1;
@@ -83,11 +77,6 @@ SctpTransport::SctpTransport(std::shared_ptr<Transport> lower, uint16_t port, me
 		throw std::runtime_error("Could not set socket option SCTP_ENABLE_STREAM_RESET, errno=" +
 		                         std::to_string(errno));
 
-	uint32_t nodelay = 1;
-	if (usrsctp_setsockopt(mSock, IPPROTO_SCTP, SCTP_NODELAY, &nodelay, sizeof(nodelay)))
-		throw std::runtime_error("Could not set socket option SCTP_NODELAY, errno=" +
-		                         std::to_string(errno));
-
 	struct sctp_event se = {};
 	se.se_assoc_id = SCTP_ALL_ASSOC;
 	se.se_on = 1;
@@ -96,6 +85,27 @@ SctpTransport::SctpTransport(std::shared_ptr<Transport> lower, uint16_t port, me
 		throw std::runtime_error("Could not set socket option SCTP_EVENT, errno=" +
 		                         std::to_string(errno));
 
+	// Disable Nagle-like algorithm to reduce delay
+	int nodelay = 1;
+	if (usrsctp_setsockopt(mSock, IPPROTO_SCTP, SCTP_NODELAY, &nodelay, sizeof(nodelay)))
+		throw std::runtime_error("Could not set socket option SCTP_NODELAY, errno=" +
+		                         std::to_string(errno));
+
+	struct sctp_paddrparams spp = {};
+#ifdef __linux__
+	// Linux UDP does path MTU discovery by default (setting DF and returning EMSGSIZE).
+	// It should be safe to enable discovery for SCTP.
+	spp.spp_flags = SPP_PMTUD_ENABLE;
+#else
+	// Otherwise, fall back to a safe MTU value.
+	spp.spp_flags = SPP_PMTUD_DISABLE;
+	spp.spp_pathmtu = 1200; // Max safe value recommended by RFC 8261
+	                        // See https://tools.ietf.org/html/rfc8261#section-5
+#endif
+	if (usrsctp_setsockopt(mSock, IPPROTO_SCTP, SCTP_PEER_ADDR_PARAMS, &spp, sizeof(spp)))
+		throw std::runtime_error("Could not set socket option SCTP_PEER_ADDR_PARAMS, errno=" +
+		                         std::to_string(errno));
+
 	// The IETF draft recommends the number of streams negotiated during SCTP association to be
 	// 65535. See https://tools.ietf.org/html/draft-ietf-rtcweb-data-channel-13#section-6.2
 	struct sctp_initmsg sinit = {};
@@ -105,6 +115,17 @@ SctpTransport::SctpTransport(std::shared_ptr<Transport> lower, uint16_t port, me
 		throw std::runtime_error("Could not set socket option SCTP_INITMSG, errno=" +
 		                         std::to_string(errno));
 
+	// The default send and receive window size of usrsctp is 256KiB, which is too small for
+	// realistic RTTs, therefore we increase it to 1MiB for better performance.
+	// See https://bugzilla.mozilla.org/show_bug.cgi?id=1051685
+	int bufSize = 1024 * 1024;
+	if (usrsctp_setsockopt(mSock, SOL_SOCKET, SO_RCVBUF, &bufSize, sizeof(bufSize)))
+		throw std::runtime_error("Could not set SCTP recv buffer size, errno=" +
+		                         std::to_string(errno));
+	if (usrsctp_setsockopt(mSock, SOL_SOCKET, SO_SNDBUF, &bufSize, sizeof(bufSize)))
+		throw std::runtime_error("Could not set SCTP send buffer size, errno=" +
+		                         std::to_string(errno));
+
 	struct sockaddr_conn sconn = {};
 	sconn.sconn_family = AF_CONN;
 	sconn.sconn_port = htons(mPort);
@@ -117,7 +138,7 @@ SctpTransport::SctpTransport(std::shared_ptr<Transport> lower, uint16_t port, me
 		throw std::runtime_error("Could not bind usrsctp socket, errno=" + std::to_string(errno));
 
 	mSendThread = std::thread(&SctpTransport::runConnectAndSendLoop, this);
-}
+		}
 
 SctpTransport::~SctpTransport() {
 	onRecv(nullptr); // unset recv callback