Don't negotiate GCM if built with system libSRTP

src/impl/dtlssrtptransport.cpp

@@ -45,6 +45,17 @@ void DtlsSrtpTransport::Init() { srtp_init(); }
 
 void DtlsSrtpTransport::Cleanup() { srtp_shutdown(); }
 
+bool DtlsSrtpTransport::IsGcmSupported() {
+#if RTC_SYSTEM_SRTP
+	// system libSRTP may not have GCM support
+	srtp_policy_t policy = {};
+	return srtp_crypto_policy_set_from_profile_for_rtp(
+	           &policy.rtp, srtp_profile_aead_aes_256_gcm) == srtp_err_status_ok;
+#else
+	return true;
+#endif
+}
+
 DtlsSrtpTransport::DtlsSrtpTransport(shared_ptr<IceTransport> lower,
                                      shared_ptr<Certificate> certificate, optional<size_t> mtu,
                                      verifier_callback verifierCallback,
@@ -326,9 +337,9 @@ void DtlsSrtpTransport::postHandshake() {
 	std::memcpy(mServerSessionKey.data() + keySize, serverSalt, saltSize);
 
 	srtp_policy_t inbound = {};
-	if(srtp_crypto_policy_set_from_profile_for_rtp(&inbound.rtp, srtpProfile))
+	if (srtp_crypto_policy_set_from_profile_for_rtp(&inbound.rtp, srtpProfile))
 		throw std::runtime_error("SRTP profile is not supported");
-	if(srtp_crypto_policy_set_from_profile_for_rtcp(&inbound.rtcp, srtpProfile))
+	if (srtp_crypto_policy_set_from_profile_for_rtcp(&inbound.rtcp, srtpProfile))
 		throw std::runtime_error("SRTP profile is not supported");
 
 	inbound.ssrc.type = ssrc_any_inbound;
@@ -342,9 +353,9 @@ void DtlsSrtpTransport::postHandshake() {
 		                         to_string(static_cast<int>(err)));
 
 	srtp_policy_t outbound = {};
-	if(srtp_crypto_policy_set_from_profile_for_rtp(&outbound.rtp, srtpProfile))
+	if (srtp_crypto_policy_set_from_profile_for_rtp(&outbound.rtp, srtpProfile))
 		throw std::runtime_error("SRTP profile is not supported");
-	if(srtp_crypto_policy_set_from_profile_for_rtcp(&outbound.rtcp, srtpProfile))
+	if (srtp_crypto_policy_set_from_profile_for_rtcp(&outbound.rtcp, srtpProfile))
 		throw std::runtime_error("SRTP profile is not supported");
 
 	outbound.ssrc.type = ssrc_any_outbound;
@@ -361,7 +372,7 @@ void DtlsSrtpTransport::postHandshake() {
 }
 
 #if !USE_GNUTLS && !USE_MBEDTLS
-ProfileParams DtlsSrtpTransport::getProfileParamsFromName(string_view name) {
+DtlsSrtpTransport::ProfileParams DtlsSrtpTransport::getProfileParamsFromName(string_view name) {
 	if (name == "SRTP_AES128_CM_SHA1_80")
 		return {srtp_profile_aes128_cm_sha1_80, SRTP_AES_128_KEY_LEN, SRTP_SALT_LEN};
 	if (name == "SRTP_AES128_CM_SHA1_32")

src/impl/dtlssrtptransport.hpp

@@ -24,16 +24,11 @@
 
 namespace rtc::impl {
 
-struct ProfileParams {
-	srtp_profile_t srtpProfile;
-	size_t keySize;
-	size_t saltSize;
-};
-
 class DtlsSrtpTransport final : public DtlsTransport {
 public:
 	static void Init();
 	static void Cleanup();
+	static bool IsGcmSupported();
 
 	DtlsSrtpTransport(shared_ptr<IceTransport> lower, certificate_ptr certificate,
 	                  optional<size_t> mtu, verifier_callback verifierCallback,
@@ -48,6 +43,12 @@ private:
 	void postHandshake() override;
 
 #if !USE_GNUTLS && !USE_MBEDTLS
+	struct ProfileParams {
+		srtp_profile_t srtpProfile;
+		size_t keySize;
+		size_t saltSize;
+	};
+
 	ProfileParams getProfileParamsFromName(string_view name);
 #endif
 

src/impl/dtlstransport.cpp

@@ -7,6 +7,7 @@
  */
 
 #include "dtlstransport.hpp"
+#include "dtlssrtptransport.hpp"
 #include "icetransport.hpp"
 #include "internals.hpp"
 #include "threadpool.hpp"
@@ -794,16 +795,23 @@ DtlsTransport::DtlsTransport(shared_ptr<IceTransport> lower, certificate_ptr cer
 		SSL_set_bio(mSsl, mInBio, mOutBio);
 
 		// RFC 8827: The DTLS-SRTP protection profile SRTP_AES128_CM_HMAC_SHA1_80 MUST be supported
-		// See https://www.rfc-editor.org/rfc/rfc8827.html#section-6.5 Warning:
-		// SSL_set_tlsext_use_srtp() returns 0 on success and 1 on error
+		// See https://www.rfc-editor.org/rfc/rfc8827.html#section-6.5
+		// Warning: SSL_set_tlsext_use_srtp() returns 0 on success and 1 on error
+#if RTC_ENABLE_MEDIA
 		// Try to use GCM suite
-		if (SSL_set_tlsext_use_srtp(
+		if (!DtlsSrtpTransport::IsGcmSupported() ||
+		    SSL_set_tlsext_use_srtp(
 		        mSsl, "SRTP_AEAD_AES_256_GCM:SRTP_AEAD_AES_128_GCM:SRTP_AES128_CM_SHA1_80")) {
+			PLOG_WARNING << "AES-GCM for SRTP is not supported, falling back to default profile";
 			if (SSL_set_tlsext_use_srtp(mSsl, "SRTP_AES128_CM_SHA1_80"))
 				throw std::runtime_error("Failed to set SRTP profile: " +
 				                         openssl::error_string(ERR_get_error()));
 		}
-
+#else
+		if (SSL_set_tlsext_use_srtp(mSsl, "SRTP_AES128_CM_SHA1_80"))
+			throw std::runtime_error("Failed to set SRTP profile: " +
+			                         openssl::error_string(ERR_get_error()));
+#endif
 	} catch (...) {
 		if (mSsl)
 			SSL_free(mSsl);