Merge pull request #936 from Sean-Der/master

Add AV1 Support

CMakeLists.txt

@@ -79,6 +79,8 @@ set(LIBDATACHANNEL_SOURCES
 	${CMAKE_CURRENT_SOURCE_DIR}/src/h264rtppacketizer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/nalunit.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/h264packetizationhandler.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/src/av1rtppacketizer.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/src/av1packetizationhandler.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/mediachainablehandler.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/mediahandlerelement.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/mediahandlerrootelement.cpp
@@ -114,6 +116,8 @@ set(LIBDATACHANNEL_HEADERS
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264rtppacketizer.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/nalunit.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264packetizationhandler.hpp
+	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1rtppacketizer.hpp
+	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1packetizationhandler.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/mediachainablehandler.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/mediahandlerelement.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/mediahandlerrootelement.hpp

include/rtc/av1packetizationhandler.hpp

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2023 Paul-Louis Ageneau
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef RTC_AV1_PACKETIZATION_HANDLER_H
+#define RTC_AV1_PACKETIZATION_HANDLER_H
+
+#if RTC_ENABLE_MEDIA
+
+#include "av1rtppacketizer.hpp"
+#include "mediachainablehandler.hpp"
+#include "nalunit.hpp"
+
+namespace rtc {
+
+/// Handler for AV1 packetization
+class RTC_CPP_EXPORT AV1PacketizationHandler final : public MediaChainableHandler {
+public:
+	/// Construct handler for AV1 packetization.
+	/// @param packetizer RTP packetizer for AV1
+	AV1PacketizationHandler(shared_ptr<AV1RtpPacketizer> packetizer);
+};
+
+} // namespace rtc
+
+#endif /* RTC_ENABLE_MEDIA */
+
+#endif /* RTC_AV1_PACKETIZATION_HANDLER_H */

include/rtc/av1rtppacketizer.hpp

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2023 Paul-Louis Ageneau
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef RTC_AV1_RTP_PACKETIZER_H
+#define RTC_AV1_RTP_PACKETIZER_H
+
+#if RTC_ENABLE_MEDIA
+
+#include "mediahandlerrootelement.hpp"
+#include "nalunit.hpp"
+#include "rtppacketizer.hpp"
+
+namespace rtc {
+
+/// RTP packetization of AV1 payload
+class RTC_CPP_EXPORT AV1RtpPacketizer final : public RtpPacketizer, public MediaHandlerRootElement {
+	shared_ptr<NalUnits> splitMessage(binary_ptr message);
+	const uint16_t maximumFragmentSize;
+
+public:
+	/// Default clock rate for AV1 in RTP
+	inline static const uint32_t defaultClockRate = 90 * 1000;
+
+	// Define how OBUs are seperated in a AV1 Sample
+	enum class Packetization {
+		Obu = RTC_OBU_PACKETIZED_OBU,
+		TemporalUnit = RTC_OBU_PACKETIZED_TEMPORAL_UNIT,
+	};
+
+	/// Constructs AV1 payload packetizer with given RTP configuration.
+	/// @note RTP configuration is used in packetization process which may change some configuration
+	/// properties such as sequence number.
+	/// @param rtpConfig  RTP configuration
+	AV1RtpPacketizer(Packetization packetization, shared_ptr<RtpPacketizationConfig> rtpConfig,
+	                 uint16_t maximumFragmentSize = NalUnits::defaultMaximumFragmentSize);
+
+	ChainedOutgoingProduct processOutgoingBinaryMessage(ChainedMessagesProduct messages,
+	                                                    message_ptr control) override;
+
+private:
+	const Packetization packetization;
+	std::shared_ptr<binary> sequenceHeader;
+
+	std::vector<shared_ptr<binary>> packetizeObu(binary_ptr message, uint16_t maximumFragmentSize);
+};
+
+} // namespace rtc
+
+#endif /* RTC_ENABLE_MEDIA */
+
+#endif /* RTC_AV1_RTP_PACKETIZER_H */

include/rtc/description.hpp

@@ -248,6 +248,7 @@ public:
 		void addH264Codec(int payloadType, optional<string> profile = DEFAULT_H264_VIDEO_PROFILE);
 		void addVP8Codec(int payloadType);
 		void addVP9Codec(int payloadType);
+		void addAV1Codec(int payloadType);
 	};
 
 	bool hasApplication() const;

include/rtc/rtc.h

@@ -276,6 +276,12 @@ RTC_C_EXPORT int rtcGetTrackDirection(int tr, rtcDirection *direction);
 
 // Media
 
+// Define how OBUs are packetizied in a AV1 Sample
+typedef enum {
+	RTC_OBU_PACKETIZED_OBU = 0,
+	RTC_OBU_PACKETIZED_TEMPORAL_UNIT = 1,
+} rtcObuPacketization;
+
 // Define how NAL units are separated in a H264 sample
 typedef enum {
 	RTC_NAL_SEPARATOR_LENGTH = 0,               // first 4 bytes are NAL unit length

include/rtc/rtc.hpp

@@ -33,8 +33,9 @@
 #include "rtcpreceivingsession.hpp"
 #include "rtcpsrreporter.hpp"
 
-// Opus/h264 streaming
+// Opus/h264/AV1 streaming
 #include "h264packetizationhandler.hpp"
+#include "av1packetizationhandler.hpp"
 #include "opuspacketizationhandler.hpp"
 
 #endif // RTC_ENABLE_MEDIA

src/av1packetizationhandler.cpp

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) 2023 Paul-Louis Ageneau
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#if RTC_ENABLE_MEDIA
+
+#include "av1packetizationhandler.hpp"
+
+namespace rtc {
+
+AV1PacketizationHandler::AV1PacketizationHandler(shared_ptr<AV1RtpPacketizer> packetizer)
+    : MediaChainableHandler(packetizer) {}
+
+} // namespace rtc
+
+#endif /* RTC_ENABLE_MEDIA */

src/av1rtppacketizer.cpp

@@ -0,0 +1,230 @@
+/**
+ * Copyright (c) 2023 Paul-Louis Ageneau
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#if RTC_ENABLE_MEDIA
+
+#include "av1rtppacketizer.hpp"
+
+#include "impl/internals.hpp"
+
+namespace rtc {
+
+const auto payloadHeaderSize = 1;
+
+const auto zMask = byte(0b10000000);
+const auto yMask = byte(0b01000000);
+const auto nMask = byte(0b00001000);
+
+const auto wBitshift = 4;
+
+const auto obuFrameTypeMask = byte(0b01111000);
+const auto obuFrameTypeBitshift = 3;
+
+const auto obuHeaderSize = 1;
+const auto obuHasExtensionMask = byte(0b00000100);
+const auto obuHasSizeMask = byte(0b00000010);
+
+const auto obuFrameTypeSequenceHeader = byte(1);
+
+const auto obuTemporalUnitDelimiter = std::vector<byte>{byte(0x12), byte(0x00)};
+
+const auto oneByteLeb128Size = 1;
+
+const uint8_t sevenLsbBitmask = 0b01111111;
+const uint8_t msbBitmask = 0b10000000;
+
+std::vector<binary_ptr> extractTemporalUnitObus(binary_ptr message) {
+	std::vector<shared_ptr<binary>> obus{};
+
+	if (message->size() <= 2 || (message->at(0) != obuTemporalUnitDelimiter.at(0)) ||
+	    (message->at(1) != obuTemporalUnitDelimiter.at(1))) {
+		return obus;
+	}
+
+	size_t messageIndex = 2;
+	while (messageIndex < message->size()) {
+		if ((message->at(messageIndex) & obuHasSizeMask) == byte(0)) {
+			return obus;
+		}
+
+		if ((message->at(messageIndex) & obuHasExtensionMask) != byte(0)) {
+			messageIndex++;
+		}
+
+		// https://aomediacodec.github.io/av1-spec/#leb128
+		uint32_t obuLength = 0;
+		uint8_t leb128Size = 0;
+		while (leb128Size < 8) {
+			auto leb128Index = messageIndex + leb128Size + obuHeaderSize;
+			if (message->size() < leb128Index) {
+				break;
+			}
+
+			auto leb128_byte = uint8_t(message->at(leb128Index));
+
+			obuLength |= ((leb128_byte & sevenLsbBitmask) << (leb128Size * 7));
+			leb128Size++;
+
+			if (!(leb128_byte & msbBitmask)) {
+				break;
+			}
+		}
+
+		obus.push_back(std::make_shared<binary>(message->begin() + messageIndex,
+		                                        message->begin() + messageIndex + obuHeaderSize +
+		                                            leb128Size + obuLength));
+
+		messageIndex += obuHeaderSize + leb128Size + obuLength;
+	}
+
+	return obus;
+}
+
+/*
+ *  0 1 2 3 4 5 6 7
+ * +-+-+-+-+-+-+-+-+
+ * |Z|Y| W |N|-|-|-|
+ * +-+-+-+-+-+-+-+-+
+ *
+ *	Z: MUST be set to 1 if the first OBU element is an
+ *	   OBU fragment that is a continuation of an OBU fragment
+ *	   from the previous packet, and MUST be set to 0 otherwise.
+ *
+ *	Y: MUST be set to 1 if the last OBU element is an OBU fragment
+ *	   that will continue in the next packet, and MUST be set to 0 otherwise.
+ *
+ *	W: two bit field that describes the number of OBU elements in the packet.
+ *	   This field MUST be set equal to 0 or equal to the number of OBU elements
+ *	   contained in the packet. If set to 0, each OBU element MUST be preceded by
+ *	   a length field. If not set to 0 (i.e., W = 1, 2 or 3) the last OBU element
+ *	   MUST NOT be preceded by a length field. Instead, the length of the last OBU
+ *	   element contained in the packet can be calculated as follows:
+ *	Length of the last OBU element =
+ *	   length of the RTP payload
+ *	 - length of aggregation header
+ *	 - length of previous OBU elements including length fields
+ *
+ *	N: MUST be set to 1 if the packet is the first packet of a coded video sequence, and MUST be set
+ *     to 0 otherwise.
+ *
+ * https://aomediacodec.github.io/av1-rtp-spec/#44-av1-aggregation-header
+ *
+ **/
+
+std::vector<binary_ptr> AV1RtpPacketizer::packetizeObu(binary_ptr message,
+                                                       uint16_t maximumFragmentSize) {
+
+	std::vector<shared_ptr<binary>> payloads{};
+	size_t messageIndex = 0;
+
+	if (message->size() < 1) {
+		return payloads;
+	}
+
+	// Cache sequence header and packetize with next OBU
+	auto frameType = (message->at(0) & obuFrameTypeMask) >> obuFrameTypeBitshift;
+	if (frameType == obuFrameTypeSequenceHeader) {
+		sequenceHeader = std::make_shared<binary>(message->begin(), message->end());
+		return payloads;
+	}
+
+	size_t messageRemaining = message->size();
+	while (messageRemaining > 0) {
+		auto obuCount = 1;
+		auto metadataSize = payloadHeaderSize;
+
+		if (sequenceHeader != nullptr) {
+			obuCount++;
+			metadataSize += /* 1 byte leb128 */ 1 + int(sequenceHeader->size());
+		}
+
+		auto payload = std::make_shared<binary>(
+		    std::min(size_t(maximumFragmentSize), messageRemaining + metadataSize));
+		auto payloadOffset = payloadHeaderSize;
+
+		payload->at(0) = byte(obuCount) << wBitshift;
+
+		// Packetize cached SequenceHeader
+		if (obuCount == 2) {
+			payload->at(0) ^= nMask;
+			payload->at(1) = byte(sequenceHeader->size() & sevenLsbBitmask);
+			payloadOffset += oneByteLeb128Size;
+
+			std::memcpy(payload->data() + payloadOffset, sequenceHeader->data(),
+			            sequenceHeader->size());
+			payloadOffset += int(sequenceHeader->size());
+
+			sequenceHeader = nullptr;
+		}
+
+		// Copy as much of OBU as possible into Payload
+		auto payloadRemaining = payload->size() - payloadOffset;
+		std::memcpy(payload->data() + payloadOffset, message->data() + messageIndex,
+		            payloadRemaining);
+		messageRemaining -= payloadRemaining;
+		messageIndex += payloadRemaining;
+
+		// Does this Fragment contain an OBU that started in a previous payload
+		if (payloads.size() > 0) {
+			payload->at(0) ^= zMask;
+		}
+
+		// This OBU will be continued in next Payload
+		if (messageIndex < message->size()) {
+			payload->at(0) ^= yMask;
+		}
+
+		payloads.push_back(payload);
+	}
+
+	return payloads;
+}
+
+AV1RtpPacketizer::AV1RtpPacketizer(AV1RtpPacketizer::Packetization packetization,
+                                   shared_ptr<RtpPacketizationConfig> rtpConfig,
+                                   uint16_t maximumFragmentSize)
+    : RtpPacketizer(rtpConfig), MediaHandlerRootElement(), maximumFragmentSize(maximumFragmentSize),
+      packetization(packetization) {}
+
+ChainedOutgoingProduct
+AV1RtpPacketizer::processOutgoingBinaryMessage(ChainedMessagesProduct messages,
+                                               message_ptr control) {
+	ChainedMessagesProduct packets = std::make_shared<std::vector<binary_ptr>>();
+	for (auto message : *messages) {
+		std::vector<binary_ptr> obus;
+
+		if (packetization == AV1RtpPacketizer::Packetization::TemporalUnit) {
+			obus = extractTemporalUnitObus(message);
+		} else {
+			obus.push_back(message);
+		}
+
+		for (auto obu : obus) {
+			auto payloads = packetizeObu(obu, maximumFragmentSize);
+			if (payloads.size() == 0) {
+				continue;
+			}
+
+			unsigned i = 0;
+			for (; i < payloads.size() - 1; i++) {
+				packets->push_back(packetize(payloads[i], false));
+			}
+			packets->push_back(packetize(payloads[i], true));
+		}
+	}
+
+	if (packets->size() == 0) {
+		return ChainedOutgoingProduct();
+	}
+
+	return {packets, control};
+}
+
+} // namespace rtc
+
+#endif /* RTC_ENABLE_MEDIA */

src/description.cpp

@@ -1183,6 +1183,10 @@ void Description::Video::addVP9Codec(int payloadType) {
 	addVideoCodec(payloadType, "VP9", nullopt);
 }
 
+void Description::Video::addAV1Codec(int pt) {
+	addVideoCodec(pt, "AV1", nullopt);
+}
+
 Description::Type Description::stringToType(const string &typeString) {
 	using TypeMap_t = std::unordered_map<string, Type>;
 	static const TypeMap_t TypeMap = {{"unspec", Type::Unspec},