Merge pull request #1134 from edmonds/h265-rtp-depacketizer

Add H265RtpDepacketizer

CMakeLists.txt

@@ -82,6 +82,7 @@ set(LIBDATACHANNEL_SOURCES
 	${CMAKE_CURRENT_SOURCE_DIR}/src/h264rtpdepacketizer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/nalunit.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/h265rtppacketizer.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/src/h265rtpdepacketizer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/h265nalunit.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/av1rtppacketizer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/rtcpnackresponder.cpp
@@ -120,6 +121,7 @@ set(LIBDATACHANNEL_HEADERS
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264rtpdepacketizer.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/nalunit.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265rtppacketizer.hpp
+	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265rtpdepacketizer.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265nalunit.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1rtppacketizer.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/rtcpnackresponder.hpp

include/rtc/h265rtpdepacketizer.hpp

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) 2020 Staz Modrzynski
+ * Copyright (c) 2020-2024 Paul-Louis Ageneau
+ * Copyright (c) 2024 Robert Edmonds
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef RTC_H265_RTP_DEPACKETIZER_H
+#define RTC_H265_RTP_DEPACKETIZER_H
+
+#if RTC_ENABLE_MEDIA
+
+#include "common.hpp"
+#include "h265nalunit.hpp"
+#include "mediahandler.hpp"
+#include "message.hpp"
+#include "rtp.hpp"
+
+#include <iterator>
+
+namespace rtc {
+
+/// RTP depacketization for H265
+class RTC_CPP_EXPORT H265RtpDepacketizer : public MediaHandler {
+public:
+	using Separator = NalUnit::Separator;
+
+	H265RtpDepacketizer(Separator separator = Separator::LongStartSequence);
+	virtual ~H265RtpDepacketizer() = default;
+
+	void incoming(message_vector &messages, const message_callback &send) override;
+
+private:
+	std::vector<message_ptr> mRtpBuffer;
+	const NalUnit::Separator separator;
+
+	void addSeparator(binary& accessUnit);
+	message_vector buildFrames(message_vector::iterator firstPkt, message_vector::iterator lastPkt,
+	                           uint8_t payloadType, uint32_t timestamp);
+};
+
+} // namespace rtc
+
+#endif // RTC_ENABLE_MEDIA
+
+#endif // RTC_H265_RTP_DEPACKETIZER_H

include/rtc/rtc.hpp

@@ -32,6 +32,7 @@
 #include "h264rtppacketizer.hpp"
 #include "h264rtpdepacketizer.hpp"
 #include "h265rtppacketizer.hpp"
+#include "h265rtpdepacketizer.hpp"
 #include "mediahandler.hpp"
 #include "plihandler.hpp"
 #include "rembhandler.hpp"

src/h265nalunit.cpp

@@ -34,7 +34,7 @@ H265NalUnitFragment::fragmentsFrom(shared_ptr<H265NalUnit> nalu, uint16_t maxFra
 	auto fragments_count = ceil(double(nalu->size()) / maxFragmentSize);
 	maxFragmentSize = uint16_t(int(ceil(nalu->size() / fragments_count)));
 
-	// 3 bytes for FU indicator and FU header
+	// 3 bytes for NALU header and FU header
 	maxFragmentSize -= (H265_NAL_HEADER_SIZE + H265_FU_HEADER_SIZE);
 	auto f = nalu->forbiddenBit();
 	uint8_t nuhLayerId = nalu->nuhLayerId() & 0x3F;        // 6 bits

src/h265rtpdepacketizer.cpp

@@ -0,0 +1,193 @@
+/**
+ * Copyright (c) 2023-2024 Paul-Louis Ageneau
+ * Copyright (c) 2024 Robert Edmonds
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#if RTC_ENABLE_MEDIA
+
+#include "h265rtpdepacketizer.hpp"
+#include "h265nalunit.hpp"
+
+#include "impl/internals.hpp"
+
+#include <algorithm>
+
+namespace rtc {
+
+const binary naluLongStartCode = {byte{0}, byte{0}, byte{0}, byte{1}};
+const binary naluShortStartCode = {byte{0}, byte{0}, byte{1}};
+
+const uint8_t naluTypeAP = 48;
+const uint8_t naluTypeFU = 49;
+
+H265RtpDepacketizer::H265RtpDepacketizer(Separator separator) : separator(separator) {
+	switch (separator) {
+	case Separator::StartSequence: [[fallthrough]];
+	case Separator::LongStartSequence: [[fallthrough]];
+	case Separator::ShortStartSequence:
+		break;
+	case Separator::Length: [[fallthrough]];
+	default:
+		throw std::invalid_argument("Invalid separator");
+	}
+}
+
+void H265RtpDepacketizer::addSeparator(binary& accessUnit)
+{
+	switch (separator) {
+	case Separator::StartSequence: [[fallthrough]];
+	case Separator::LongStartSequence:
+		accessUnit.insert(accessUnit.end(),
+		                  naluLongStartCode.begin(),
+		                  naluLongStartCode.end());
+		break;
+	case Separator::ShortStartSequence:
+		accessUnit.insert(accessUnit.end(),
+		                  naluShortStartCode.begin(),
+		                  naluShortStartCode.end());
+		break;
+	case Separator::Length: [[fallthrough]];
+	default:
+		throw std::invalid_argument("Invalid separator");
+	}
+}
+
+message_vector H265RtpDepacketizer::buildFrames(message_vector::iterator begin,
+                                                message_vector::iterator end,
+                                                uint8_t payloadType, uint32_t timestamp) {
+	message_vector out = {};
+	auto accessUnit = binary{};
+	auto frameInfo = std::make_shared<FrameInfo>(payloadType, timestamp);
+
+	for (auto it = begin; it != end; ++it) {
+		auto pkt = it->get();
+		auto pktParsed = reinterpret_cast<const rtc::RtpHeader *>(pkt->data());
+		auto rtpHeaderSize = pktParsed->getSize() + pktParsed->getExtensionHeaderSize();
+		auto rtpPaddingSize = 0;
+
+		if (pktParsed->padding()) {
+			rtpPaddingSize = std::to_integer<uint8_t>(pkt->at(pkt->size() - 1));
+		}
+
+		if (pkt->size() == rtpHeaderSize + rtpPaddingSize) {
+			PLOG_VERBOSE << "H.265 RTP packet has empty payload";
+			continue;
+		}
+
+		auto nalUnitHeader =
+		    H265NalUnitHeader{std::to_integer<uint8_t>(pkt->at(rtpHeaderSize)),
+		                      std::to_integer<uint8_t>(pkt->at(rtpHeaderSize + 1))};
+
+		if (nalUnitHeader.unitType() == naluTypeFU) {
+			auto nalUnitFragmentHeader = H265NalUnitFragmentHeader{
+			    std::to_integer<uint8_t>(pkt->at(rtpHeaderSize + sizeof(H265NalUnitHeader)))};
+
+			// RFC 7798: "When set to 1, the S bit indicates the start of a fragmented
+			// NAL unit, i.e., the first byte of the FU payload is also the first byte of
+			// the payload of the fragmented NAL unit. When the FU payload is not the start
+			// of the fragmented NAL unit payload, the S bit MUST be set to 0."
+			if (nalUnitFragmentHeader.isStart() || accessUnit.empty()) {
+				addSeparator(accessUnit);
+				nalUnitHeader.setUnitType(nalUnitFragmentHeader.unitType());
+				accessUnit.emplace_back(byte(nalUnitHeader._first));
+				accessUnit.emplace_back(byte(nalUnitHeader._second));
+			}
+
+			accessUnit.insert(accessUnit.end(),
+			                  pkt->begin() + rtpHeaderSize + sizeof(H265NalUnitHeader) +
+			                      sizeof(H265NalUnitFragmentHeader),
+			                  pkt->end());
+		} else if (nalUnitHeader.unitType() == naluTypeAP) {
+			auto currOffset = rtpHeaderSize + sizeof(H265NalUnitHeader);
+
+			while (currOffset + sizeof(uint16_t) < pkt->size()) {
+				auto naluSize = std::to_integer<uint16_t>(pkt->at(currOffset)) << 8 |
+				                std::to_integer<uint16_t>(pkt->at(currOffset + 1));
+
+				currOffset += sizeof(uint16_t);
+
+				if (pkt->size() < currOffset + naluSize) {
+					throw std::runtime_error("H265 AP declared size is larger than buffer");
+				}
+
+				addSeparator(accessUnit);
+				accessUnit.insert(accessUnit.end(), pkt->begin() + currOffset,
+				                  pkt->begin() + currOffset + naluSize);
+
+				currOffset += naluSize;
+			}
+		} else if (nalUnitHeader.unitType() < naluTypeAP) {
+			// "NAL units with NAL unit type values in the range of 0 to 47, inclusive, may be
+			// passed to the decoder."
+			addSeparator(accessUnit);
+			accessUnit.insert(accessUnit.end(), pkt->begin() + rtpHeaderSize, pkt->end());
+		} else {
+			// "NAL-unit-like structures with NAL unit type values in the range of 48 to 63,
+			// inclusive, MUST NOT be passed to the decoder."
+		}
+	}
+
+	if (!accessUnit.empty()) {
+		out.emplace_back(make_message(accessUnit.begin(), accessUnit.end(), Message::Binary, 0,
+		                              nullptr, frameInfo));
+	}
+
+	return out;
+}
+
+void H265RtpDepacketizer::incoming(message_vector &messages, const message_callback &) {
+	messages.erase(std::remove_if(messages.begin(), messages.end(),
+	                              [&](message_ptr message) {
+		                              if (message->type == Message::Control) {
+			                              return false;
+		                              }
+
+		                              if (message->size() < sizeof(RtpHeader)) {
+			                              PLOG_VERBOSE << "RTP packet is too small, size="
+			                                           << message->size();
+			                              return true;
+		                              }
+
+		                              mRtpBuffer.push_back(std::move(message));
+		                              return true;
+	                              }),
+	               messages.end());
+
+	while (mRtpBuffer.size() != 0) {
+		uint8_t payload_type = 0;
+		uint32_t current_timestamp = 0;
+		size_t packets_in_timestamp = 0;
+
+		for (const auto &pkt : mRtpBuffer) {
+			auto p = reinterpret_cast<const rtc::RtpHeader *>(pkt->data());
+
+			if (current_timestamp == 0) {
+				current_timestamp = p->timestamp();
+				payload_type = p->payloadType(); // should all be the same for data of the same codec
+			} else if (current_timestamp != p->timestamp()) {
+				break;
+			}
+
+			packets_in_timestamp++;
+		}
+
+		if (packets_in_timestamp == mRtpBuffer.size()) {
+			break;
+		}
+
+		auto begin = mRtpBuffer.begin();
+		auto end = mRtpBuffer.begin() + (packets_in_timestamp - 1);
+
+		auto frames = buildFrames(begin, end + 1, payload_type, current_timestamp);
+		messages.insert(messages.end(), frames.begin(), frames.end());
+		mRtpBuffer.erase(mRtpBuffer.begin(), mRtpBuffer.begin() + packets_in_timestamp);
+	}
+}
+
+} // namespace rtc
+
+#endif // RTC_ENABLE_MEDIA