//Should the ObjectType be allowed to be given...? 3 or 5 for Audio?
//No longer needed
/// <summary>
/// Write the required prerequesite data to the Buffer along with the Video Frame.
/// </summary>
/// <param name="profileLevelId">The profileLevelId (if not 1)</param>
//public void Depacketize(byte profileLevelId = 1)
//{
// DisposeBuffer();
// if (profileLevelId < 1) throw new ArgumentException("Must be a valid Mpeg 4 Profile Level Id with a value >= 1", "profileLevelId");
// /*
// Example usages for the "profile-level-id" parameter are:
// 1 : MPEG-4 Visual Simple Profile/Level 1
// 15 : AAC?
// 34 : MPEG-4 Visual Core Profile/Level 2
// 145: MPEG-4 Visual Advanced Real Time Simple Profile/Level 1
// */
// m_Buffer = new MemoryStream(Media.Containers.Mpeg.StartCodes.StartCodePrefix. //00 00 01
// Concat(Media.Common.Extensions.Linq.LinqExtensions.Yield(Media.Codecs.Video.Mpeg4.StartCodes.VisualObjectSequence)).
// Concat(Media.Common.Extensions.Linq.LinqExtensions.Yield(profileLevelId)). // B0 XX (ID)
// Concat(Media.Containers.Mpeg.StartCodes.StartCodePrefix).Concat(Assemble()).ToArray()); // 00 00 01 XX (DATA)
//}
public virtual void ProcessPacket(Rtp.RtpPacket packet, byte profileLevelId = 1)
{
int addIndex = packet.Timestamp - packet.SequenceNumber; // Depacketized.Count > 0 ? Depacketized.Keys.Last() : 0;
//byte[] t = new byte[] { profileLevelId, Media.Codecs.Video.Mpeg4.StartCodes.VisualObjectSequence };
//Common.MemorySegment a = new Common.MemorySegment(t, 0, 1);
//Common.MemorySegment b = new Common.MemorySegment(t, 1, 1);
//Creates 4 byte array each time with the first 3 bytes being the same, would need a way to combine MemorySegments for this to be any more efficient.
Depacketized.Add(addIndex++, CreatePrefixedStartCodeSegment(Media.Codecs.Video.Mpeg4.StartCodes.VisualObjectSequence));
//Depacketized.Add(addIndex++, StartCodePrefixSegment);
//Depacketized.Add(addIndex++, b);
//Or
//Depacketized.Add(addIndex++, StartCodePrefixSegment);
//Depacketized.Add(addIndex++, new Common.MemorySegment(new byte[] { Media.Codecs.Video.Mpeg4.StartCodes.VisualObjectSequence }));
//What a waste, 4 bytes + to describe 1
Depacketized.Add(addIndex++, new Common.MemorySegment(new byte[] { profileLevelId }));
Depacketized.Add(addIndex++, StartCodePrefixSegment);
// or
//Depacketized.Add(addIndex++, a);
//Depacketized.Add(addIndex++, StartCodePrefixSegment);
Depacketized.Add(addIndex++, packet.PayloadDataSegment);
}
/// <summary>
/// /// <summary>
/// Todo, break down logic, ParseHeaders, ParseAuxiliaryData, ParseAccessUnits(header)
/// Todo, Test in BigEndian, ensure that BitReader is not needed or at least the BitOrder overload which can detect the BitOrder the system is before reading the value in reverse of the way they should be.
/// </summary>
/// <param name="parsedAccessUnits"></param>
/// <param name="remainsInAu"></param>
/// <param name="headersPresent"></param>
/// <param name="profileId"></param>
/// <param name="channelConfiguration"></param>
/// <param name="frequencyIndex"></param>
/// <param name="sizeLength"></param>
/// <param name="indexLength"></param>
/// <param name="indexDeltaLength"></param>
/// <param name="CTSDeltaLength"></param>
/// <param name="DTSDeltaLength"></param>
/// <param name="auxDataSizeLength"></param>
/// <param name="randomAccessIndication"></param>
/// <param name="streamStateIndication"></param>
/// <param name="defaultAuSize"></param>
/// <param name="frameHeader"></param>
/// <param name="addPadding"></param>
/// <param name="includeAuHeaders"></param>
/// <param name="includeAuxData"></param>
public void Depacketize(Media.Rtp.RtpPacket packet, out int parsedAccessUnits, out int remainsInAu, bool headersPresent = true, int profileId = 0, int channelConfiguration = 0,
int frequencyIndex = 0, int sizeLength = 0, int indexLength = 0, int indexDeltaLength = 0, int CTSDeltaLength = 0,
int DTSDeltaLength = 0, int auxDataSizeLength = 0, bool randomAccessIndication = false, int streamStateIndication = 0,
int constantAuSize = 0, IEnumerable <byte> frameHeader = null,
bool addPadding = false, bool includeAuHeaders = false, bool includeAuxData = false)
{
remainsInAu = parsedAccessUnits = 0;
if (Common.IDisposedExtensions.IsNullOrDisposed(packet))
{
return;
}
//This key is based on the SequenceNumber in signed form
//int packetKey = GetPacketKey(packet.SequenceNumber);
int packetKey = Depacketized.Count; //(short)packet.SequenceNumber;
//Since the auIndex may be stored in highest 16 bits we could mask them out.
//0xffff, but would require unsigned key in Depacketized.
//There is also no reason to think that some other implementation wouldn't use more than 16 bits for the index and delta.
//if (Depacketized.ContainsKey(packetKey)) return;
//I thought about adding the key for this packet with Empty bytes which woudln't effect the stream and would keep it from being repacketized.
//Depacketized.Add(packetKey, null);
//From the beginning of the data in the actual payload
int offset = packet.Payload.Offset,
headerOctets = packet.HeaderOctets,
padding = packet.PaddingOctets,
max = packet.Payload.Count - (padding + headerOctets),
auIndex = 0, //Indicates the serial number of the associated Access Unit
auHeadersAvailable = 0; //The amount of Au Headers in the Au Header section
#region 3.2. RTP Payload Structure
/*
*
*
* 3.2.1. The AU Header Section
*
* When present, the AU Header Section consists of the AU-headers-length
* field, followed by a number of AU-headers, see Figure 2.
*
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+-+
|AU-headers-length|AU-header|AU-header| |AU-header|padding|
| | (1) | (2) | | (n) | bits |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+-+
|
| Figure 2: The AU Header Section
|
| 1) The AU-headers are configured using MIME format parameters and MAY be
| empty. If the AU-header is configured empty, the AU-headers-length
| field SHALL NOT be present and consequently the AU Header Section is
| empty. If the AU-header is not configured empty, then the AU-
| headers-length is a two octet field that specifies the length in bits
| of the immediately following AU-headers, excluding the padding bits.
|
| 2) Each AU-header is associated with a single Access Unit (fragment)
| contained in the Access Unit Data Section in the same RTP packet.
|
| 3) For each contained Access Unit (fragment), there is exactly one AU-
| header. Within the AU Header Section, the AU-headers are bit-wise
| concatenated in the order in which the Access Units are contained in
| the Access Unit Data Section. Hence, the n-th AU-header refers to
| the n-th AU (fragment). If the concatenated AU-headers consume a
| non-integer number of octets, up to 7 zero-padding bits MUST be
| inserted at the end in order to achieve octet-alignment of the AU
| Header Section.
*/
#endregion
#region Reference
//http://www.netmite.com/android/mydroid/donut/external/opencore/protocols/rtp_payload_parser/rfc_3640/src/rfc3640_payload_parser.cpp
/* https://gstrtpmp4adepay.c#L189
* Parse StreamMuxConfig according to ISO/IEC 14496-3:
*
* audioMuxVersion == 0 (1 bit)
* allStreamsSameTimeFraming == 1 (1 bit)
* numSubFrames == rtpmp4adepay->numSubFrames (6 bits)
* numProgram == 0 (4 bits)
* numLayer == 0 (3 bits)
*
* We only require audioMuxVersion == 0;
*
* The remaining bit of the second byte and the rest of the bits are used
* for audioSpecificConfig which we need to set in codec_info.
*/
//Object Type 5 Bits.
//SampleRate Index 4 Bits.
//Channels 4 Bits
//If SampleRate Index == 15, 24 Bits of SampleRate
//For object types 1-7 Then Parse the Frame Flags. (Which indicate a frameLength of 960?)
#endregion
//Determine the AU Headers Length (in bits)
int auHeaderLengthBits = headersPresent ? Common.Binary.ReadU16(packet.Payload.Array, ref offset, Common.Binary.IsLittleEndian) : 0,
//It will then be converted to bytes.
auHeaderLengthBytes = 0,
//AU-Size And the length of the underlying Elementary Stream Data for that access unit
auSize = 0,
//The offset of current the au Header being parsed
auHeaderOffset = offset,
//The offset of the auxiliary data if present
auxHeaderOffset = 0,
//The length of the auxiliary data if present
auxLengthBytes = 0,
//The absolute offset in the bitstream of the packet payload
bitOffset = Media.Common.Binary.BytesToBits(ref offset),
//The amount of bytes required to read the sizeLength bit field
sizeLengthBytes = Media.Common.Binary.BitsToBytes(sizeLength),
indexLengthBytes = Media.Common.Binary.BitsToBytes(indexLength),
indexDeltaLengthBytes = Media.Common.Binary.BitsToBytes(indexDeltaLength);
//If there are any auHeaders indicated
if (auHeaderLengthBits > 0)
{
//Convert bits to bytes
auHeaderLengthBytes = Media.Common.Binary.BitsToBytes(auHeaderLengthBits);
//Check enough bytes are available (2 is the size of the RFC Profile Header read above)
if (auHeaderLengthBytes >= max - 2)
{
throw new InvalidOperationException("Invalid Rfc Headers?");
}
//This many bits in each auAuheader + more if CTS or DTS is present.
int usedBits = sizeLength + indexDeltaLength;
//Throw an error if the length indicated is not equal to the amount of bits in use by the profile
if (auHeaderLengthBits != usedBits)
{
throw new InvalidOperationException("Invalid Au Headers?");
}
//Determine the amount of AU Headers possibly contained in the header section
auHeadersAvailable = 1 + (auHeaderLengthBits - usedBits) / usedBits;
//padding should be tracked here
//int lastHeaderPadding;
//auHeadersAvailable = Math.DivRem(1 + (auHeaderLengthBits - usedBits), usedBits, out lastHeaderPadding);
}
#region No AU Headers Length
// The AU Headers Length is either not present or known..
//{
// //Read the 'ES_ID'
// //ushort esId = Common.Binary.ReadU16(rtp.Payload, offset, Common.Binary.IsLittleEndian);
// //if (esId == 16) //This is AAC Audio 00 10?
//}
#endregion
//The length in bytes of all auHeaders is known, the last auHeader may have padding to achieve Octet alignment.
//Parse AU Headers should be seperate logic, probably in Audio.Aac
//Create a sorted list to allow the de-interleaving of access units if required.
//SortedList<int, IEnumerable<byte>> accessUnits = new SortedList<int, IEnumerable<byte>>();
//Should occur only after the last AU Header but before the individual payloads.
#region Parse Auxiliary Data
//Now skip over the aux data region.
if (0 != auxDataSizeLength)
{
//Store the offset where the auxData occurs
//This should be after all AuHeaders... e.g. @ auHeaderLengthBytes
auxHeaderOffset = offset;
//Note that when used in a BigEndian system that the Media.Common.Binary.ReadBigEndianInteger should be used.
//Read the size in bits of that section
int auxDataSizeBits = (int)Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, auxDataSizeLength);
if (auxDataSizeBits > 0)
{
//Calculate the amount of bytes in the auxillary data section
auxLengthBytes = (int)Media.Common.Binary.BitsToBytes(ref auxDataSizeBits);
//Ensure the amount of bytes indicated in the section are present in the contained data
if (max - offset < auxLengthBytes)
{
throw new InvalidOperationException("Invalid Au Aux Data?");
}
//Skip the bits indicated
bitOffset += auxDataSizeBits;
}
// as per 3) skip padding
//if (bitOffset > 0)
//{
// bitOffset = 0;
// ++offset;
//}
}
//Default auxData
Common.MemorySegment auxillaryData = Common.MemorySegment.Empty;
//If there was any auxillary data
if (auxLengthBytes > 0)
{
//Get the auxData
auxillaryData = new Common.MemorySegment(packet.Payload.Array, auxHeaderOffset, auxLengthBytes);
}
#endregion
//Look for Access Units in the packet
//Must also track offset.
while (Common.Binary.BitsToBytes(ref bitOffset) < max)
{
//Should read from auHeaderOffset
#region ParseAuHeader
#region AU-size
/*
* AU-size: Indicates the size in octets of the associated Access Unit
* in the Access Unit Data Section in the same RTP packet. When the
* AU-size is associated with an AU fragment, the AU size indicates
* the size of the entire AU and not the size of the fragment. In
* this case, the size of the fragment is known from the size of the
* AU data section. This can be exploited to determine whether a
* packet contains an entire AU or a fragment, which is particularly
* useful after losing a packet carrying the last fragment of an AU.
*/
if (sizeLength > 0)
{
//Ensure there are enough bytes required to read the auSize
if (sizeLengthBytes > (remainsInAu = (max - offset) - 1))
{
offset = max;
continue;
}
//Read the bits for the size and the index to align the read.
auSize = (int)Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, sizeLength);
//auSize can never be greater than max.
if (auSize >= max)
{
throw new InvalidOperationException("auSize is larger than expected.");
}
}
else
{
auSize = constantAuSize;
}
#endregion
//Check for auSize == 0 because there is nothing to consume
if (auSize == 0)
{
//Todo, offset didn't move for the bits just read...
continue;
}
#region AU-Index / AU-Index-delta
/*
* AU-Index-delta: The AU-Index-delta field is an unsigned integer that
* specifies the serial number of the associated AU as the difference
* with respect to the serial number of the previous Access Unit.
* Hence, for the n-th (n>1) AU, the serial number is found from:
*
* AU-Index(n) = AU-Index(n-1) + AU-Index-delta(n) + 1
*
* If the AU-Index field is present in the first AU-header in the AU
* Header Section, then the AU-Index-delta field MUST be present in
* any subsequent (non-first) AU-header. When the AU-Index-delta is
* coded with the value 0, it indicates that the Access Units are
* consecutive in decoding order. An AU-Index-delta value larger
* than 0 signals that interleaving is applied.
*/
//This should check that if indexLength == 0 and indexDeltaLength == 0 there is an invalid operation exception
//This can be done before we even read into the packet.
if (Depacketized.Count == 0 && indexLength > 0)
{
//Ensure enough data is present to read the bit field
if (indexLength > (remainsInAu = (max - offset) - 1))
{
offset = max;
continue;
}
//Notes that this should be read out of the Au Headers Section for the Au being parsed
auIndex = (int)Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, indexLength);
}
else if (indexDeltaLength > 0)//Must be present... should check.
{
//If Depacketized.Count == 0 this should not occur...
//Ensure enough data is present to read the bit field
if (indexDeltaLength > (remainsInAu = (max - offset) - 1))
{
offset = max;
bitOffset = Common.Binary.BytesToBits(ref max);
continue;
}
//Notes that this should be read out of the Au Headers Section for the Au being parsed
auIndex = (int)Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, indexDeltaLength);
//An AU-Index-delta value larger than 0 signals that interleaving is applied.
if (auIndex == 0)
{
auIndex = Depacketized.Count + 1;
}
}
#endregion
#region CTSDeltaLength
//From RFC3640: "The CTS-flag field MUST be present in each AU-header
// if the length of the CTS-delta field is signaled to
// be larger than zero."
if (0 != CTSDeltaLength)
{
bool CTSFlag = Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, 1) > 0;
if (CTSFlag)
{
int CTSDelta = (int)Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, CTSDeltaLength);
}
}
#endregion
#region DTSDeltaLength
if (0 != DTSDeltaLength)
{
bool DTSFlag = Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, 1) > 0;
if (DTSFlag)
{
int DTSDelta = (int)Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, DTSDeltaLength);
}
}
#endregion
#region randomAccessIndication
if (randomAccessIndication)
{
bool RAPFlag = Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, 1) > 0;
}
#endregion
#region StreamState
//StreamState
//https://www.ietf.org/proceedings/54/slides/avt-6.pdf
if (0 != streamStateIndication)
{
//number of bits for Stream-State
int streamState = (int)Media.Common.Binary.ReadBitsMSB(packet.Payload.Array, ref bitOffset, streamStateIndication);
//Read that many more bits
//Media.Common.Binary.ReadBinaryInteger(rtp.Payload.Array, ref offset, ref bitOffset, streamState, Common.Binary.IsLittleEndian);
bitOffset += streamState;
}
#endregion
//Might not need to skip padding, this should be determined by auHeaderLengthBits % 8
// as per 3) skip padding
int localPadding = (bitOffset & 7);
if (localPadding > 0) //Common.Binary.Align (bitOffset), ref, out byteOffset
{
bitOffset += localPadding;
}
#endregion
//Get the header which corresponds to this access unit (note that this should be done in ParseAuHeader)
//This should also take into account that the last auHeader may be padded with up to 7 '0' bits.
using (var accessUnitHeader = new Common.MemorySegment(packet.Payload.Array, auHeaderOffset, auHeaderLengthBytes, false == includeAuHeaders))
{
//Use a auHeader from the header section, if there is more then one move the offset to the next header
if (--auHeadersAvailable > 0)
{
auHeaderOffset += auHeaderLengthBytes;
}
//Determine how much data remains in the payload
int remains = max - (offset - packet.Payload.Offset);
//If the size of the accessUnit is greater than what remains
if (auSize > remains)
{
//Indicate that much remains
remainsInAu = auSize - remains;
//Truncated or Fragmented?
//Make auSize equal to what actually remains.
auSize = remains;
}
else
{
remainsInAu = 0; //Nothing else remains if the auSize indicated was completely contained in the payload
}
//Project the data in the payload from the offset of the access unit until its declared size.
using (Common.MemorySegment accessUnitData = new Common.MemorySegment(packet.Payload.Array, Common.Binary.BitsToBytes(ref bitOffset), auSize, false)) //offset, auSize))
{
//Prepend the accessUnitHeaer with the data to create a depacketized au if the option was specified
//var depacketizedAccessUnit = includeAuHeaders ? Enumerable.Concat(accessUnitHeader, accessUnitData) : accessUnitData;
if (includeAuHeaders)
{
Depacketized.Add(auIndex++, accessUnitHeader);
}
//If there aux data then add it after the Au header we just added if the option was specified
if (includeAuxData && auxLengthBytes > 0)
{
//Add the auxillary data
//depacketizedAccessUnit = Enumerable.Concat(depacketizedAccessUnit, auxillaryData);
Depacketized.Add(auIndex++, auxillaryData);
}
#region Todo allow partial data from previous parsing, (needs bitOffset to be ref)
//If data exists from the previous packet which is not part of the payload
//if (previouslyIncompleteAccessUnitWithAuxHeaderAndData != null)
//{
// //May have to write the length in the frameData
// depacketizedAccessUnit = Enumerable.Concat(frameHeader, depacketizedAccessUnit);
// previouslyIncompleteAccessUnitWithAuxHeaderAndData = null;
//}
//else
//{
// //Could just update the length but
// //This is a bit field though and needs to have the WriteBits methods available then
// //Media.Common.Binary.Write16(adtsHeader, 6, false, (short)auSize);
// //Create the header for the frame, (should only be done once and the length should be updated each time)
// //Should also have the ADTS header only 1 time, and only after the frameLength is known.
// depacketizedAccessUnit = Enumerable.Concat(CreateADTSHeader(profileId, frequencyIndex, channelConfiguration, auSize), depacketizedAccessUnit);
//}
#endregion
//Add padding if required.. ?Allow custom
if (addPadding)
{
int required = max - offset;
//if (required > 0) depacketizedAccessUnit = Enumerable.Concat(depacketizedAccessUnit, Enumerable.Repeat<byte>(byte.MinValue, required));
if (required > 0)
{
Depacketized.Add(auIndex++, new Common.MemorySegment(Enumerable.Repeat <byte>(byte.MinValue, required).ToArray()));
}
}
//Add the Access Unit to the list (copies memory)
if (auSize > 1)
{
Depacketized.Add(auIndex++, accessUnitData);
}
//Move the byte offset leaving the bit offset in tact
offset += auSize;
bitOffset += Common.Binary.BytesToBits(ref auSize);
//Keep track of the amount of access units parsed
++parsedAccessUnits;
}
}
}
//Return the access units in decoding order from the lamda, which will be written to the buffer
return;
}
//internal protected void WriteStartCode(ref byte nalHeader, bool fullStartCodes = false)
//{
// int addIndex = Depacketized.Count;
// DepacketizeStartCode(ref addIndex, ref nalHeader, fullStartCodes);
//}
//internal static Common.MemorySegment CreateStartCode(ref byte nalType)
//{
//}
internal protected void DepacketizeStartCode(ref int addIndex, ref byte nalHeader, bool fullStartCodes = false)
{
//Determine the type of Nal
byte nalType = (byte)(nalHeader & Common.Binary.FiveBitMaxValue);
//Store the nalType contained (this is possibly not in sorted order of which they occur)
m_ContainedNalTypes.Add(nalType);
if (fullStartCodes)
{
Depacketized.Add(addIndex++, FullStartSequenceSegment);
return;
}
//Determine the type of start code prefix required.
switch (nalType)
{
////Should technically only be written for first iframe in au and only when not precceded by sps and pps
//case Media.Codecs.Video.H264.NalUnitType.InstantaneousDecoderRefresh://5
//case Media.Codecs.Video.H264.NalUnitType.SequenceParameterSetSubset:// 15 (6190)
// {
// //Check if first nal in Access Unit m_ContainedNalTypes[0] == Media.Codecs.Video.H264.NalUnitType.SequenceParameterSetSubset;
// if (m_Buffer.Position == 0) goto case Media.Codecs.Video.H264.NalUnitType.SequenceParameterSet;
// //Handle without extra byte
// goto default;
// }
case Media.Codecs.Video.H264.NalUnitType.SupplementalEncoderInformation: //6:
case Media.Codecs.Video.H264.NalUnitType.SequenceParameterSet: //7:
case Media.Codecs.Video.H264.NalUnitType.PictureParameterSet: //8:
case Media.Codecs.Video.H264.NalUnitType.AccessUnitDelimiter: //9
{
//Use the FullStartSequence
Depacketized.Add(addIndex++, FullStartSequenceSegment);
return;
}
// See: [ITU-T H.264] 7.4.1.2.4 Detection of the first VCL NAL unit of a primary coded picture
//case Media.Codecs.Video.H264.NalUnitType.CodedSlice:1 (6190)
//case Media.Codecs.Video.H264.NalUnitType.SliceExtension:20 (6190)
// {
// //Write the extra 0 byte to the Buffer (could also check for a contained slice header to eliminate the possibility of needing to check?)
// if (m_Buffer.Position == 0 && isFirstVclInPrimaryCodedPicture()) goto case Media.Codecs.Video.H264.NalUnitType.AccessUnitDelimiter;
// //Handle as normal
// goto default;
// }
default:
{
#region Notes
/* 7.1 NAL Unit Semantics
*
* 1) The first byte of the RBSP contains the (most significant, left-most) eight bits of the SODB; the next byte of
* the RBSP shall contain the next eight bits of the SODB, etc., until fewer than eight bits of the SODB remain.
*
* 2) rbsp_trailing_bits( ) are present after the SODB as follows:
* i) The first (most significant, left-most) bits of the final RBSP byte contains the remaining bits of the SODB,
* (if any)
* ii) The next bit consists of a single rbsp_stop_one_bit equal to 1, and
* iii) When the rbsp_stop_one_bit is not the last bit of a byte-aligned byte, one or more
* rbsp_alignment_zero_bit is present to result in byte alignment.
*
* 3) One or more cabac_zero_word 16-bit syntax elements equal to 0x0000 may be present in some RBSPs after
* the rbsp_trailing_bits( ) at the end of the RBSP.
* Syntax structures having these RBSP properties are denoted in the syntax tables using an "_rbsp" suffix. These
* structures shall be carried within NAL units as the content of the rbsp_byte[ i ] data bytes. The association of the RBSP
* syntax structures to the NAL units shall be as specified in Table 7-1.
* NOTE - When the boundaries of the RBSP are known, the decoder can extract the SODB from the RBSP by concatenating the bits
* of the bytes of the RBSP and discarding the rbsp_stop_one_bit, which is the last (least significant, right-most) bit equal to 1, and
* discarding any following (less significant, farther to the right) bits that follow it, which are equal to 0. The data necessary for the
* decoding process is contained in the SODB part of the RBSP.
* emulation_prevention_three_byte is a byte equal to 0x03. When an emulation_prevention_three_byte is present in the
* NAL unit, it shall be discarded by the decoding process.
* The last byte of the NAL unit shall not be equal to 0x00.
* Within the NAL unit, the following three-byte sequences shall not occur at any byte-aligned position:
* – 0x000000
* – 0x000001
* – 0x000002
* Within the NAL unit, any four-byte sequence that starts with 0x000003 other than the following sequences shall not
* occur at any byte-aligned position:
* – 0x00000300
* – 0x00000301
* – 0x00000302
* – 0x00000303
*
*/
//Could also check last byte(s) in buffer to ensure no 0...
//FFMPEG changed to always emit full start codes.
//https://ffmpeg.org/doxygen/trunk/rtpdec__h264_8c_source.html
#endregion
//Add the short start sequence
Depacketized.Add(addIndex++, ShortStartSequenceSegment);
//Done
return;
}
}
}
//Could be called Depacketize
//Virtual because the RFC6190 logic defers to this method for non SVC nal types.
/// <summary>
/// Depacketizes a single packet.
/// </summary>
/// <param name="packet"></param>
/// <param name="containsSps"></param>
/// <param name="containsPps"></param>
/// <param name="containsSei"></param>
/// <param name="containsSlice"></param>
/// <param name="isIdr"></param>
internal protected virtual void ProcessPacket(Rtp.RtpPacket packet, bool ignoreForbiddenZeroBit = true, bool fullStartCodes = false)
{
//If the packet is null or disposed then do not process it.
if (Common.IDisposedExtensions.IsNullOrDisposed(packet))
{
return;
}
//Just put the packets into Depacketized at the end for most cases.
int packetKey = Depacketized.Count;
//The packets are not stored by SequenceNumber in Depacketized, they are stored in whatever Decoder Order is necessary.
//Already contained. (Might want to wait for the Decoder Order Number to be checked.
//if (Depacketized.ContainsKey(packetKey)) return;
//(May need to handle re-ordering)
//In such cases this step needs to place the packets into a seperate collection for sorting on DON / TSOFFSET before writing to the buffer.
//From the beginning of the data in the actual payload
int offset = packet.Payload.Offset + packet.HeaderOctets,
padding = packet.PaddingOctets,
count = (packet.Payload.Count - padding),
end = packet.Length - padding;
//Must have at least 2 bytes (When nalUnitType is a FragmentUnit.. 3)
if (count <= 2 || offset > packet.Payload.Count)
{
return;
}
//Obtain the data of the packet with respect to extensions and csrcs present.
byte[] packetData = packet.Payload.Array;
//Determine if the forbidden bit is set and the type of nal from the first byte
byte firstByte = packetData[offset];
//Should never be set... (unless decoding errors are present)
if (false == ignoreForbiddenZeroBit && ((firstByte & 0x80) >> 7) != 0)
{
return; //throw new Exception("forbiddenZeroBit");
}
byte nalUnitType = (byte)(firstByte & Common.Binary.FiveBitMaxValue);
//RFC6184 @ Page 20
//o The F bit MUST be cleared if all F bits of the aggregated NAL units are zero; otherwise, it MUST be set.
//if (forbiddenZeroBit && nalUnitType <= 23 && nalUnitType > 29) throw new InvalidOperationException("Forbidden Zero Bit is Set.");
//Needs other state to check if previously F was set or not
//Media.Codecs.Video.H264.NalUnitPriority priority = (Media.Codecs.Video.H264.NalUnitPriority)((firstByte & 0x60) >> 5);
//Determine what to do
switch (nalUnitType)
{
//Reserved - Ignore
case Media.Codecs.Video.H264.NalUnitType.Unknown:
case Media.Codecs.Video.H264.NalUnitType.PayloadContentScalabilityInformation:
case Media.Codecs.Video.H264.NalUnitType.Reserved:
{
//May have 4 byte NAL header.
//Do not handle
return;
}
case Media.Codecs.Video.H264.NalUnitType.SingleTimeAggregationA: //STAP - A
case Media.Codecs.Video.H264.NalUnitType.SingleTimeAggregationB: //STAP - B
case Media.Codecs.Video.H264.NalUnitType.MultiTimeAggregation16: //MTAP - 16
case Media.Codecs.Video.H264.NalUnitType.MultiTimeAggregation24: //MTAP - 24
{
//Move to Nal Data
++offset;
//Todo Determine if need to Order by DON first.
//EAT DON for ALL BUT STAP - A
if (nalUnitType != Media.Codecs.Video.H264.NalUnitType.SingleTimeAggregationA)
{
//Should check for 2 bytes.
//Read the DecoderOrderingNumber and add the value from the index.
packetKey = Common.Binary.ReadU16(packetData, ref offset, BitConverter.IsLittleEndian);
//If the number was already observed skip this packet.
//if (Depacketized.ContainsKey(packetKey)) return;
}
//Should check for 2 bytes.
//Consume the rest of the data from the packet
while (offset < count) // + 2 <=
{
//Determine the nal unit size which does not include the nal header
int tmp_nal_size = Common.Binary.Read16(packetData, ref offset, BitConverter.IsLittleEndian);
//Should check for tmp_nal_size > 0
//If the nal had data and that data is in this packet then write it
if (tmp_nal_size >= 0)
{
//For DOND and TSOFFSET
switch (nalUnitType)
{
case Media.Codecs.Video.H264.NalUnitType.MultiTimeAggregation16: // MTAP - 16 (May require re-ordering)
{
//Should check for 3 bytes.
//DOND 1 byte
//Read DOND and TSOFFSET, combine the values with the existing index
packetKey = (int)Common.Binary.ReadU24(packetData, ref offset, BitConverter.IsLittleEndian);
//If the number was already observed skip this packet.
//if (Depacketized.ContainsKey(packetKey)) return;
goto default;
}
case Media.Codecs.Video.H264.NalUnitType.MultiTimeAggregation24: // MTAP - 24 (May require re-ordering)
{
//Should check for 4 bytes.
//DOND 2 bytes
//Read DOND and TSOFFSET , combine the values with the existing index
packetKey = (int)Common.Binary.ReadU32(packetData, ref offset, BitConverter.IsLittleEndian);
//If the number was already observed skip this packet.
//if (Depacketized.ContainsKey(packetKey)) return;
goto default;
}
default:
{
//Should check for tmp_nal_size > 0
//Could check for extra bytes or emulation prevention
//https://github.com/raspberrypi/userland/blob/master/containers/rtp/rtp_h264.c
//(Stores the nalType) Write the start code
DepacketizeStartCode(ref packetKey, ref packetData[offset], fullStartCodes);
//Add the depacketized data and increase the index.
//Ensure the size is within the count.
//When tmp_nal_size is 0 packetData which is referenced by this segment which will have a 0 count.
Depacketized.Add(packetKey++, new Common.MemorySegment(packetData, offset, Common.Binary.Min(tmp_nal_size, count - offset)));
//Move the offset past the nal
offset += tmp_nal_size;
continue;
}
}
}
}
//No more data in packet.
return;
}
case Media.Codecs.Video.H264.NalUnitType.FragmentationUnitA: //FU - A
case Media.Codecs.Video.H264.NalUnitType.FragmentationUnitB: //FU - B (May require re-ordering)
{
/*
* Informative note: When an FU-A occurs in interleaved mode, it
* always follows an FU-B, which sets its DON.
* Informative note: If a transmitter wants to encapsulate a single
* NAL unit per packet and transmit packets out of their decoding
* order, STAP-B packet type can be used.
*/
//Needs atleast 2 bytes to reconstruct...
//3 bytes for any valid data to follow after the header.
if (count >= 2)
{
//Offset still at the firstByte (FU Indicator) move to and read FU Header
byte FUHeader = packetData[++offset];
bool Start = ((FUHeader & 0x80) >> 7) > 0;
//https://tools.ietf.org/html/rfc6184 page 31...
//A fragmented NAL unit MUST NOT be transmitted in one FU; that is, the
//Start bit and End bit MUST NOT both be set to one in the same FU
//header.
//bool End = ((FUHeader & 0x40) >> 6) > 0;
//ignoreReservedBit
//bool Reserved = (FUHeader & 0x20) != 0;
//Should not be set
//if (Reserved) throw new InvalidOperationException("Reserved Bit Set");
//Move to data (Just read the FU Header)
++offset;
//packet.SequenceNumber - packet.Timestamp;
//Store the DecoderingOrderNumber we will derive from the timestamp and sequence number.
//int DecodingOrderNumber = packetKey;
//DON Present in FU - B, add the DON to the DecodingOrderNumber
if (nalUnitType == Media.Codecs.Video.H264.NalUnitType.FragmentationUnitB)
{
//Needs 2 more bytes...
Common.Binary.ReadU16(packetData, ref offset, BitConverter.IsLittleEndian); //offset += 2;
}
//Should verify count... just consumed 1 - 3 bytes and only required 2.
//Determine the fragment size
int fragment_size = count - offset;
//Don't emit empty fragments
//if (fragment_size == 0) return;
//If the start bit was set
if (Start)
{
//ignoreEndBit
//if (End) throw new InvalidOperationException("Start and End Bit Set in same FU");
//Reconstruct the nal header
//Use the first 3 bits of the first byte and last 5 bites of the FU Header
byte nalHeader = (byte)((firstByte & 0xE0) | (FUHeader & Common.Binary.FiveBitMaxValue));
//(Stores the nal) Write the start code
DepacketizeStartCode(ref packetKey, ref nalHeader, fullStartCodes);
//Wasteful but there is no other way to provide this byte since it is constructor from two values in the header
//Unless of course a FragmentHeader : MemorySegment was created, which could have a NalType property ...
//Could also just have an overload which writes the NalHeader
//Would need a CreateNalSegment static method with option for full (4 + 1) or short code ( 3 + 1)/
Depacketized.Add(packetKey++, new Common.MemorySegment(new byte[] { nalHeader }));
}
//Add the depacketized data
Depacketized.Add(packetKey, new Common.MemorySegment(packetData, offset, fragment_size));
//Allow If End to Write End Sequence?
//Should only be done if last byte is 0?
//if (End) Buffer.WriteByte(Media.Codecs.Video.H264.NalUnitType.EndOfSequence);
}
//No more data?
return;
}
default: //Any other type excluding PayloadContentScalabilityInformation(30) and Reserved(31)
{
//(Stores the nalUnitType) Write the start code
DepacketizeStartCode(ref packetKey, ref nalUnitType, fullStartCodes);
//Add the depacketized data
Depacketized.Add(packetKey, new Common.MemorySegment(packetData, offset, count - offset));
return;
}
}
}
