/// <summary>
/// Adds a block to the playback blocks by converting the given frame.
/// If there are no more blocks in the pool, the oldest block is returned to the pool
/// and reused for the new block. The source frame is automatically disposed.
/// </summary>
/// <param name="source">The source.</param>
/// <param name="container">The container.</param>
/// <returns>The filled block.</returns>
public MediaBlock Add(MediaFrame source, MediaContainer container)
{
lock (SyncRoot)
{
// Check if we already have a block at the given time
if (IsInRange(source.StartTime))
{
var reapeatedBlock = PlaybackBlocks.FirstOrDefault(f => f.StartTime.Ticks == source.StartTime.Ticks);
if (reapeatedBlock != null)
{
PlaybackBlocks.Remove(reapeatedBlock);
PoolBlocks.Enqueue(reapeatedBlock);
}
}
// if there are no available blocks, make room!
if (PoolBlocks.Count <= 0)
{
var firstBlock = PlaybackBlocks[0];
PlaybackBlocks.RemoveAt(0);
PoolBlocks.Enqueue(firstBlock);
}
// Get a block reference from the pool and convert it!
var targetBlock = PoolBlocks.Dequeue();
container.Convert(source, ref targetBlock, true);
// Add the converted block to the playback list and sort it.
PlaybackBlocks.Add(targetBlock);
PlaybackBlocks.Sort();
return(targetBlock);
}
}
/// <summary>
/// Downloads the frame from the hardware into a software frame if possible.
/// The input hardware frame gets freed and the return value will point to the new software frame
/// </summary>
/// <param name="codecContext">The codec context.</param>
/// <param name="input">The input frame coming from the decoder (may or may not be hardware).</param>
/// <param name="isHardwareFrame">if set to <c>true</c> [comes from hardware] otherwise, hardware decoding was not perfomred.</param>
/// <returns>
/// The frame downloaded from the device into RAM
/// </returns>
/// <exception cref="Exception">Failed to transfer data to output frame</exception>
public AVFrame *ExchangeFrame(AVCodecContext *codecContext, AVFrame *input, out bool isHardwareFrame)
{
isHardwareFrame = false;
if (codecContext->hw_device_ctx == null)
{
return(input);
}
isHardwareFrame = true;
if (input->format != (int)PixelFormat)
{
return(input);
}
var output = MediaFrame.CreateAVFrame();
var result = ffmpeg.av_hwframe_transfer_data(output, input, 0);
ffmpeg.av_frame_copy_props(output, input);
if (result < 0)
{
MediaFrame.ReleaseAVFrame(output);
throw new MediaContainerException("Failed to transfer data to output frame");
}
MediaFrame.ReleaseAVFrame(input);
return(output);
}
/// <summary>
/// Creates a frame source object given the raw FFmpeg frame reference.
/// </summary>
/// <param name="framePointer">The raw FFmpeg frame pointer.</param>
/// <returns>The media frame</returns>
protected override unsafe MediaFrame CreateFrameSource(IntPtr framePointer)
{
// Validate the audio frame
var frame = (AVFrame *)framePointer;
if (frame == null || (*frame).extended_data == null || frame->channels <= 0 ||
frame->nb_samples <= 0 || frame->sample_rate <= 0)
{
return(null);
}
if (string.IsNullOrWhiteSpace(FilterString) == false)
{
InitializeFilterGraph(frame);
}
AVFrame *outputFrame = null;
// Filtergraph can be changed by issuing a ChangeMedia command
if (FilterGraph != null)
{
// Allocate the output frame
outputFrame = MediaFrame.CloneAVFrame(frame);
var result = ffmpeg.av_buffersrc_add_frame(SourceFilter, outputFrame);
while (result >= 0)
{
result = ffmpeg.av_buffersink_get_frame_flags(SinkFilter, outputFrame, 0);
}
if (outputFrame->nb_samples <= 0)
{
// If we don't have a valid output frame simply release it and
// return the original input frame
MediaFrame.ReleaseAVFrame(outputFrame);
outputFrame = frame;
}
else
{
// the output frame is the new valid frame (output frame).
// threfore, we need to release the original
MediaFrame.ReleaseAVFrame(frame);
}
}
else
{
outputFrame = frame;
}
// Check if the output frame is valid
if (outputFrame->nb_samples <= 0)
{
return(null);
}
var frameHolder = new AudioFrame(outputFrame, this);
return(frameHolder);
}
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param>
/// <returns>
/// Return the updated output frame
/// </returns>
/// <exception cref="ArgumentNullException">input cannot be null</exception>
public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings)
{
if (output == null)
{
output = new SubtitleBlock();
}
var source = input as SubtitleFrame;
var target = output as SubtitleBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Set the target data
target.EndTime = source.EndTime;
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.StreamIndex = input.StreamIndex;
target.OriginalText.Clear();
if (source.Text.Count > 0)
{
target.OriginalText.AddRange(source.Text);
}
target.OriginalTextType = source.TextType;
target.Text.Clear();
foreach (var text in source.Text)
{
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
if (source.TextType == AVSubtitleType.SUBTITLE_ASS)
{
var strippedText = StripAssFormat(text);
if (string.IsNullOrWhiteSpace(strippedText) == false)
{
target.Text.Add(strippedText);
}
}
else
{
var strippedText = StripSrtFormat(text);
if (string.IsNullOrWhiteSpace(strippedText) == false)
{
target.Text.Add(strippedText);
}
}
}
return(target);
}
/// <summary>
/// Adds a block to the playback blocks by converting the given frame.
/// If there are no more blocks in the pool, the oldest block is returned to the pool
/// and reused for the new block. The source frame is automatically disposed.
/// </summary>
/// <param name="source">The source.</param>
/// <param name="container">The container.</param>
/// <returns>The filled block.</returns>
public MediaBlock Add(MediaFrame source, MediaContainer container)
{
lock (SyncRoot)
{
// Check if we already have a block at the given time
if (IsInRange(source.StartTime))
{
var reapeatedBlock = PlaybackBlocks.FirstOrDefault(f => f.StartTime.Ticks == source.StartTime.Ticks);
if (reapeatedBlock != null)
{
PlaybackBlocks.Remove(reapeatedBlock);
PoolBlocks.Enqueue(reapeatedBlock);
}
}
// if there are no available blocks, make room!
if (PoolBlocks.Count <= 0)
{
var firstBlock = PlaybackBlocks[0];
PlaybackBlocks.RemoveAt(0);
PoolBlocks.Enqueue(firstBlock);
}
// Get a block reference from the pool and convert it!
var targetBlock = PoolBlocks.Dequeue();
container.Convert(source, ref targetBlock, PlaybackBlocks, true);
// Discard a frame with incorrect timing
if (targetBlock.IsStartTimeGuessed && IsMonotonic && PlaybackBlocks.Count > 1 &&
targetBlock.Duration != PlaybackBlocks.Last().Duration)
{
// return the converted block to the pool
PoolBlocks.Enqueue(targetBlock);
return(null);
}
else
{
// Add the converted block to the playback list and sort it.
PlaybackBlocks.Add(targetBlock);
PlaybackBlocks.Sort();
}
return(targetBlock);
}
}
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <returns>
/// Return the updated output frame
/// </returns>
/// <exception cref="System.ArgumentNullException">input</exception>
internal override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output)
{
if (output == null)
{
output = new SubtitleBlock();
}
var source = input as SubtitleFrame;
var target = output as SubtitleBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Set the target data
target.EndTime = source.EndTime;
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.Text.Clear();
target.Text.AddRange(source.Text);
return(target);
}
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param> /// <returns> /// Returns true of the operation succeeded. False otherwise. /// </returns> public abstract bool MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings);
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <param name="siblings">The siblings to help guess additional frame parameters.</param>
/// <returns>
/// Return the updated output frame
/// </returns>
/// <exception cref="System.ArgumentNullException">input</exception>
public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings)
{
if (output == null)
{
output = new VideoBlock();
}
var source = input as VideoFrame;
var target = output as VideoBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Retrieve a suitable scaler or create it on the fly
Scaler = ffmpeg.sws_getCachedContext(
Scaler,
source.Pointer->width,
source.Pointer->height,
NormalizePixelFormat(source.Pointer),
source.Pointer->width,
source.Pointer->height,
OutputPixelFormat,
ScalerFlags,
null,
null,
null);
RC.Current.Add(Scaler, $"311: {nameof(VideoComponent)}.{nameof(MaterializeFrame)}()");
// Perform scaling and save the data to our unmanaged buffer pointer
var targetBufferStride = ffmpeg.av_image_get_linesize(OutputPixelFormat, source.Pointer->width, 0);
var targetStride = new int[] { targetBufferStride };
var targetLength = ffmpeg.av_image_get_buffer_size(OutputPixelFormat, source.Pointer->width, source.Pointer->height, 1);
// Ensure proper allocation of the buffer
// If there is a size mismatch between the wanted buffer length and the existing one,
// then let's reallocate the buffer and set the new size (dispose of the existing one if any)
if (target.PictureBufferLength != targetLength)
{
if (target.PictureBuffer != IntPtr.Zero)
{
Marshal.FreeHGlobal(target.PictureBuffer);
}
target.PictureBufferLength = targetLength;
target.PictureBuffer = Marshal.AllocHGlobal(target.PictureBufferLength);
}
var targetScan = default(byte_ptrArray8);
targetScan[0] = (byte *)target.PictureBuffer;
// The scaling is done here
var outputHeight = ffmpeg.sws_scale(Scaler, source.Pointer->data, source.Pointer->linesize, 0, source.Pointer->height, targetScan, targetStride);
// Flag the block if we have to
target.IsStartTimeGuessed = source.HasValidStartTime == false;
// Try to fix the start time, duration and End time if we don't have valid data
if (source.HasValidStartTime == false && siblings != null && siblings.Count > 0)
{
// Get timing information from the last sibling
var lastSibling = siblings[siblings.Count - 1];
// We set the target properties
target.StartTime = lastSibling.EndTime;
target.Duration = source.Duration.Ticks > 0 ? source.Duration : lastSibling.Duration;
target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks);
}
else
{
// We set the target properties directly from the source
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.EndTime = source.EndTime;
}
target.StreamIndex = input.StreamIndex;
target.SmtpeTimecode = source.SmtpeTimecode;
target.DisplayPictureNumber = source.DisplayPictureNumber;
target.CodedPictureNumber = source.DisplayPictureNumber;
target.BufferStride = targetStride[0];
target.PixelHeight = source.Pointer->height;
target.PixelWidth = source.Pointer->width;
var aspectRatio = source.Pointer->sample_aspect_ratio;
if (aspectRatio.num == 0 || aspectRatio.den == 0)
{
target.AspectWidth = 1;
target.AspectHeight = 1;
}
else
{
target.AspectWidth = aspectRatio.num;
target.AspectHeight = aspectRatio.den;
}
return(target);
}
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <returns>Return the updated output frame</returns> internal abstract MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output);
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <returns>
/// Return the updated output frame
/// </returns>
/// <exception cref="System.ArgumentNullException">input</exception>
internal override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output)
{
if (output == null)
{
output = new AudioBlock();
}
var source = input as AudioFrame;
var target = output as AudioBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Create the source and target ausio specs. We might need to scale from
// the source to the target
var sourceSpec = AudioParams.CreateSource(source.Pointer);
var targetSpec = AudioParams.CreateTarget(source.Pointer);
// Initialize or update the audio scaler if required
if (Scaler == null || LastSourceSpec == null || AudioParams.AreCompatible(LastSourceSpec, sourceSpec) == false)
{
Scaler = ffmpeg.swr_alloc_set_opts(Scaler, targetSpec.ChannelLayout, targetSpec.Format, targetSpec.SampleRate,
sourceSpec.ChannelLayout, sourceSpec.Format, sourceSpec.SampleRate, 0, null);
RC.Current.Add(Scaler, $"109: {nameof(AudioComponent)}.{nameof(MaterializeFrame)}()");
ffmpeg.swr_init(Scaler);
LastSourceSpec = sourceSpec;
}
// Allocate the unmanaged output buffer
if (target.AudioBufferLength != targetSpec.BufferLength)
{
if (target.AudioBuffer != IntPtr.Zero)
{
Marshal.FreeHGlobal(target.AudioBuffer);
}
target.AudioBufferLength = targetSpec.BufferLength;
target.AudioBuffer = Marshal.AllocHGlobal(targetSpec.BufferLength);
}
var outputBufferPtr = (byte *)target.AudioBuffer;
// Execute the conversion (audio scaling). It will return the number of samples that were output
var outputSamplesPerChannel =
ffmpeg.swr_convert(Scaler, &outputBufferPtr, targetSpec.SamplesPerChannel,
source.Pointer->extended_data, source.Pointer->nb_samples);
// Compute the buffer length
var outputBufferLength =
ffmpeg.av_samples_get_buffer_size(null, targetSpec.ChannelCount, outputSamplesPerChannel, targetSpec.Format, 1);
// set the target properties
target.StartTime = source.StartTime;
target.EndTime = source.EndTime;
target.BufferLength = outputBufferLength;
target.ChannelCount = targetSpec.ChannelCount;
target.Duration = source.Duration;
target.SampleRate = targetSpec.SampleRate;
target.SamplesPerChannel = outputSamplesPerChannel;
return(target);
}
internal void RunQuickBuffering(MediaEngine m)
{
// We need to perform some packet reading and decoding
MediaFrame frame = null;
var main = MainMediaType;
var auxs = MediaTypes.Except(main);
var mediaTypes = MediaTypes;
var mainBlocks = m.Blocks[main];
// Read and decode blocks until the main component is half full
while (m.ShouldReadMorePackets)
{
// Read some packets
m.Container.Read();
// Decode frames and add the blocks
foreach (var t in mediaTypes)
{
frame = this[t].ReceiveNextFrame();
m.Blocks[t].Add(frame, m.Container);
}
// Check if we have at least a half a buffer on main
if (mainBlocks.CapacityPercent >= 0.5)
{
break;
}
}
// Check if we have a valid range. If not, just set it what the main component is dictating
if (mainBlocks.Count > 0 && mainBlocks.IsInRange(m.WallClock) == false)
{
m.ChangePosition(mainBlocks.RangeStartTime);
}
// Have the other components catch up
foreach (var t in auxs)
{
if (mainBlocks.Count <= 0)
{
break;
}
if (t != MediaType.Audio && t != MediaType.Video)
{
continue;
}
while (m.Blocks[t].RangeEndTime < mainBlocks.RangeEndTime)
{
if (m.ShouldReadMorePackets == false)
{
break;
}
// Read some packets
m.Container.Read();
// Decode frames and add the blocks
frame = this[t].ReceiveNextFrame();
m.Blocks[t].Add(frame, m.Container);
}
}
}
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <returns>
/// Return the updated output frame
/// </returns>
/// <exception cref="System.ArgumentNullException">input</exception>
internal override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output)
{
if (output == null)
{
output = new VideoBlock();
}
var source = input as VideoFrame;
var target = output as VideoBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Retrieve a suitable scaler or create it on the fly
Scaler = ffmpeg.sws_getCachedContext(Scaler,
source.Pointer->width, source.Pointer->height, GetPixelFormat(source.Pointer),
source.Pointer->width, source.Pointer->height,
OutputPixelFormat, ScalerFlags, null, null, null);
RC.Current.Add(Scaler, $"311: {nameof(VideoComponent)}.{nameof(MaterializeFrame)}()");
// Perform scaling and save the data to our unmanaged buffer pointer
var targetBufferStride = ffmpeg.av_image_get_linesize(OutputPixelFormat, source.Pointer->width, 0);
var targetStride = new int[] { targetBufferStride };
var targetLength = ffmpeg.av_image_get_buffer_size(OutputPixelFormat, source.Pointer->width, source.Pointer->height, 1);
// Ensure proper allocation of the buffer
// If there is a size mismatch between the wanted buffer length and the existing one,
// then let's reallocate the buffer and set the new size (dispose of the existing one if any)
if (target.PictureBufferLength != targetLength)
{
if (target.PictureBuffer != IntPtr.Zero)
{
Marshal.FreeHGlobal(target.PictureBuffer);
}
target.PictureBufferLength = targetLength;
target.PictureBuffer = Marshal.AllocHGlobal(target.PictureBufferLength);
}
var targetScan = new byte_ptrArray8();
targetScan[0] = (byte *)target.PictureBuffer;
// The scaling is done here
var outputHeight = ffmpeg.sws_scale(Scaler, source.Pointer->data, source.Pointer->linesize, 0, source.Pointer->height, targetScan, targetStride);
// We set the target properties
target.EndTime = source.EndTime;
target.StartTime = source.StartTime;
target.BufferStride = targetStride[0];
target.Duration = source.Duration;
target.PixelHeight = source.Pointer->height;
target.PixelWidth = source.Pointer->width;
var aspectRatio = source.Pointer->sample_aspect_ratio;
if (aspectRatio.num == 0 || aspectRatio.den == 0)
{
target.AspectWidth = 1;
target.AspectHeight = 1;
}
else
{
target.AspectWidth = aspectRatio.num;
target.AspectHeight = aspectRatio.den;
}
return(target);
}
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <param name="siblings">The siblings to help guess additional frame parameters.</param>
/// <returns>
/// Return the updated output frame
/// </returns>
/// <exception cref="ArgumentNullException">input</exception>
public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings)
{
if (output == null)
{
output = new VideoBlock();
}
var source = input as VideoFrame;
var target = output as VideoBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Retrieve a suitable scaler or create it on the fly
var newScaler = ffmpeg.sws_getCachedContext(
Scaler,
source.Pointer->width,
source.Pointer->height,
NormalizePixelFormat(source.Pointer),
source.Pointer->width,
source.Pointer->height,
Constants.Video.VideoPixelFormat,
ScalerFlags,
null,
null,
null);
// if it's the first time we set the scaler, simply assign it.
if (Scaler == null)
{
Scaler = newScaler;
RC.Current.Add(Scaler, $"311: {nameof(VideoComponent)}.{nameof(MaterializeFrame)}()");
}
// Reassign to the new scaler and remove the reference to the existing one
// The get cached context function automatically frees the existing scaler.
if (Scaler != newScaler)
{
RC.Current.Remove(Scaler);
Scaler = newScaler;
}
// Perform scaling and save the data to our unmanaged buffer pointer
target.EnsureAllocated(source, Constants.Video.VideoPixelFormat);
var targetStride = new int[] { target.PictureBufferStride };
var targetScan = default(byte_ptrArray8);
targetScan[0] = (byte *)target.PictureBuffer;
// The scaling is done here
var outputHeight = ffmpeg.sws_scale(
Scaler,
source.Pointer->data,
source.Pointer->linesize,
0,
source.Pointer->height,
targetScan,
targetStride);
// After scaling, we need to copy and guess some of the block properties
// Flag the block if we have to
target.IsStartTimeGuessed = source.HasValidStartTime == false;
// Try to fix the start time, duration and End time if we don't have valid data
if (source.HasValidStartTime == false && siblings != null && siblings.Count > 0)
{
// Get timing information from the last sibling
var lastSibling = siblings[siblings.Count - 1];
// We set the target properties
target.StartTime = lastSibling.EndTime;
target.Duration = source.Duration.Ticks > 0 ? source.Duration : lastSibling.Duration;
target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks);
// Guess picture number and SMTPE
var timeBase = ffmpeg.av_guess_frame_rate(Container.InputContext, Stream, source.Pointer);
target.DisplayPictureNumber = Extensions.ComputePictureNumber(target.StartTime, target.Duration, 1);
target.SmtpeTimecode = Extensions.ComputeSmtpeTimeCode(StartTimeOffset, target.Duration, timeBase, target.DisplayPictureNumber);
}
else
{
// We set the target properties directly from the source
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.EndTime = source.EndTime;
// Copy picture number and SMTPE
target.DisplayPictureNumber = source.DisplayPictureNumber;
target.SmtpeTimecode = source.SmtpeTimecode;
}
// Fill out other properties
target.CodedPictureNumber = source.CodedPictureNumber;
target.StreamIndex = source.StreamIndex;
target.ClosedCaptions = new ReadOnlyCollection <ClosedCaptions.ClosedCaptionPacket>(source.ClosedCaptions);
// Process the aspect ratio
var aspectRatio = source.Pointer->sample_aspect_ratio;
if (aspectRatio.num == 0 || aspectRatio.den == 0)
{
target.AspectWidth = 1;
target.AspectHeight = 1;
}
else
{
target.AspectWidth = aspectRatio.num;
target.AspectHeight = aspectRatio.den;
}
return(target);
}
/// <summary>
/// Receives 0 or more frames from the next available packet in the Queue.
/// This sends the first available packet to dequeue to the decoder
/// and uses the decoded frames (if any) to their corresponding
/// ProcessFrame method.
/// </summary>
/// <returns>The list of frames</returns>
private List <MediaFrame> DecodeNextPacketInternal()
{
var result = new List <MediaFrame>();
// Ensure there is at least one packet in the queue
if (PacketBufferCount <= 0)
{
return(result);
}
// Setup some initial state variables
var packet = Packets.Dequeue();
// The packets are alwasy sent. We dequeue them and keep a reference to them
// in the SentPackets queue
SentPackets.Push(packet);
var receiveFrameResult = 0;
if (MediaType == MediaType.Audio || MediaType == MediaType.Video)
{
// If it's audio or video, we use the new API and the decoded frames are stored in AVFrame
// Let us send the packet to the codec for decoding a frame of uncompressed data later.
// TODO: sendPacketResult is never checked for errors... We require some error handling.
// for example when using h264_qsv codec, this returns -40 (Function not implemented)
var sendPacketResult = ffmpeg.avcodec_send_packet(CodecContext, IsEmptyPacket(packet) ? null : packet);
// Let's check and see if we can get 1 or more frames from the packet we just sent to the decoder.
// Audio packets will typically contain 1 or more audioframes
// Video packets might require several packets to decode 1 frame
MediaFrame managedFrame = null;
while (receiveFrameResult == 0)
{
// Allocate a frame in unmanaged memory and
// Try to receive the decompressed frame data
var outputFrame = ffmpeg.av_frame_alloc();
RC.Current.Add(outputFrame, $"327: {nameof(MediaComponent)}[{MediaType}].{nameof(DecodeNextPacketInternal)}()");
receiveFrameResult = ffmpeg.avcodec_receive_frame(CodecContext, outputFrame);
try
{
managedFrame = null;
if (receiveFrameResult == 0)
{
// Send the frame to processing
managedFrame = CreateFrameSource(ref outputFrame);
if (managedFrame != null)
{
result.Add(managedFrame);
}
}
if (managedFrame == null)
{
RC.Current.Remove(outputFrame);
ffmpeg.av_frame_free(&outputFrame);
}
}
catch
{
// Release the frame as the decoded data could not be processed
RC.Current.Remove(outputFrame);
ffmpeg.av_frame_free(&outputFrame);
throw;
}
}
}
else if (MediaType == MediaType.Subtitle)
{
// Fors subtitles we use the old API (new API send_packet/receive_frame) is not yet available
var gotFrame = 0;
var outputFrame = SubtitleFrame.AllocateSubtitle();
receiveFrameResult = ffmpeg.avcodec_decode_subtitle2(CodecContext, outputFrame, &gotFrame, packet);
// Check if there is an error decoding the packet.
// If there is, remove the packet clear the sent packets
if (receiveFrameResult < 0)
{
SubtitleFrame.DeallocateSubtitle(outputFrame);
SentPackets.Clear();
Container.Parent?.Log(MediaLogMessageType.Error, $"{MediaType}: Error decoding. Error Code: {receiveFrameResult}");
}
else
{
// Process the first frame if we got it from the packet
// Note that there could be more frames (subtitles) in the packet
if (gotFrame != 0)
{
try
{
// Send the frame to processing
var managedFrame = CreateFrameSource(outputFrame);
if (managedFrame == null)
{
throw new MediaContainerException($"{MediaType} Component does not implement {nameof(CreateFrameSource)}");
}
result.Add(managedFrame);
}
catch
{
// Once processed, we don't need it anymore. Release it.
SubtitleFrame.DeallocateSubtitle(outputFrame);
throw;
}
}
// Let's check if we have more decoded frames from the same single packet
// Packets may contain more than 1 frame and the decoder is drained
// by passing an empty packet (data = null, size = 0)
while (gotFrame != 0 && receiveFrameResult > 0)
{
outputFrame = SubtitleFrame.AllocateSubtitle();
var emptyPacket = ffmpeg.av_packet_alloc();
RC.Current.Add(emptyPacket, $"406: {nameof(MediaComponent)}[{MediaType}].{nameof(DecodeNextPacketInternal)}()");
// Receive the frames in a loop
try
{
receiveFrameResult = ffmpeg.avcodec_decode_subtitle2(CodecContext, outputFrame, &gotFrame, emptyPacket);
if (gotFrame != 0 && receiveFrameResult > 0)
{
// Send the subtitle to processing
var managedFrame = CreateFrameSource(outputFrame);
if (managedFrame == null)
{
throw new MediaContainerException($"{MediaType} Component does not implement {nameof(CreateFrameSource)}");
}
result.Add(managedFrame);
}
}
catch
{
// once the subtitle is processed. Release it from memory
SubtitleFrame.DeallocateSubtitle(outputFrame);
throw;
}
finally
{
// free the empty packet
RC.Current.Remove(emptyPacket);
ffmpeg.av_packet_free(&emptyPacket);
}
}
}
}
// Release the sent packets if 1 or more frames were received in the packet
if (result.Count >= 1 || (Container.IsAtEndOfStream && IsEmptyPacket(packet) && PacketBufferCount == 0))
{
// We clear the sent packet queue (releasing packet from unmanaged memory also)
// because we got at least 1 frame from the packet.
SentPackets.Clear();
}
return(result);
}
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param>
/// <returns>
/// Returns true if successful. False otherwise
/// </returns>
/// <exception cref="ArgumentNullException">input cannot be null</exception>
public override bool MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings)
{
if (output == null)
{
output = new SubtitleBlock();
}
var source = input as SubtitleFrame;
var target = output as SubtitleBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Set the target data
target.EndTime = source.EndTime;
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.StreamIndex = input.StreamIndex;
// Process time offsets
if (Delay != TimeSpan.Zero)
{
target.StartTime = TimeSpan.FromTicks(target.StartTime.Ticks + Delay.Ticks);
target.EndTime = TimeSpan.FromTicks(target.EndTime.Ticks + Delay.Ticks);
target.Duration = TimeSpan.FromTicks(target.EndTime.Ticks - target.StartTime.Ticks);
}
target.OriginalText.Clear();
if (source.Text.Count > 0)
{
target.OriginalText.AddRange(source.Text);
}
target.OriginalTextType = source.TextType;
target.Text.Clear();
foreach (var text in source.Text)
{
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
if (source.TextType == AVSubtitleType.SUBTITLE_ASS)
{
var strippedText = StripAssFormat(text);
if (string.IsNullOrWhiteSpace(strippedText) == false)
{
target.Text.Add(strippedText);
}
}
else
{
var strippedText = StripSrtFormat(text);
if (string.IsNullOrWhiteSpace(strippedText) == false)
{
target.Text.Add(strippedText);
}
}
}
// TODO: CompressedSize is just an estimate.
// It would be better if we counted chars in all text lines.
target.CompressedSize = source.CompressedSize;
return(true);
}
/// <inheritdoc />
protected override MediaFrame CreateFrameSource(IntPtr framePointer)
{
// Validate the video frame
var frame = (AVFrame *)framePointer;
if (framePointer == IntPtr.Zero || frame->width <= 0 || frame->height <= 0)
{
return(null);
}
// Move the frame from hardware (GPU) memory to RAM (CPU)
if (HardwareAccelerator != null)
{
frame = HardwareAccelerator.ExchangeFrame(CodecContext, frame, out var isHardwareFrame);
IsUsingHardwareDecoding = isHardwareFrame;
}
// Init the filter graph for the frame
if (string.IsNullOrWhiteSpace(FilterString) == false)
{
InitializeFilterGraph(frame);
}
AVFrame *outputFrame;
// Changes in the filter graph can be applied by calling the ChangeMedia command
if (FilterGraph != null)
{
// Allocate the output frame
outputFrame = MediaFrame.CloneAVFrame(frame);
var result = ffmpeg.av_buffersrc_add_frame(SourceFilter, outputFrame);
while (result >= 0)
{
result = ffmpeg.av_buffersink_get_frame_flags(SinkFilter, outputFrame, 0);
}
if (outputFrame->width <= 0 || outputFrame->height <= 0)
{
// If we don't have a valid output frame simply release it and
// return the original input frame
MediaFrame.ReleaseAVFrame(outputFrame);
outputFrame = frame;
}
else
{
// the output frame is the new valid frame (output frame).
// therefore, we need to release the original
MediaFrame.ReleaseAVFrame(frame);
}
}
else
{
outputFrame = frame;
}
// Check if the output frame is valid
if (outputFrame->width <= 0 || outputFrame->height <= 0)
{
return(null);
}
// Create the frame holder object and return it.
return(new VideoFrame(outputFrame, this));
}
/// <inheritdoc />
public override bool MaterializeFrame(MediaFrame input, ref MediaBlock output, MediaBlock previousBlock)
{
if (output == null)
{
output = new VideoBlock();
}
if (input is VideoFrame == false || output is VideoBlock == false)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
var source = (VideoFrame)input;
var target = (VideoBlock)output;
// Retrieve a suitable scaler or create it on the fly
var newScaler = ffmpeg.sws_getCachedContext(
Scaler,
source.Pointer->width,
source.Pointer->height,
NormalizePixelFormat(source.Pointer),
source.Pointer->width,
source.Pointer->height,
Constants.Video.VideoPixelFormat,
ScalerFlags,
null,
null,
null);
// if it's the first time we set the scaler, simply assign it.
if (Scaler == null)
{
Scaler = newScaler;
RC.Current.Add(Scaler);
}
// Reassign to the new scaler and remove the reference to the existing one
// The get cached context function automatically frees the existing scaler.
if (Scaler != newScaler)
{
RC.Current.Remove(Scaler);
Scaler = newScaler;
}
// Perform scaling and save the data to our unmanaged buffer pointer
if (target.Allocate(source, Constants.Video.VideoPixelFormat) &&
target.TryAcquireWriterLock(out var writeLock))
{
using (writeLock)
{
var targetStride = new[] { target.PictureBufferStride };
var targetScan = default(byte_ptrArray8);
targetScan[0] = (byte *)target.Buffer;
// The scaling is done here
var outputHeight = ffmpeg.sws_scale(
Scaler,
source.Pointer->data,
source.Pointer->linesize,
0,
source.Pointer->height,
targetScan,
targetStride);
if (outputHeight <= 0)
{
return(false);
}
}
}
else
{
return(false);
}
// After scaling, we need to copy and guess some of the block properties
// Flag the block if we have to
target.IsStartTimeGuessed = source.HasValidStartTime == false;
// Try to fix the start time, duration and End time if we don't have valid data
if (source.HasValidStartTime == false && previousBlock != null)
{
// Get timing information from the previous block
target.StartTime = TimeSpan.FromTicks(previousBlock.EndTime.Ticks + 1);
target.Duration = source.Duration.Ticks > 0 ? source.Duration : previousBlock.Duration;
target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks);
// Guess picture number and SMTPE
var timeBase = ffmpeg.av_guess_frame_rate(Container.InputContext, Stream, source.Pointer);
target.DisplayPictureNumber = Extensions.ComputePictureNumber(target.StartTime, target.Duration, 1);
target.SmtpeTimeCode = Extensions.ComputeSmtpeTimeCode(StartTime, target.Duration, timeBase, target.DisplayPictureNumber);
}
else
{
// We set the target properties directly from the source
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.EndTime = source.EndTime;
// Copy picture number and SMTPE
target.DisplayPictureNumber = source.DisplayPictureNumber;
target.SmtpeTimeCode = source.SmtpeTimeCode;
}
// Fill out other properties
target.IsHardwareFrame = source.IsHardwareFrame;
target.HardwareAcceleratorName = source.HardwareAcceleratorName;
target.CompressedSize = source.CompressedSize;
target.CodedPictureNumber = source.CodedPictureNumber;
target.StreamIndex = source.StreamIndex;
target.ClosedCaptions = new ReadOnlyCollection <ClosedCaptionPacket>(source.ClosedCaptions);
// Update the stream info object if we get Closed Caption Data
if (StreamInfo.HasClosedCaptions == false && target.ClosedCaptions.Count > 0)
{
StreamInfo.HasClosedCaptions = true;
}
// Process the aspect ratio
var aspectRatio = ffmpeg.av_guess_sample_aspect_ratio(Container.InputContext, Stream, source.Pointer);
if (aspectRatio.num == 0 || aspectRatio.den == 0)
{
target.PixelAspectWidth = 1;
target.PixelAspectHeight = 1;
}
else
{
target.PixelAspectWidth = aspectRatio.num;
target.PixelAspectHeight = aspectRatio.den;
}
return(true);
}
/// <inheritdoc />
public override bool MaterializeFrame(MediaFrame input, ref MediaBlock output, MediaBlock previousBlock)
{
if (output == null)
{
output = new AudioBlock();
}
if (input is AudioFrame == false || output is AudioBlock == false)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
var source = (AudioFrame)input;
var target = (AudioBlock)output;
// Create the source and target audio specs. We might need to scale from
// the source to the target
var sourceSpec = FFAudioParams.CreateSource(source.Pointer);
var targetSpec = FFAudioParams.CreateTarget(source.Pointer);
// Initialize or update the audio scaler if required
if (Scaler == null || LastSourceSpec == null || FFAudioParams.AreCompatible(LastSourceSpec, sourceSpec) == false)
{
Scaler = ffmpeg.swr_alloc_set_opts(
Scaler,
targetSpec.ChannelLayout,
targetSpec.Format,
targetSpec.SampleRate,
sourceSpec.ChannelLayout,
sourceSpec.Format,
sourceSpec.SampleRate,
0,
null);
RC.Current.Add(Scaler);
ffmpeg.swr_init(Scaler);
LastSourceSpec = sourceSpec;
}
// Allocate the unmanaged output buffer and convert to stereo.
int outputSamplesPerChannel;
if (target.Allocate(targetSpec.BufferLength) &&
target.TryAcquireWriterLock(out var writeLock))
{
using (writeLock)
{
var outputBufferPtr = (byte *)target.Buffer;
// Execute the conversion (audio scaling). It will return the number of samples that were output
outputSamplesPerChannel = ffmpeg.swr_convert(
Scaler,
&outputBufferPtr,
targetSpec.SamplesPerChannel,
source.Pointer->extended_data,
source.Pointer->nb_samples);
}
}
else
{
return(false);
}
// Compute the buffer length
var outputBufferLength =
ffmpeg.av_samples_get_buffer_size(null, targetSpec.ChannelCount, outputSamplesPerChannel, targetSpec.Format, 1);
// Flag the block if we have to
target.IsStartTimeGuessed = source.HasValidStartTime == false;
// Try to fix the start time, duration and End time if we don't have valid data
if (source.HasValidStartTime == false && previousBlock != null)
{
// Get timing information from the previous block
target.StartTime = TimeSpan.FromTicks(previousBlock.EndTime.Ticks + 1);
target.Duration = source.Duration.Ticks > 0 ? source.Duration : previousBlock.Duration;
target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks);
}
else
{
// We set the target properties directly from the source
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.EndTime = source.EndTime;
}
target.CompressedSize = source.CompressedSize;
target.SamplesBufferLength = outputBufferLength;
target.ChannelCount = targetSpec.ChannelCount;
target.SampleRate = targetSpec.SampleRate;
target.SamplesPerChannel = outputSamplesPerChannel;
target.StreamIndex = input.StreamIndex;
return(true);
}
/// <summary>
/// Converts decoded, raw frame data in the frame source into a a usable frame. <br />
/// The process includes performing picture, samples or text conversions
/// so that the decoded source frame data is easily usable in multimedia applications
/// </summary>
/// <param name="input">The source frame to use as an input.</param>
/// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param>
/// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param>
/// <returns>
/// Return the updated output frame
/// </returns>
/// <exception cref="System.ArgumentNullException">input</exception>
public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings)
{
if (output == null)
{
output = new AudioBlock();
}
var source = input as AudioFrame;
var target = output as AudioBlock;
if (source == null || target == null)
{
throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'");
}
// Create the source and target ausio specs. We might need to scale from
// the source to the target
var sourceSpec = AudioParams.CreateSource(source.Pointer);
var targetSpec = AudioParams.CreateTarget(source.Pointer);
// Initialize or update the audio scaler if required
if (Scaler == null || LastSourceSpec == null || AudioParams.AreCompatible(LastSourceSpec, sourceSpec) == false)
{
Scaler = ffmpeg.swr_alloc_set_opts(
Scaler,
targetSpec.ChannelLayout,
targetSpec.Format,
targetSpec.SampleRate,
sourceSpec.ChannelLayout,
sourceSpec.Format,
sourceSpec.SampleRate,
0,
null);
RC.Current.Add(Scaler, $"109: {nameof(AudioComponent)}.{nameof(MaterializeFrame)}()");
ffmpeg.swr_init(Scaler);
LastSourceSpec = sourceSpec;
}
// Allocate the unmanaged output buffer
if (target.AudioBufferLength != targetSpec.BufferLength)
{
if (target.AudioBuffer != IntPtr.Zero)
{
Marshal.FreeHGlobal(target.AudioBuffer);
}
target.AudioBufferLength = targetSpec.BufferLength;
target.AudioBuffer = Marshal.AllocHGlobal(targetSpec.BufferLength);
}
var outputBufferPtr = (byte *)target.AudioBuffer;
// Execute the conversion (audio scaling). It will return the number of samples that were output
var outputSamplesPerChannel =
ffmpeg.swr_convert(
Scaler,
&outputBufferPtr,
targetSpec.SamplesPerChannel,
source.Pointer->extended_data,
source.Pointer->nb_samples);
// Compute the buffer length
var outputBufferLength =
ffmpeg.av_samples_get_buffer_size(null, targetSpec.ChannelCount, outputSamplesPerChannel, targetSpec.Format, 1);
// Flag the block if we have to
target.IsStartTimeGuessed = source.HasValidStartTime == false;
// Try to fix the start time, duration and End time if we don't have valid data
if (source.HasValidStartTime == false && siblings != null && siblings.Count > 0)
{
// Get timing information from the last sibling
var lastSibling = siblings[siblings.Count - 1];
// We set the target properties
target.StartTime = lastSibling.EndTime;
target.Duration = source.Duration.Ticks > 0 ? source.Duration : lastSibling.Duration;
target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks);
}
else
{
// We set the target properties directly from the source
target.StartTime = source.StartTime;
target.Duration = source.Duration;
target.EndTime = source.EndTime;
}
target.BufferLength = outputBufferLength;
target.ChannelCount = targetSpec.ChannelCount;
target.SampleRate = targetSpec.SampleRate;
target.SamplesPerChannel = outputSamplesPerChannel;
target.StreamIndex = input.StreamIndex;
return(target);
}
/// <summary>
/// Pushes the specified frame into the queue.
/// In other words, enqueues the frame.
/// </summary>
/// <param name="frame">The frame.</param>
public void Push(MediaFrame frame)
{
lock (SyncRoot)
Frames.Add(frame);
}
