/// <summary> /// Adds a block to the playback blocks by converting the given frame. /// If there are no more blocks in the pool, the oldest block is returned to the pool /// and reused for the new block. The source frame is automatically disposed. /// </summary> /// <param name="source">The source.</param> /// <param name="container">The container.</param> /// <returns>The filled block.</returns> public MediaBlock Add(MediaFrame source, MediaContainer container) { lock (SyncRoot) { // Check if we already have a block at the given time if (IsInRange(source.StartTime)) { var reapeatedBlock = PlaybackBlocks.FirstOrDefault(f => f.StartTime.Ticks == source.StartTime.Ticks); if (reapeatedBlock != null) { PlaybackBlocks.Remove(reapeatedBlock); PoolBlocks.Enqueue(reapeatedBlock); } } // if there are no available blocks, make room! if (PoolBlocks.Count <= 0) { var firstBlock = PlaybackBlocks[0]; PlaybackBlocks.RemoveAt(0); PoolBlocks.Enqueue(firstBlock); } // Get a block reference from the pool and convert it! var targetBlock = PoolBlocks.Dequeue(); container.Convert(source, ref targetBlock, true); // Add the converted block to the playback list and sort it. PlaybackBlocks.Add(targetBlock); PlaybackBlocks.Sort(); return(targetBlock); } }
/// <summary> /// Downloads the frame from the hardware into a software frame if possible. /// The input hardware frame gets freed and the return value will point to the new software frame /// </summary> /// <param name="codecContext">The codec context.</param> /// <param name="input">The input frame coming from the decoder (may or may not be hardware).</param> /// <param name="isHardwareFrame">if set to <c>true</c> [comes from hardware] otherwise, hardware decoding was not perfomred.</param> /// <returns> /// The frame downloaded from the device into RAM /// </returns> /// <exception cref="Exception">Failed to transfer data to output frame</exception> public AVFrame *ExchangeFrame(AVCodecContext *codecContext, AVFrame *input, out bool isHardwareFrame) { isHardwareFrame = false; if (codecContext->hw_device_ctx == null) { return(input); } isHardwareFrame = true; if (input->format != (int)PixelFormat) { return(input); } var output = MediaFrame.CreateAVFrame(); var result = ffmpeg.av_hwframe_transfer_data(output, input, 0); ffmpeg.av_frame_copy_props(output, input); if (result < 0) { MediaFrame.ReleaseAVFrame(output); throw new MediaContainerException("Failed to transfer data to output frame"); } MediaFrame.ReleaseAVFrame(input); return(output); }
/// <summary> /// Creates a frame source object given the raw FFmpeg frame reference. /// </summary> /// <param name="framePointer">The raw FFmpeg frame pointer.</param> /// <returns>The media frame</returns> protected override unsafe MediaFrame CreateFrameSource(IntPtr framePointer) { // Validate the audio frame var frame = (AVFrame *)framePointer; if (frame == null || (*frame).extended_data == null || frame->channels <= 0 || frame->nb_samples <= 0 || frame->sample_rate <= 0) { return(null); } if (string.IsNullOrWhiteSpace(FilterString) == false) { InitializeFilterGraph(frame); } AVFrame *outputFrame = null; // Filtergraph can be changed by issuing a ChangeMedia command if (FilterGraph != null) { // Allocate the output frame outputFrame = MediaFrame.CloneAVFrame(frame); var result = ffmpeg.av_buffersrc_add_frame(SourceFilter, outputFrame); while (result >= 0) { result = ffmpeg.av_buffersink_get_frame_flags(SinkFilter, outputFrame, 0); } if (outputFrame->nb_samples <= 0) { // If we don't have a valid output frame simply release it and // return the original input frame MediaFrame.ReleaseAVFrame(outputFrame); outputFrame = frame; } else { // the output frame is the new valid frame (output frame). // threfore, we need to release the original MediaFrame.ReleaseAVFrame(frame); } } else { outputFrame = frame; } // Check if the output frame is valid if (outputFrame->nb_samples <= 0) { return(null); } var frameHolder = new AudioFrame(outputFrame, this); return(frameHolder); }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param> /// <returns> /// Return the updated output frame /// </returns> /// <exception cref="ArgumentNullException">input cannot be null</exception> public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings) { if (output == null) { output = new SubtitleBlock(); } var source = input as SubtitleFrame; var target = output as SubtitleBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Set the target data target.EndTime = source.EndTime; target.StartTime = source.StartTime; target.Duration = source.Duration; target.StreamIndex = input.StreamIndex; target.OriginalText.Clear(); if (source.Text.Count > 0) { target.OriginalText.AddRange(source.Text); } target.OriginalTextType = source.TextType; target.Text.Clear(); foreach (var text in source.Text) { if (string.IsNullOrWhiteSpace(text)) { continue; } if (source.TextType == AVSubtitleType.SUBTITLE_ASS) { var strippedText = StripAssFormat(text); if (string.IsNullOrWhiteSpace(strippedText) == false) { target.Text.Add(strippedText); } } else { var strippedText = StripSrtFormat(text); if (string.IsNullOrWhiteSpace(strippedText) == false) { target.Text.Add(strippedText); } } } return(target); }
/// <summary> /// Adds a block to the playback blocks by converting the given frame. /// If there are no more blocks in the pool, the oldest block is returned to the pool /// and reused for the new block. The source frame is automatically disposed. /// </summary> /// <param name="source">The source.</param> /// <param name="container">The container.</param> /// <returns>The filled block.</returns> public MediaBlock Add(MediaFrame source, MediaContainer container) { lock (SyncRoot) { // Check if we already have a block at the given time if (IsInRange(source.StartTime)) { var reapeatedBlock = PlaybackBlocks.FirstOrDefault(f => f.StartTime.Ticks == source.StartTime.Ticks); if (reapeatedBlock != null) { PlaybackBlocks.Remove(reapeatedBlock); PoolBlocks.Enqueue(reapeatedBlock); } } // if there are no available blocks, make room! if (PoolBlocks.Count <= 0) { var firstBlock = PlaybackBlocks[0]; PlaybackBlocks.RemoveAt(0); PoolBlocks.Enqueue(firstBlock); } // Get a block reference from the pool and convert it! var targetBlock = PoolBlocks.Dequeue(); container.Convert(source, ref targetBlock, PlaybackBlocks, true); // Discard a frame with incorrect timing if (targetBlock.IsStartTimeGuessed && IsMonotonic && PlaybackBlocks.Count > 1 && targetBlock.Duration != PlaybackBlocks.Last().Duration) { // return the converted block to the pool PoolBlocks.Enqueue(targetBlock); return(null); } else { // Add the converted block to the playback list and sort it. PlaybackBlocks.Add(targetBlock); PlaybackBlocks.Sort(); } return(targetBlock); } }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <returns> /// Return the updated output frame /// </returns> /// <exception cref="System.ArgumentNullException">input</exception> internal override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output) { if (output == null) { output = new SubtitleBlock(); } var source = input as SubtitleFrame; var target = output as SubtitleBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Set the target data target.EndTime = source.EndTime; target.StartTime = source.StartTime; target.Duration = source.Duration; target.Text.Clear(); target.Text.AddRange(source.Text); return(target); }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param> /// <returns> /// Returns true of the operation succeeded. False otherwise. /// </returns> public abstract bool MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings);
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <param name="siblings">The siblings to help guess additional frame parameters.</param> /// <returns> /// Return the updated output frame /// </returns> /// <exception cref="System.ArgumentNullException">input</exception> public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings) { if (output == null) { output = new VideoBlock(); } var source = input as VideoFrame; var target = output as VideoBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Retrieve a suitable scaler or create it on the fly Scaler = ffmpeg.sws_getCachedContext( Scaler, source.Pointer->width, source.Pointer->height, NormalizePixelFormat(source.Pointer), source.Pointer->width, source.Pointer->height, OutputPixelFormat, ScalerFlags, null, null, null); RC.Current.Add(Scaler, $"311: {nameof(VideoComponent)}.{nameof(MaterializeFrame)}()"); // Perform scaling and save the data to our unmanaged buffer pointer var targetBufferStride = ffmpeg.av_image_get_linesize(OutputPixelFormat, source.Pointer->width, 0); var targetStride = new int[] { targetBufferStride }; var targetLength = ffmpeg.av_image_get_buffer_size(OutputPixelFormat, source.Pointer->width, source.Pointer->height, 1); // Ensure proper allocation of the buffer // If there is a size mismatch between the wanted buffer length and the existing one, // then let's reallocate the buffer and set the new size (dispose of the existing one if any) if (target.PictureBufferLength != targetLength) { if (target.PictureBuffer != IntPtr.Zero) { Marshal.FreeHGlobal(target.PictureBuffer); } target.PictureBufferLength = targetLength; target.PictureBuffer = Marshal.AllocHGlobal(target.PictureBufferLength); } var targetScan = default(byte_ptrArray8); targetScan[0] = (byte *)target.PictureBuffer; // The scaling is done here var outputHeight = ffmpeg.sws_scale(Scaler, source.Pointer->data, source.Pointer->linesize, 0, source.Pointer->height, targetScan, targetStride); // Flag the block if we have to target.IsStartTimeGuessed = source.HasValidStartTime == false; // Try to fix the start time, duration and End time if we don't have valid data if (source.HasValidStartTime == false && siblings != null && siblings.Count > 0) { // Get timing information from the last sibling var lastSibling = siblings[siblings.Count - 1]; // We set the target properties target.StartTime = lastSibling.EndTime; target.Duration = source.Duration.Ticks > 0 ? source.Duration : lastSibling.Duration; target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks); } else { // We set the target properties directly from the source target.StartTime = source.StartTime; target.Duration = source.Duration; target.EndTime = source.EndTime; } target.StreamIndex = input.StreamIndex; target.SmtpeTimecode = source.SmtpeTimecode; target.DisplayPictureNumber = source.DisplayPictureNumber; target.CodedPictureNumber = source.DisplayPictureNumber; target.BufferStride = targetStride[0]; target.PixelHeight = source.Pointer->height; target.PixelWidth = source.Pointer->width; var aspectRatio = source.Pointer->sample_aspect_ratio; if (aspectRatio.num == 0 || aspectRatio.den == 0) { target.AspectWidth = 1; target.AspectHeight = 1; } else { target.AspectWidth = aspectRatio.num; target.AspectHeight = aspectRatio.den; } return(target); }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <returns>Return the updated output frame</returns> internal abstract MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output);
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <returns> /// Return the updated output frame /// </returns> /// <exception cref="System.ArgumentNullException">input</exception> internal override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output) { if (output == null) { output = new AudioBlock(); } var source = input as AudioFrame; var target = output as AudioBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Create the source and target ausio specs. We might need to scale from // the source to the target var sourceSpec = AudioParams.CreateSource(source.Pointer); var targetSpec = AudioParams.CreateTarget(source.Pointer); // Initialize or update the audio scaler if required if (Scaler == null || LastSourceSpec == null || AudioParams.AreCompatible(LastSourceSpec, sourceSpec) == false) { Scaler = ffmpeg.swr_alloc_set_opts(Scaler, targetSpec.ChannelLayout, targetSpec.Format, targetSpec.SampleRate, sourceSpec.ChannelLayout, sourceSpec.Format, sourceSpec.SampleRate, 0, null); RC.Current.Add(Scaler, $"109: {nameof(AudioComponent)}.{nameof(MaterializeFrame)}()"); ffmpeg.swr_init(Scaler); LastSourceSpec = sourceSpec; } // Allocate the unmanaged output buffer if (target.AudioBufferLength != targetSpec.BufferLength) { if (target.AudioBuffer != IntPtr.Zero) { Marshal.FreeHGlobal(target.AudioBuffer); } target.AudioBufferLength = targetSpec.BufferLength; target.AudioBuffer = Marshal.AllocHGlobal(targetSpec.BufferLength); } var outputBufferPtr = (byte *)target.AudioBuffer; // Execute the conversion (audio scaling). It will return the number of samples that were output var outputSamplesPerChannel = ffmpeg.swr_convert(Scaler, &outputBufferPtr, targetSpec.SamplesPerChannel, source.Pointer->extended_data, source.Pointer->nb_samples); // Compute the buffer length var outputBufferLength = ffmpeg.av_samples_get_buffer_size(null, targetSpec.ChannelCount, outputSamplesPerChannel, targetSpec.Format, 1); // set the target properties target.StartTime = source.StartTime; target.EndTime = source.EndTime; target.BufferLength = outputBufferLength; target.ChannelCount = targetSpec.ChannelCount; target.Duration = source.Duration; target.SampleRate = targetSpec.SampleRate; target.SamplesPerChannel = outputSamplesPerChannel; return(target); }
internal void RunQuickBuffering(MediaEngine m) { // We need to perform some packet reading and decoding MediaFrame frame = null; var main = MainMediaType; var auxs = MediaTypes.Except(main); var mediaTypes = MediaTypes; var mainBlocks = m.Blocks[main]; // Read and decode blocks until the main component is half full while (m.ShouldReadMorePackets) { // Read some packets m.Container.Read(); // Decode frames and add the blocks foreach (var t in mediaTypes) { frame = this[t].ReceiveNextFrame(); m.Blocks[t].Add(frame, m.Container); } // Check if we have at least a half a buffer on main if (mainBlocks.CapacityPercent >= 0.5) { break; } } // Check if we have a valid range. If not, just set it what the main component is dictating if (mainBlocks.Count > 0 && mainBlocks.IsInRange(m.WallClock) == false) { m.ChangePosition(mainBlocks.RangeStartTime); } // Have the other components catch up foreach (var t in auxs) { if (mainBlocks.Count <= 0) { break; } if (t != MediaType.Audio && t != MediaType.Video) { continue; } while (m.Blocks[t].RangeEndTime < mainBlocks.RangeEndTime) { if (m.ShouldReadMorePackets == false) { break; } // Read some packets m.Container.Read(); // Decode frames and add the blocks frame = this[t].ReceiveNextFrame(); m.Blocks[t].Add(frame, m.Container); } } }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <returns> /// Return the updated output frame /// </returns> /// <exception cref="System.ArgumentNullException">input</exception> internal override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output) { if (output == null) { output = new VideoBlock(); } var source = input as VideoFrame; var target = output as VideoBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Retrieve a suitable scaler or create it on the fly Scaler = ffmpeg.sws_getCachedContext(Scaler, source.Pointer->width, source.Pointer->height, GetPixelFormat(source.Pointer), source.Pointer->width, source.Pointer->height, OutputPixelFormat, ScalerFlags, null, null, null); RC.Current.Add(Scaler, $"311: {nameof(VideoComponent)}.{nameof(MaterializeFrame)}()"); // Perform scaling and save the data to our unmanaged buffer pointer var targetBufferStride = ffmpeg.av_image_get_linesize(OutputPixelFormat, source.Pointer->width, 0); var targetStride = new int[] { targetBufferStride }; var targetLength = ffmpeg.av_image_get_buffer_size(OutputPixelFormat, source.Pointer->width, source.Pointer->height, 1); // Ensure proper allocation of the buffer // If there is a size mismatch between the wanted buffer length and the existing one, // then let's reallocate the buffer and set the new size (dispose of the existing one if any) if (target.PictureBufferLength != targetLength) { if (target.PictureBuffer != IntPtr.Zero) { Marshal.FreeHGlobal(target.PictureBuffer); } target.PictureBufferLength = targetLength; target.PictureBuffer = Marshal.AllocHGlobal(target.PictureBufferLength); } var targetScan = new byte_ptrArray8(); targetScan[0] = (byte *)target.PictureBuffer; // The scaling is done here var outputHeight = ffmpeg.sws_scale(Scaler, source.Pointer->data, source.Pointer->linesize, 0, source.Pointer->height, targetScan, targetStride); // We set the target properties target.EndTime = source.EndTime; target.StartTime = source.StartTime; target.BufferStride = targetStride[0]; target.Duration = source.Duration; target.PixelHeight = source.Pointer->height; target.PixelWidth = source.Pointer->width; var aspectRatio = source.Pointer->sample_aspect_ratio; if (aspectRatio.num == 0 || aspectRatio.den == 0) { target.AspectWidth = 1; target.AspectHeight = 1; } else { target.AspectWidth = aspectRatio.num; target.AspectHeight = aspectRatio.den; } return(target); }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <param name="siblings">The siblings to help guess additional frame parameters.</param> /// <returns> /// Return the updated output frame /// </returns> /// <exception cref="ArgumentNullException">input</exception> public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings) { if (output == null) { output = new VideoBlock(); } var source = input as VideoFrame; var target = output as VideoBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Retrieve a suitable scaler or create it on the fly var newScaler = ffmpeg.sws_getCachedContext( Scaler, source.Pointer->width, source.Pointer->height, NormalizePixelFormat(source.Pointer), source.Pointer->width, source.Pointer->height, Constants.Video.VideoPixelFormat, ScalerFlags, null, null, null); // if it's the first time we set the scaler, simply assign it. if (Scaler == null) { Scaler = newScaler; RC.Current.Add(Scaler, $"311: {nameof(VideoComponent)}.{nameof(MaterializeFrame)}()"); } // Reassign to the new scaler and remove the reference to the existing one // The get cached context function automatically frees the existing scaler. if (Scaler != newScaler) { RC.Current.Remove(Scaler); Scaler = newScaler; } // Perform scaling and save the data to our unmanaged buffer pointer target.EnsureAllocated(source, Constants.Video.VideoPixelFormat); var targetStride = new int[] { target.PictureBufferStride }; var targetScan = default(byte_ptrArray8); targetScan[0] = (byte *)target.PictureBuffer; // The scaling is done here var outputHeight = ffmpeg.sws_scale( Scaler, source.Pointer->data, source.Pointer->linesize, 0, source.Pointer->height, targetScan, targetStride); // After scaling, we need to copy and guess some of the block properties // Flag the block if we have to target.IsStartTimeGuessed = source.HasValidStartTime == false; // Try to fix the start time, duration and End time if we don't have valid data if (source.HasValidStartTime == false && siblings != null && siblings.Count > 0) { // Get timing information from the last sibling var lastSibling = siblings[siblings.Count - 1]; // We set the target properties target.StartTime = lastSibling.EndTime; target.Duration = source.Duration.Ticks > 0 ? source.Duration : lastSibling.Duration; target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks); // Guess picture number and SMTPE var timeBase = ffmpeg.av_guess_frame_rate(Container.InputContext, Stream, source.Pointer); target.DisplayPictureNumber = Extensions.ComputePictureNumber(target.StartTime, target.Duration, 1); target.SmtpeTimecode = Extensions.ComputeSmtpeTimeCode(StartTimeOffset, target.Duration, timeBase, target.DisplayPictureNumber); } else { // We set the target properties directly from the source target.StartTime = source.StartTime; target.Duration = source.Duration; target.EndTime = source.EndTime; // Copy picture number and SMTPE target.DisplayPictureNumber = source.DisplayPictureNumber; target.SmtpeTimecode = source.SmtpeTimecode; } // Fill out other properties target.CodedPictureNumber = source.CodedPictureNumber; target.StreamIndex = source.StreamIndex; target.ClosedCaptions = new ReadOnlyCollection <ClosedCaptions.ClosedCaptionPacket>(source.ClosedCaptions); // Process the aspect ratio var aspectRatio = source.Pointer->sample_aspect_ratio; if (aspectRatio.num == 0 || aspectRatio.den == 0) { target.AspectWidth = 1; target.AspectHeight = 1; } else { target.AspectWidth = aspectRatio.num; target.AspectHeight = aspectRatio.den; } return(target); }
/// <summary> /// Receives 0 or more frames from the next available packet in the Queue. /// This sends the first available packet to dequeue to the decoder /// and uses the decoded frames (if any) to their corresponding /// ProcessFrame method. /// </summary> /// <returns>The list of frames</returns> private List <MediaFrame> DecodeNextPacketInternal() { var result = new List <MediaFrame>(); // Ensure there is at least one packet in the queue if (PacketBufferCount <= 0) { return(result); } // Setup some initial state variables var packet = Packets.Dequeue(); // The packets are alwasy sent. We dequeue them and keep a reference to them // in the SentPackets queue SentPackets.Push(packet); var receiveFrameResult = 0; if (MediaType == MediaType.Audio || MediaType == MediaType.Video) { // If it's audio or video, we use the new API and the decoded frames are stored in AVFrame // Let us send the packet to the codec for decoding a frame of uncompressed data later. // TODO: sendPacketResult is never checked for errors... We require some error handling. // for example when using h264_qsv codec, this returns -40 (Function not implemented) var sendPacketResult = ffmpeg.avcodec_send_packet(CodecContext, IsEmptyPacket(packet) ? null : packet); // Let's check and see if we can get 1 or more frames from the packet we just sent to the decoder. // Audio packets will typically contain 1 or more audioframes // Video packets might require several packets to decode 1 frame MediaFrame managedFrame = null; while (receiveFrameResult == 0) { // Allocate a frame in unmanaged memory and // Try to receive the decompressed frame data var outputFrame = ffmpeg.av_frame_alloc(); RC.Current.Add(outputFrame, $"327: {nameof(MediaComponent)}[{MediaType}].{nameof(DecodeNextPacketInternal)}()"); receiveFrameResult = ffmpeg.avcodec_receive_frame(CodecContext, outputFrame); try { managedFrame = null; if (receiveFrameResult == 0) { // Send the frame to processing managedFrame = CreateFrameSource(ref outputFrame); if (managedFrame != null) { result.Add(managedFrame); } } if (managedFrame == null) { RC.Current.Remove(outputFrame); ffmpeg.av_frame_free(&outputFrame); } } catch { // Release the frame as the decoded data could not be processed RC.Current.Remove(outputFrame); ffmpeg.av_frame_free(&outputFrame); throw; } } } else if (MediaType == MediaType.Subtitle) { // Fors subtitles we use the old API (new API send_packet/receive_frame) is not yet available var gotFrame = 0; var outputFrame = SubtitleFrame.AllocateSubtitle(); receiveFrameResult = ffmpeg.avcodec_decode_subtitle2(CodecContext, outputFrame, &gotFrame, packet); // Check if there is an error decoding the packet. // If there is, remove the packet clear the sent packets if (receiveFrameResult < 0) { SubtitleFrame.DeallocateSubtitle(outputFrame); SentPackets.Clear(); Container.Parent?.Log(MediaLogMessageType.Error, $"{MediaType}: Error decoding. Error Code: {receiveFrameResult}"); } else { // Process the first frame if we got it from the packet // Note that there could be more frames (subtitles) in the packet if (gotFrame != 0) { try { // Send the frame to processing var managedFrame = CreateFrameSource(outputFrame); if (managedFrame == null) { throw new MediaContainerException($"{MediaType} Component does not implement {nameof(CreateFrameSource)}"); } result.Add(managedFrame); } catch { // Once processed, we don't need it anymore. Release it. SubtitleFrame.DeallocateSubtitle(outputFrame); throw; } } // Let's check if we have more decoded frames from the same single packet // Packets may contain more than 1 frame and the decoder is drained // by passing an empty packet (data = null, size = 0) while (gotFrame != 0 && receiveFrameResult > 0) { outputFrame = SubtitleFrame.AllocateSubtitle(); var emptyPacket = ffmpeg.av_packet_alloc(); RC.Current.Add(emptyPacket, $"406: {nameof(MediaComponent)}[{MediaType}].{nameof(DecodeNextPacketInternal)}()"); // Receive the frames in a loop try { receiveFrameResult = ffmpeg.avcodec_decode_subtitle2(CodecContext, outputFrame, &gotFrame, emptyPacket); if (gotFrame != 0 && receiveFrameResult > 0) { // Send the subtitle to processing var managedFrame = CreateFrameSource(outputFrame); if (managedFrame == null) { throw new MediaContainerException($"{MediaType} Component does not implement {nameof(CreateFrameSource)}"); } result.Add(managedFrame); } } catch { // once the subtitle is processed. Release it from memory SubtitleFrame.DeallocateSubtitle(outputFrame); throw; } finally { // free the empty packet RC.Current.Remove(emptyPacket); ffmpeg.av_packet_free(&emptyPacket); } } } } // Release the sent packets if 1 or more frames were received in the packet if (result.Count >= 1 || (Container.IsAtEndOfStream && IsEmptyPacket(packet) && PacketBufferCount == 0)) { // We clear the sent packet queue (releasing packet from unmanaged memory also) // because we got at least 1 frame from the packet. SentPackets.Clear(); } return(result); }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param> /// <returns> /// Returns true if successful. False otherwise /// </returns> /// <exception cref="ArgumentNullException">input cannot be null</exception> public override bool MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings) { if (output == null) { output = new SubtitleBlock(); } var source = input as SubtitleFrame; var target = output as SubtitleBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Set the target data target.EndTime = source.EndTime; target.StartTime = source.StartTime; target.Duration = source.Duration; target.StreamIndex = input.StreamIndex; // Process time offsets if (Delay != TimeSpan.Zero) { target.StartTime = TimeSpan.FromTicks(target.StartTime.Ticks + Delay.Ticks); target.EndTime = TimeSpan.FromTicks(target.EndTime.Ticks + Delay.Ticks); target.Duration = TimeSpan.FromTicks(target.EndTime.Ticks - target.StartTime.Ticks); } target.OriginalText.Clear(); if (source.Text.Count > 0) { target.OriginalText.AddRange(source.Text); } target.OriginalTextType = source.TextType; target.Text.Clear(); foreach (var text in source.Text) { if (string.IsNullOrWhiteSpace(text)) { continue; } if (source.TextType == AVSubtitleType.SUBTITLE_ASS) { var strippedText = StripAssFormat(text); if (string.IsNullOrWhiteSpace(strippedText) == false) { target.Text.Add(strippedText); } } else { var strippedText = StripSrtFormat(text); if (string.IsNullOrWhiteSpace(strippedText) == false) { target.Text.Add(strippedText); } } } // TODO: CompressedSize is just an estimate. // It would be better if we counted chars in all text lines. target.CompressedSize = source.CompressedSize; return(true); }
/// <inheritdoc /> protected override MediaFrame CreateFrameSource(IntPtr framePointer) { // Validate the video frame var frame = (AVFrame *)framePointer; if (framePointer == IntPtr.Zero || frame->width <= 0 || frame->height <= 0) { return(null); } // Move the frame from hardware (GPU) memory to RAM (CPU) if (HardwareAccelerator != null) { frame = HardwareAccelerator.ExchangeFrame(CodecContext, frame, out var isHardwareFrame); IsUsingHardwareDecoding = isHardwareFrame; } // Init the filter graph for the frame if (string.IsNullOrWhiteSpace(FilterString) == false) { InitializeFilterGraph(frame); } AVFrame *outputFrame; // Changes in the filter graph can be applied by calling the ChangeMedia command if (FilterGraph != null) { // Allocate the output frame outputFrame = MediaFrame.CloneAVFrame(frame); var result = ffmpeg.av_buffersrc_add_frame(SourceFilter, outputFrame); while (result >= 0) { result = ffmpeg.av_buffersink_get_frame_flags(SinkFilter, outputFrame, 0); } if (outputFrame->width <= 0 || outputFrame->height <= 0) { // If we don't have a valid output frame simply release it and // return the original input frame MediaFrame.ReleaseAVFrame(outputFrame); outputFrame = frame; } else { // the output frame is the new valid frame (output frame). // therefore, we need to release the original MediaFrame.ReleaseAVFrame(frame); } } else { outputFrame = frame; } // Check if the output frame is valid if (outputFrame->width <= 0 || outputFrame->height <= 0) { return(null); } // Create the frame holder object and return it. return(new VideoFrame(outputFrame, this)); }
/// <inheritdoc /> public override bool MaterializeFrame(MediaFrame input, ref MediaBlock output, MediaBlock previousBlock) { if (output == null) { output = new VideoBlock(); } if (input is VideoFrame == false || output is VideoBlock == false) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } var source = (VideoFrame)input; var target = (VideoBlock)output; // Retrieve a suitable scaler or create it on the fly var newScaler = ffmpeg.sws_getCachedContext( Scaler, source.Pointer->width, source.Pointer->height, NormalizePixelFormat(source.Pointer), source.Pointer->width, source.Pointer->height, Constants.Video.VideoPixelFormat, ScalerFlags, null, null, null); // if it's the first time we set the scaler, simply assign it. if (Scaler == null) { Scaler = newScaler; RC.Current.Add(Scaler); } // Reassign to the new scaler and remove the reference to the existing one // The get cached context function automatically frees the existing scaler. if (Scaler != newScaler) { RC.Current.Remove(Scaler); Scaler = newScaler; } // Perform scaling and save the data to our unmanaged buffer pointer if (target.Allocate(source, Constants.Video.VideoPixelFormat) && target.TryAcquireWriterLock(out var writeLock)) { using (writeLock) { var targetStride = new[] { target.PictureBufferStride }; var targetScan = default(byte_ptrArray8); targetScan[0] = (byte *)target.Buffer; // The scaling is done here var outputHeight = ffmpeg.sws_scale( Scaler, source.Pointer->data, source.Pointer->linesize, 0, source.Pointer->height, targetScan, targetStride); if (outputHeight <= 0) { return(false); } } } else { return(false); } // After scaling, we need to copy and guess some of the block properties // Flag the block if we have to target.IsStartTimeGuessed = source.HasValidStartTime == false; // Try to fix the start time, duration and End time if we don't have valid data if (source.HasValidStartTime == false && previousBlock != null) { // Get timing information from the previous block target.StartTime = TimeSpan.FromTicks(previousBlock.EndTime.Ticks + 1); target.Duration = source.Duration.Ticks > 0 ? source.Duration : previousBlock.Duration; target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks); // Guess picture number and SMTPE var timeBase = ffmpeg.av_guess_frame_rate(Container.InputContext, Stream, source.Pointer); target.DisplayPictureNumber = Extensions.ComputePictureNumber(target.StartTime, target.Duration, 1); target.SmtpeTimeCode = Extensions.ComputeSmtpeTimeCode(StartTime, target.Duration, timeBase, target.DisplayPictureNumber); } else { // We set the target properties directly from the source target.StartTime = source.StartTime; target.Duration = source.Duration; target.EndTime = source.EndTime; // Copy picture number and SMTPE target.DisplayPictureNumber = source.DisplayPictureNumber; target.SmtpeTimeCode = source.SmtpeTimeCode; } // Fill out other properties target.IsHardwareFrame = source.IsHardwareFrame; target.HardwareAcceleratorName = source.HardwareAcceleratorName; target.CompressedSize = source.CompressedSize; target.CodedPictureNumber = source.CodedPictureNumber; target.StreamIndex = source.StreamIndex; target.ClosedCaptions = new ReadOnlyCollection <ClosedCaptionPacket>(source.ClosedCaptions); // Update the stream info object if we get Closed Caption Data if (StreamInfo.HasClosedCaptions == false && target.ClosedCaptions.Count > 0) { StreamInfo.HasClosedCaptions = true; } // Process the aspect ratio var aspectRatio = ffmpeg.av_guess_sample_aspect_ratio(Container.InputContext, Stream, source.Pointer); if (aspectRatio.num == 0 || aspectRatio.den == 0) { target.PixelAspectWidth = 1; target.PixelAspectHeight = 1; } else { target.PixelAspectWidth = aspectRatio.num; target.PixelAspectHeight = aspectRatio.den; } return(true); }
/// <inheritdoc /> public override bool MaterializeFrame(MediaFrame input, ref MediaBlock output, MediaBlock previousBlock) { if (output == null) { output = new AudioBlock(); } if (input is AudioFrame == false || output is AudioBlock == false) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } var source = (AudioFrame)input; var target = (AudioBlock)output; // Create the source and target audio specs. We might need to scale from // the source to the target var sourceSpec = FFAudioParams.CreateSource(source.Pointer); var targetSpec = FFAudioParams.CreateTarget(source.Pointer); // Initialize or update the audio scaler if required if (Scaler == null || LastSourceSpec == null || FFAudioParams.AreCompatible(LastSourceSpec, sourceSpec) == false) { Scaler = ffmpeg.swr_alloc_set_opts( Scaler, targetSpec.ChannelLayout, targetSpec.Format, targetSpec.SampleRate, sourceSpec.ChannelLayout, sourceSpec.Format, sourceSpec.SampleRate, 0, null); RC.Current.Add(Scaler); ffmpeg.swr_init(Scaler); LastSourceSpec = sourceSpec; } // Allocate the unmanaged output buffer and convert to stereo. int outputSamplesPerChannel; if (target.Allocate(targetSpec.BufferLength) && target.TryAcquireWriterLock(out var writeLock)) { using (writeLock) { var outputBufferPtr = (byte *)target.Buffer; // Execute the conversion (audio scaling). It will return the number of samples that were output outputSamplesPerChannel = ffmpeg.swr_convert( Scaler, &outputBufferPtr, targetSpec.SamplesPerChannel, source.Pointer->extended_data, source.Pointer->nb_samples); } } else { return(false); } // Compute the buffer length var outputBufferLength = ffmpeg.av_samples_get_buffer_size(null, targetSpec.ChannelCount, outputSamplesPerChannel, targetSpec.Format, 1); // Flag the block if we have to target.IsStartTimeGuessed = source.HasValidStartTime == false; // Try to fix the start time, duration and End time if we don't have valid data if (source.HasValidStartTime == false && previousBlock != null) { // Get timing information from the previous block target.StartTime = TimeSpan.FromTicks(previousBlock.EndTime.Ticks + 1); target.Duration = source.Duration.Ticks > 0 ? source.Duration : previousBlock.Duration; target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks); } else { // We set the target properties directly from the source target.StartTime = source.StartTime; target.Duration = source.Duration; target.EndTime = source.EndTime; } target.CompressedSize = source.CompressedSize; target.SamplesBufferLength = outputBufferLength; target.ChannelCount = targetSpec.ChannelCount; target.SampleRate = targetSpec.SampleRate; target.SamplesPerChannel = outputSamplesPerChannel; target.StreamIndex = input.StreamIndex; return(true); }
/// <summary> /// Converts decoded, raw frame data in the frame source into a a usable frame. <br /> /// The process includes performing picture, samples or text conversions /// so that the decoded source frame data is easily usable in multimedia applications /// </summary> /// <param name="input">The source frame to use as an input.</param> /// <param name="output">The target frame that will be updated with the source frame. If null is passed the frame will be instantiated.</param> /// <param name="siblings">The sibling blocks that may help guess some additional parameters for the input frame.</param> /// <returns> /// Return the updated output frame /// </returns> /// <exception cref="System.ArgumentNullException">input</exception> public override MediaBlock MaterializeFrame(MediaFrame input, ref MediaBlock output, List <MediaBlock> siblings) { if (output == null) { output = new AudioBlock(); } var source = input as AudioFrame; var target = output as AudioBlock; if (source == null || target == null) { throw new ArgumentNullException($"{nameof(input)} and {nameof(output)} are either null or not of a compatible media type '{MediaType}'"); } // Create the source and target ausio specs. We might need to scale from // the source to the target var sourceSpec = AudioParams.CreateSource(source.Pointer); var targetSpec = AudioParams.CreateTarget(source.Pointer); // Initialize or update the audio scaler if required if (Scaler == null || LastSourceSpec == null || AudioParams.AreCompatible(LastSourceSpec, sourceSpec) == false) { Scaler = ffmpeg.swr_alloc_set_opts( Scaler, targetSpec.ChannelLayout, targetSpec.Format, targetSpec.SampleRate, sourceSpec.ChannelLayout, sourceSpec.Format, sourceSpec.SampleRate, 0, null); RC.Current.Add(Scaler, $"109: {nameof(AudioComponent)}.{nameof(MaterializeFrame)}()"); ffmpeg.swr_init(Scaler); LastSourceSpec = sourceSpec; } // Allocate the unmanaged output buffer if (target.AudioBufferLength != targetSpec.BufferLength) { if (target.AudioBuffer != IntPtr.Zero) { Marshal.FreeHGlobal(target.AudioBuffer); } target.AudioBufferLength = targetSpec.BufferLength; target.AudioBuffer = Marshal.AllocHGlobal(targetSpec.BufferLength); } var outputBufferPtr = (byte *)target.AudioBuffer; // Execute the conversion (audio scaling). It will return the number of samples that were output var outputSamplesPerChannel = ffmpeg.swr_convert( Scaler, &outputBufferPtr, targetSpec.SamplesPerChannel, source.Pointer->extended_data, source.Pointer->nb_samples); // Compute the buffer length var outputBufferLength = ffmpeg.av_samples_get_buffer_size(null, targetSpec.ChannelCount, outputSamplesPerChannel, targetSpec.Format, 1); // Flag the block if we have to target.IsStartTimeGuessed = source.HasValidStartTime == false; // Try to fix the start time, duration and End time if we don't have valid data if (source.HasValidStartTime == false && siblings != null && siblings.Count > 0) { // Get timing information from the last sibling var lastSibling = siblings[siblings.Count - 1]; // We set the target properties target.StartTime = lastSibling.EndTime; target.Duration = source.Duration.Ticks > 0 ? source.Duration : lastSibling.Duration; target.EndTime = TimeSpan.FromTicks(target.StartTime.Ticks + target.Duration.Ticks); } else { // We set the target properties directly from the source target.StartTime = source.StartTime; target.Duration = source.Duration; target.EndTime = source.EndTime; } target.BufferLength = outputBufferLength; target.ChannelCount = targetSpec.ChannelCount; target.SampleRate = targetSpec.SampleRate; target.SamplesPerChannel = outputSamplesPerChannel; target.StreamIndex = input.StreamIndex; return(target); }
/// <summary> /// Pushes the specified frame into the queue. /// In other words, enqueues the frame. /// </summary> /// <param name="frame">The frame.</param> public void Push(MediaFrame frame) { lock (SyncRoot) Frames.Add(frame); }