/// <summary>
/// Initializes a new instance of the <see cref="OptionMeta"/> class.
/// </summary>
/// <param name="option">The option.</param>
internal unsafe OptionMeta(AVOption *option)
{
OptionType = option->type;
Flags = option->flags;
HelpText = FFInterop.PtrToStringUTF8(option->help);
Name = FFInterop.PtrToStringUTF8(option->name);
Min = option->min;
Max = option->max;
// Default values
// DefaultString = FFInterop.PtrToStringUTF8(option->default_val.str); // TODO: This throws a memory violation for some reason
DefaultDouble = option->default_val.dbl;
DefaultLong = option->default_val.i64;
DefaultRational = option->default_val.q;
// Flag Parsing
IsAudioOption = (option->flags & ffmpeg.AV_OPT_FLAG_AUDIO_PARAM) > 0;
IsBsfOption = (option->flags & ffmpeg.AV_OPT_FLAG_BSF_PARAM) > 0;
IsDecodingOption = (option->flags & ffmpeg.AV_OPT_FLAG_DECODING_PARAM) > 0;
IsEncodingOption = (option->flags & ffmpeg.AV_OPT_FLAG_ENCODING_PARAM) > 0;
IsExported = (option->flags & ffmpeg.AV_OPT_FLAG_EXPORT) > 0;
IsFilteringOption = (option->flags & ffmpeg.AV_OPT_FLAG_FILTERING_PARAM) > 0;
IsReadonly = (option->flags & ffmpeg.AV_OPT_FLAG_READONLY) > 0;
IsSubtitleOption = (option->flags & ffmpeg.AV_OPT_FLAG_SUBTITLE_PARAM) > 0;
IsVideoOption = (option->flags & ffmpeg.AV_OPT_FLAG_VIDEO_PARAM) > 0;
}
/// <summary>
/// Gets the available hardware decoder codecs for the given codec id (codec family).
/// </summary>
/// <param name="codecFamily">The codec family.</param>
/// <returns>A list of hardware-enabled decoder codec names</returns>
private static List <string> GetHardwareDecoders(AVCodecID codecFamily)
{
var result = new List <string>(16);
foreach (var c in MediaEngine.AllCodecs)
{
if (ffmpeg.av_codec_is_decoder(c) == 0)
{
continue;
}
if (c->id != codecFamily)
{
continue;
}
if ((c->capabilities & ffmpeg.AV_CODEC_CAP_HARDWARE) != 0 ||
(c->capabilities & ffmpeg.AV_CODEC_CAP_HYBRID) != 0)
{
result.Add(FFInterop.PtrToStringUTF8(c->name));
}
}
return(result);
}
/// <summary>
/// Initializes a new instance of the <see cref="SubtitleFrame" /> class.
/// </summary>
/// <param name="frame">The frame.</param>
/// <param name="component">The component.</param>
internal SubtitleFrame(AVSubtitle *frame, MediaComponent component)
: base(frame, component)
{
m_Pointer = (AVSubtitle *)InternalPointer;
// Extract timing information (pts for Subtitles is always in AV_TIME_BASE units)
HasValidStartTime = frame->pts != ffmpeg.AV_NOPTS_VALUE;
var timeOffset = TimeSpan.FromTicks(frame->pts.ToTimeSpan(ffmpeg.AV_TIME_BASE).Ticks - component.Container.MediaStartTimeOffset.Ticks);
// start_display_time and end_display_time are relative to timeOffset
StartTime = TimeSpan.FromTicks(timeOffset.Ticks + Convert.ToInt64(frame->start_display_time).ToTimeSpan(StreamTimeBase).Ticks);
EndTime = TimeSpan.FromTicks(timeOffset.Ticks + Convert.ToInt64(frame->end_display_time).ToTimeSpan(StreamTimeBase).Ticks);
Duration = TimeSpan.FromTicks(EndTime.Ticks - StartTime.Ticks);
// Extract text strings
TextType = AVSubtitleType.SUBTITLE_NONE;
for (var i = 0; i < frame->num_rects; i++)
{
var rect = frame->rects[i];
if (rect->type == AVSubtitleType.SUBTITLE_TEXT)
{
if (rect->text != null)
{
Text.Add(FFInterop.PtrToStringUTF8(rect->text));
TextType = AVSubtitleType.SUBTITLE_TEXT;
break;
}
}
else if (rect->type == AVSubtitleType.SUBTITLE_ASS)
{
if (rect->ass != null)
{
Text.Add(FFInterop.PtrToStringUTF8(rect->ass));
TextType = AVSubtitleType.SUBTITLE_ASS;
break;
}
}
else
{
TextType = rect->type;
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SubtitleFrame" /> class.
/// </summary>
/// <param name="frame">The frame.</param>
/// <param name="component">The component.</param>
internal SubtitleFrame(AVSubtitle *frame, MediaComponent component)
: base(frame, component)
{
// Extract timing information (pts for Subtitles is always in AV_TIME_BASE units)
HasValidStartTime = frame->pts != ffmpeg.AV_NOPTS_VALUE;
var mainOffset = component.Container.Components.Main.StartTime;
var timeOffset = TimeSpan.FromTicks(frame->pts.ToTimeSpan(ffmpeg.AV_TIME_BASE).Ticks - mainOffset.Ticks);
// start_display_time and end_display_time are relative to timeOffset
StartTime = TimeSpan.FromMilliseconds(timeOffset.TotalMilliseconds + frame->start_display_time);
EndTime = TimeSpan.FromMilliseconds(timeOffset.TotalMilliseconds + frame->end_display_time);
Duration = TimeSpan.FromMilliseconds(frame->end_display_time - frame->start_display_time);
// Extract text strings
TextType = AVSubtitleType.SUBTITLE_NONE;
for (var i = 0; i < frame->num_rects; i++)
{
var rect = frame->rects[i];
if (rect->type == AVSubtitleType.SUBTITLE_TEXT)
{
if (rect->text == null)
{
continue;
}
Text.Add(FFInterop.PtrToStringUTF8(rect->text));
TextType = AVSubtitleType.SUBTITLE_TEXT;
break;
}
if (rect->type == AVSubtitleType.SUBTITLE_ASS)
{
if (rect->ass == null)
{
continue;
}
Text.Add(FFInterop.PtrToStringUTF8(rect->ass));
TextType = AVSubtitleType.SUBTITLE_ASS;
break;
}
TextType = rect->type;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="MediaInfo"/> class.
/// </summary>
/// <param name="container">The container.</param>
internal MediaInfo(MediaContainer container)
{
// The below logic was implemented using the same ideas conveyed by the following code:
// Reference: https://ffmpeg.org/doxygen/3.2/dump_8c_source.html --
var ic = container.InputContext;
InputUrl = container.MediaUrl;
Format = FFInterop.PtrToStringUTF8(ic->iformat->name);
Metadata = container.Metadata;
StartTime = ic->start_time != ffmpeg.AV_NOPTS_VALUE ? ic->start_time.ToTimeSpan() : TimeSpan.MinValue;
Duration = ic->duration != ffmpeg.AV_NOPTS_VALUE ? ic->duration.ToTimeSpan() : TimeSpan.MinValue;
BitRate = ic->bit_rate < 0 ? 0 : ic->bit_rate;
Streams = new ReadOnlyDictionary <int, StreamInfo>(ExtractStreams(ic).ToDictionary(k => k.StreamIndex, v => v));
Chapters = new ReadOnlyCollection <ChapterInfo>(ExtractChapters(ic));
Programs = new ReadOnlyCollection <ProgramInfo>(ExtractPrograms(ic, Streams));
BestStreams = new ReadOnlyDictionary <AVMediaType, StreamInfo>(FindBestStreams(ic, Streams));
}
/// <summary>
/// Initializes a new instance of the <see cref="MediaComponent"/> class.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="streamIndex">Index of the stream.</param>
/// <exception cref="ArgumentNullException">container</exception>
/// <exception cref="MediaContainerException">The container exception.</exception>
protected MediaComponent(MediaContainer container, int streamIndex)
{
// Parted from: https://github.com/FFmpeg/FFmpeg/blob/master/fftools/ffplay.c#L2559
// avctx = avcodec_alloc_context3(NULL);
Container = container ?? throw new ArgumentNullException(nameof(container));
CodecContext = ffmpeg.avcodec_alloc_context3(null);
RC.Current.Add(CodecContext, $"134: {nameof(MediaComponent)}[{MediaType}].ctor()");
StreamIndex = streamIndex;
Stream = container.InputContext->streams[StreamIndex];
StreamInfo = container.MediaInfo.Streams[StreamIndex];
// Set default codec context options from probed stream
var setCodecParamsResult = ffmpeg.avcodec_parameters_to_context(CodecContext, Stream->codecpar);
if (setCodecParamsResult < 0)
{
Container.Parent?.Log(MediaLogMessageType.Warning, $"Could not set codec parameters. Error code: {setCodecParamsResult}");
}
// We set the packet timebase in the same timebase as the stream as opposed to the tpyical AV_TIME_BASE
if (this is VideoComponent && Container.MediaOptions.VideoForcedFps > 0)
{
var fpsRational = ffmpeg.av_d2q(Container.MediaOptions.VideoForcedFps, 1000000);
Stream->r_frame_rate = fpsRational;
CodecContext->pkt_timebase = new AVRational {
num = fpsRational.den, den = fpsRational.num
};
}
else
{
CodecContext->pkt_timebase = Stream->time_base;
}
// Find the default decoder codec from the stream and set it.
var defaultCodec = ffmpeg.avcodec_find_decoder(Stream->codec->codec_id);
AVCodec *forcedCodec = null;
// If set, change the codec to the forced codec.
if (Container.MediaOptions.DecoderCodec.ContainsKey(StreamIndex) &&
string.IsNullOrWhiteSpace(Container.MediaOptions.DecoderCodec[StreamIndex]) == false)
{
var forcedCodecName = Container.MediaOptions.DecoderCodec[StreamIndex];
forcedCodec = ffmpeg.avcodec_find_decoder_by_name(forcedCodecName);
if (forcedCodec == null)
{
Container.Parent?.Log(MediaLogMessageType.Warning,
$"COMP {MediaType.ToString().ToUpperInvariant()}: Unable to set decoder codec to '{forcedCodecName}' on stream index {StreamIndex}");
}
}
// Check we have a valid codec to open and process the stream.
if (defaultCodec == null && forcedCodec == null)
{
var errorMessage = $"Fatal error. Unable to find suitable decoder for {Stream->codec->codec_id.ToString()}";
CloseComponent();
throw new MediaContainerException(errorMessage);
}
var codecCandidates = new AVCodec *[] { forcedCodec, defaultCodec };
AVCodec *selectedCodec = null;
var codecOpenResult = 0;
foreach (var codec in codecCandidates)
{
if (codec == null)
{
continue;
}
// Pass default codec stuff to the codec contect
CodecContext->codec_id = codec->id;
if ((codec->capabilities & ffmpeg.AV_CODEC_CAP_TRUNCATED) != 0)
{
CodecContext->flags |= ffmpeg.AV_CODEC_FLAG_TRUNCATED;
}
if ((codec->capabilities & ffmpeg.AV_CODEC_FLAG2_CHUNKS) != 0)
{
CodecContext->flags |= ffmpeg.AV_CODEC_FLAG2_CHUNKS;
}
// Process the decoder options
{
var decoderOptions = Container.MediaOptions.DecoderParams;
// Configure the codec context flags
if (decoderOptions.EnableFastDecoding)
{
CodecContext->flags2 |= ffmpeg.AV_CODEC_FLAG2_FAST;
}
if (decoderOptions.EnableLowDelay)
{
CodecContext->flags |= ffmpeg.AV_CODEC_FLAG_LOW_DELAY;
}
// process the low res option
if (decoderOptions.EnableLowRes && codec->max_lowres > 0)
{
decoderOptions.LowResIndex = codec->max_lowres.ToString(CultureInfo.InvariantCulture);
}
// Ensure ref counted frames for audio and video decoding
if (CodecContext->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO || CodecContext->codec_type == AVMediaType.AVMEDIA_TYPE_AUDIO)
{
decoderOptions.RefCountedFrames = "1";
}
}
// Setup additional settings. The most important one is Threads -- Setting it to 1 decoding is very slow. Setting it to auto
// decoding is very fast in most scenarios.
var codecOptions = Container.MediaOptions.DecoderParams.GetStreamCodecOptions(Stream->index);
// Enable Hardware acceleration if requested
if (this is VideoComponent && container.MediaOptions.VideoHardwareDevice != null)
{
HardwareAccelerator.Attach(this as VideoComponent, container.MediaOptions.VideoHardwareDevice);
}
// Open the CodecContext. This requires exclusive FFmpeg access
lock (CodecOpenLock)
{
fixed(AVDictionary **codecOptionsRef = &codecOptions.Pointer)
codecOpenResult = ffmpeg.avcodec_open2(CodecContext, codec, codecOptionsRef);
}
// Check if the codec opened successfully
if (codecOpenResult < 0)
{
Container.Parent?.Log(MediaLogMessageType.Warning,
$"Unable to open codec '{FFInterop.PtrToStringUTF8(codec->name)}' on stream {streamIndex}");
continue;
}
// If there are any codec options left over from passing them, it means they were not consumed
var currentEntry = codecOptions.First();
while (currentEntry != null && currentEntry?.Key != null)
{
Container.Parent?.Log(MediaLogMessageType.Warning,
$"Invalid codec option: '{currentEntry.Key}' for codec '{FFInterop.PtrToStringUTF8(codec->name)}', stream {streamIndex}");
currentEntry = codecOptions.Next(currentEntry);
}
selectedCodec = codec;
break;
}
if (selectedCodec == null)
{
CloseComponent();
throw new MediaContainerException($"Unable to find suitable decoder codec for stream {streamIndex}. Error code {codecOpenResult}");
}
// Startup done. Set some options.
Stream->discard = AVDiscard.AVDISCARD_DEFAULT;
MediaType = (MediaType)CodecContext->codec_type;
// Compute the start time
if (Stream->start_time == ffmpeg.AV_NOPTS_VALUE)
{
StartTimeOffset = Container.MediaStartTimeOffset;
}
else
{
StartTimeOffset = Stream->start_time.ToTimeSpan(Stream->time_base);
}
// compute the duration
if (Stream->duration == ffmpeg.AV_NOPTS_VALUE || Stream->duration == 0)
{
Duration = Container.InputContext->duration.ToTimeSpan();
}
else
{
Duration = Stream->duration.ToTimeSpan(Stream->time_base);
}
CodecId = Stream->codec->codec_id;
CodecName = FFInterop.PtrToStringUTF8(selectedCodec->name);
Bitrate = Stream->codec->bit_rate < 0 ? 0 : Convert.ToUInt64(Stream->codec->bit_rate);
Container.Parent?.Log(MediaLogMessageType.Debug,
$"COMP {MediaType.ToString().ToUpperInvariant()}: Start Offset: {StartTimeOffset.Format()}; Duration: {Duration.Format()}");
}
/// <summary>
/// Performs the actions that this command implements.
/// </summary>
internal override void ExecuteInternal()
{
var m = Manager.MediaCore;
if (m.IsDisposed || m.State.IsOpen || m.State.IsOpening)
{
return;
}
try
{
// TODO: Sometimes when the stream can't be read, the sample player stays as if it were trying to open
// until the interrupt timeout occurs but and the Real-Time Clock continues. Strange behavior. Investigate more.
// Signal the initial state
m.State.ResetMediaProperties();
m.State.Source = Source;
m.State.IsOpening = true;
// Register FFmpeg libraries if not already done
if (FFInterop.Initialize(MediaEngine.FFmpegDirectory, MediaEngine.FFmpegLoadModeFlags))
{
// Set the folders and lib identifiers
MediaEngine.FFmpegDirectory = FFInterop.LibrariesPath;
MediaEngine.FFmpegLoadModeFlags = FFInterop.LibraryIdentifiers;
// Log an init message
m.Log(MediaLogMessageType.Info,
$"{nameof(FFInterop)}.{nameof(FFInterop.Initialize)}: FFmpeg v{ffmpeg.av_version_info()}");
}
// Create the stream container
// the async protocol prefix allows for increased performance for local files.
var streamOptions = new StreamOptions();
// Convert the URI object to something the Media Container understands
var mediaUrl = Source.ToString();
try
{
if (Source.IsFile || Source.IsUnc)
{
// Set the default protocol Prefix
mediaUrl = Source.LocalPath;
streamOptions.ProtocolPrefix = "async";
}
}
catch { }
// GDIGRAB: Example URI: device://gdigrab?desktop
if (string.IsNullOrWhiteSpace(Source.Scheme) == false &&
(Source.Scheme.Equals("format") || Source.Scheme.Equals("device")) &&
string.IsNullOrWhiteSpace(Source.Host) == false &&
string.IsNullOrWhiteSpace(streamOptions.Input.ForcedInputFormat) &&
string.IsNullOrWhiteSpace(Source.Query) == false)
{
// Update the Input format and container input URL
// It is also possible to set some input options as follows:
// streamOptions.Input.Add(StreamInputOptions.Names.FrameRate, "20");
streamOptions.Input.ForcedInputFormat = Source.Host;
mediaUrl = Uri.UnescapeDataString(Source.Query).TrimStart('?');
m.Log(MediaLogMessageType.Info, $"Media URI will be updated. Input Format: {Source.Host}, Input Argument: {mediaUrl}");
}
// Allow the stream input options to be changed
m.SendOnMediaInitializing(streamOptions, mediaUrl);
// Instantiate the internal container
m.Container = new MediaContainer(mediaUrl, streamOptions, m);
// Reset buffering properties
m.State.InitializeBufferingProperties();
// Notify the user media is opening and allow for media options to be modified
// Stuff like audio and video filters and stream selection can be performed here.
m.SendOnMediaOpening();
// Notify Media will start opening
m.Log(MediaLogMessageType.Debug, $"{nameof(OpenCommand)}: Entered");
// Side-load subtitles if requested
m.PreloadSubtitles();
// Get the main container open
m.Container.Open();
// Check if we have at least audio or video here
if (m.Container.Components.HasAudio == false && m.Container.Components.HasVideo == false)
{
throw new MediaContainerException($"Unable to initialize at least one audio or video component fron the input stream.");
}
// Charge! We are good to go, fire up the worker threads!
m.StartWorkers();
// Set the state to stopped and exit the IsOpening state
m.State.IsOpening = false;
m.State.UpdateMediaState(PlaybackStatus.Stop);
// Raise the opened event
m.SendOnMediaOpened();
}
catch (Exception ex)
{
try { m.Container?.Dispose(); } catch { }
m.DisposePreloadedSubtitles();
m.Container = null;
m.State.UpdateMediaState(PlaybackStatus.Close);
m.SendOnMediaFailed(ex);
}
finally
{
// Signal we are no longer in the opening state
// so we can enqueue commands in the command handler
m.State.IsOpening = false;
m.Log(MediaLogMessageType.Debug, $"{nameof(OpenCommand)}: Completed");
}
}
private void InitializeFilterGraph(AVFrame *frame)
{
// References: https://www.ffmpeg.org/doxygen/2.0/doc_2examples_2filtering_audio_8c-example.html
const string SourceFilterName = "abuffer";
const string SourceFilterInstance = "audio_buffer";
const string SinkFilterName = "abuffersink";
const string SinkFilterInstance = "audio_buffersink";
// Get a snapshot of the FilterString
var filterString = FilterString;
// For empty filter strings ensure filtegraph is destroyed
if (string.IsNullOrWhiteSpace(filterString))
{
DestroyFilterGraph();
return;
}
// Recreate the filtergraph if we have to
if (filterString != AppliedFilterString)
{
DestroyFilterGraph();
}
// Ensure the filtergraph is compatible with the frame
var filterArguments = ComputeFilterArguments(frame);
if (filterArguments != CurrentFilterArguments)
{
DestroyFilterGraph();
}
else
{
return;
}
FilterGraph = ffmpeg.avfilter_graph_alloc();
RC.Current.Add(FilterGraph);
try
{
AVFilterContext *sourceFilterRef = null;
AVFilterContext *sinkFilterRef = null;
var result = ffmpeg.avfilter_graph_create_filter(
&sourceFilterRef, ffmpeg.avfilter_get_by_name(SourceFilterName), SourceFilterInstance, filterArguments, null, FilterGraph);
if (result != 0)
{
throw new MediaContainerException(
$"{nameof(ffmpeg.avfilter_graph_create_filter)} ({SourceFilterInstance}) failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
result = ffmpeg.avfilter_graph_create_filter(
&sinkFilterRef, ffmpeg.avfilter_get_by_name(SinkFilterName), SinkFilterInstance, null, null, FilterGraph);
if (result != 0)
{
throw new MediaContainerException(
$"{nameof(ffmpeg.avfilter_graph_create_filter)} ({SinkFilterInstance}) failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
SourceFilter = sourceFilterRef;
SinkFilter = sinkFilterRef;
if (string.IsNullOrWhiteSpace(filterString))
{
result = ffmpeg.avfilter_link(SourceFilter, 0, SinkFilter, 0);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_link)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
else
{
var initFilterCount = FilterGraph->nb_filters;
SourceOutput = ffmpeg.avfilter_inout_alloc();
SourceOutput->name = ffmpeg.av_strdup("in");
SourceOutput->filter_ctx = SourceFilter;
SourceOutput->pad_idx = 0;
SourceOutput->next = null;
SinkInput = ffmpeg.avfilter_inout_alloc();
SinkInput->name = ffmpeg.av_strdup("out");
SinkInput->filter_ctx = SinkFilter;
SinkInput->pad_idx = 0;
SinkInput->next = null;
result = ffmpeg.avfilter_graph_parse(FilterGraph, filterString, SinkInput, SourceOutput, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_parse)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
// Reorder the filters to ensure that inputs of the custom filters are merged first
for (var i = 0; i < FilterGraph->nb_filters - initFilterCount; i++)
{
var sourceAddress = FilterGraph->filters[i];
var targetAddress = FilterGraph->filters[i + initFilterCount];
FilterGraph->filters[i] = targetAddress;
FilterGraph->filters[i + initFilterCount] = sourceAddress;
}
}
result = ffmpeg.avfilter_graph_config(FilterGraph, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_config)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
catch (Exception ex)
{
this.LogError(Aspects.Component, $"Audio filter graph could not be built: {filterString}.", ex);
DestroyFilterGraph();
}
finally
{
CurrentFilterArguments = filterArguments;
AppliedFilterString = filterString;
}
}
/// <summary>
/// If necessary, disposes the existing filtergraph and creates a new one based on the frame arguments.
/// </summary>
/// <param name="frame">The frame.</param>
/// <exception cref="MediaContainerException">
/// avfilter_graph_create_filter
/// or
/// avfilter_graph_create_filter
/// or
/// avfilter_link
/// or
/// avfilter_graph_parse
/// or
/// avfilter_graph_config
/// </exception>
private void InitializeFilterGraph(AVFrame *frame)
{
/*
* References:
* https://www.ffmpeg.org/doxygen/2.0/doc_2examples_2filtering_audio_8c-example.html
*/
var frameArguments = ComputeFilterArguments(frame);
if (string.IsNullOrWhiteSpace(CurrentFilterArguments) || frameArguments.Equals(CurrentFilterArguments) == false)
{
DestroyFiltergraph();
}
else
{
return;
}
FilterGraph = ffmpeg.avfilter_graph_alloc();
RC.Current.Add(FilterGraph, $"264: {nameof(AudioComponent)}.{nameof(InitializeFilterGraph)}()");
CurrentFilterArguments = frameArguments;
try
{
var result = 0;
fixed(AVFilterContext **source = &SourceFilter)
fixed(AVFilterContext **sink = &SinkFilter)
{
result = ffmpeg.avfilter_graph_create_filter(source, ffmpeg.avfilter_get_by_name("abuffer"), "audio_buffer", CurrentFilterArguments, null, FilterGraph);
if (result != 0)
{
throw new MediaContainerException(
$"{nameof(ffmpeg.avfilter_graph_create_filter)} (audio_buffer) failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
result = ffmpeg.avfilter_graph_create_filter(sink, ffmpeg.avfilter_get_by_name("abuffersink"), "audio_buffersink", null, null, FilterGraph);
if (result != 0)
{
throw new MediaContainerException(
$"{nameof(ffmpeg.avfilter_graph_create_filter)} (audio_buffersink) failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
if (string.IsNullOrWhiteSpace(FilterString))
{
result = ffmpeg.avfilter_link(SourceFilter, 0, SinkFilter, 0);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_link)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
else
{
var initFilterCount = FilterGraph->nb_filters;
SourceOutput = ffmpeg.avfilter_inout_alloc();
SourceOutput->name = ffmpeg.av_strdup("in");
SourceOutput->filter_ctx = SourceFilter;
SourceOutput->pad_idx = 0;
SourceOutput->next = null;
SinkInput = ffmpeg.avfilter_inout_alloc();
SinkInput->name = ffmpeg.av_strdup("out");
SinkInput->filter_ctx = SinkFilter;
SinkInput->pad_idx = 0;
SinkInput->next = null;
result = ffmpeg.avfilter_graph_parse(FilterGraph, FilterString, SinkInput, SourceOutput, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_parse)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
// Reorder the filters to ensure that inputs of the custom filters are merged first
for (var i = 0; i < FilterGraph->nb_filters - initFilterCount; i++)
{
var sourceAddress = FilterGraph->filters[i];
var targetAddress = FilterGraph->filters[i + initFilterCount];
FilterGraph->filters[i] = targetAddress;
FilterGraph->filters[i + initFilterCount] = sourceAddress;
}
}
result = ffmpeg.avfilter_graph_config(FilterGraph, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_config)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
catch (Exception ex)
{
Container.Parent?.Log(MediaLogMessageType.Error, $"Audio filter graph could not be built: {FilterString}.\r\n{ex.Message}");
DestroyFiltergraph();
}
}
/// <summary>
/// If necessary, disposes the existing filtergraph and creates a new one based on the frame arguments.
/// </summary>
/// <param name="frame">The frame.</param>
/// <exception cref="MediaContainerException">
/// avfilter_graph_create_filter
/// or
/// avfilter_graph_create_filter
/// or
/// avfilter_link
/// or
/// avfilter_graph_parse
/// or
/// avfilter_graph_config
/// </exception>
private void InitializeFilterGraph(AVFrame *frame)
{
/*
* References:
* http://libav-users.943685.n4.nabble.com/Libav-user-yadif-deinterlace-how-td3606561.html
* https://www.ffmpeg.org/doxygen/trunk/filtering_8c-source.html
* https://raw.githubusercontent.com/FFmpeg/FFmpeg/release/3.2/ffplay.c
*/
var frameArguments = ComputeFilterArguments(frame);
if (string.IsNullOrWhiteSpace(CurrentFilterArguments) || frameArguments.Equals(CurrentFilterArguments) == false)
{
DestroyFiltergraph();
}
else
{
return;
}
FilterGraph = ffmpeg.avfilter_graph_alloc();
RC.Current.Add(FilterGraph, $"144: {nameof(VideoComponent)}.{nameof(InitializeFilterGraph)}()");
CurrentFilterArguments = frameArguments;
try
{
var result = 0;
fixed(AVFilterContext **source = &SourceFilter)
fixed(AVFilterContext **sink = &SinkFilter)
{
result = ffmpeg.avfilter_graph_create_filter(source, ffmpeg.avfilter_get_by_name("buffer"), "video_buffer", CurrentFilterArguments, null, FilterGraph);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_create_filter)} (buffer) failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
result = ffmpeg.avfilter_graph_create_filter(sink, ffmpeg.avfilter_get_by_name("buffersink"), "video_buffersink", null, null, FilterGraph);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_create_filter)} (buffersink) failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
// TODO: from ffplay, ffmpeg.av_opt_set_int_list(sink, "pix_fmts", (byte*)&f0, 1, ffmpeg.AV_OPT_SEARCH_CHILDREN);
}
if (string.IsNullOrWhiteSpace(FilterString))
{
result = ffmpeg.avfilter_link(SourceFilter, 0, SinkFilter, 0);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_link)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
else
{
var initFilterCount = FilterGraph->nb_filters;
SourceOutput = ffmpeg.avfilter_inout_alloc();
SourceOutput->name = ffmpeg.av_strdup("in");
SourceOutput->filter_ctx = SourceFilter;
SourceOutput->pad_idx = 0;
SourceOutput->next = null;
SinkInput = ffmpeg.avfilter_inout_alloc();
SinkInput->name = ffmpeg.av_strdup("out");
SinkInput->filter_ctx = SinkFilter;
SinkInput->pad_idx = 0;
SinkInput->next = null;
result = ffmpeg.avfilter_graph_parse(FilterGraph, FilterString, SinkInput, SourceOutput, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_parse)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
// Reorder the filters to ensure that inputs of the custom filters are merged first
for (var i = 0; i < FilterGraph->nb_filters - initFilterCount; i++)
{
var sourceAddress = FilterGraph->filters[i];
var targetAddress = FilterGraph->filters[i + initFilterCount];
FilterGraph->filters[i] = targetAddress;
FilterGraph->filters[i + initFilterCount] = sourceAddress;
}
}
result = ffmpeg.avfilter_graph_config(FilterGraph, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_config)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
catch (Exception ex)
{
Container.Parent?.Log(MediaLogMessageType.Error, $"Video filter graph could not be built: {FilterString}.\r\n{ex.Message}");
DestroyFiltergraph();
}
}
/// <summary>
/// Performs the actions that this command implements.
/// </summary>
internal override void ExecuteInternal()
{
var m = Manager.MediaCore;
if (m.IsDisposed || m.IsOpen || m.IsOpening)
{
return;
}
try
{
// TODO: Sometimes when the stream can't be read, the sample player stays as if it were trying to open
// until the interrupt timeout occurs but and the Real-Time Clock continues. Strange behavior.
// Signal the initial state
m.ResetControllerProperties();
m.IsOpening = true;
m.MediaState = MediaEngineState.Manual;
// Register FFmpeg libraries if not already done
if (FFInterop.Initialize(MediaEngine.FFmpegDirectory, MediaEngine.FFmpegLoadModeFlags))
{
// Set the folders and lib identifiers
MediaEngine.FFmpegDirectory = FFInterop.LibrariesPath;
MediaEngine.FFmpegLoadModeFlags = FFInterop.LibraryIdentifiers;
// Log an init message
m.Log(MediaLogMessageType.Info,
$"{nameof(FFInterop)}.{nameof(FFInterop.Initialize)}: FFmpeg v{ffmpeg.av_version_info()}");
}
// Convert the URI object to something the Media Container understands
var mediaUrl = Source.IsFile ? Source.LocalPath : Source.ToString();
// Create the stream container
// the async protocol prefix allows for increased performance for local files.
m.Container = new MediaContainer(mediaUrl, m, Source.IsFile ? "async" : null);
m.SendOnMediaOpening();
m.Log(MediaLogMessageType.Debug, $"{nameof(OpenCommand)}: Entered");
m.Container.Open();
m.ResetBufferingProperties();
// Set the state to stopped
m.MediaState = MediaEngineState.Stop;
// Signal we are no longer in the opening state
// so we can enqueue commands in the event handler
m.IsOpening = false;
// Charge! Fire up the worker threads!
m.StartWorkers();
// Raise the opened event
m.SendOnMediaOpened();
}
catch (Exception ex)
{
m.MediaState = MediaEngineState.Close;
m.SendOnMediaFailed(ex);
}
finally
{
m.IsOpening = false;
m.NotifyPropertyChanges();
m.Log(MediaLogMessageType.Debug, $"{nameof(OpenCommand)}: Completed");
}
}
private void InitializeFilterGraph(AVFrame *frame)
{
/*
* References:
* http://libav-users.943685.n4.nabble.com/Libav-user-yadif-deinterlace-how-td3606561.html
* https://www.ffmpeg.org/doxygen/trunk/filtering_8c-source.html
* https://raw.githubusercontent.com/FFmpeg/FFmpeg/release/3.2/ffplay.c
*/
const string SourceFilterName = "buffer";
const string SourceFilterInstance = "video_buffer";
const string SinkFilterName = "buffersink";
const string SinkFilterInstance = "video_buffersink";
// Get a snapshot of the FilterString
var filterString = FilterString;
// For empty filter strings ensure filtegraph is destroyed
if (string.IsNullOrWhiteSpace(filterString))
{
DestroyFilterGraph();
return;
}
// Recreate the filtergraph if we have to
if (filterString != AppliedFilterString)
{
DestroyFilterGraph();
}
// Ensure the filtergraph is compatible with the frame
var filterArguments = ComputeFilterArguments(frame);
if (filterArguments != CurrentFilterArguments)
{
DestroyFilterGraph();
}
else
{
return;
}
FilterGraph = ffmpeg.avfilter_graph_alloc();
RC.Current.Add(FilterGraph);
try
{
// Get a couple of pointers for source and sink buffers
AVFilterContext *sourceFilterRef = null;
AVFilterContext *sinkFilterRef = null;
// Create the source filter
var result = ffmpeg.avfilter_graph_create_filter(
&sourceFilterRef, ffmpeg.avfilter_get_by_name(SourceFilterName), SourceFilterInstance, filterArguments, null, FilterGraph);
// Check filter creation
if (result != 0)
{
throw new MediaContainerException(
$"{nameof(ffmpeg.avfilter_graph_create_filter)} ({SourceFilterName}) failed. " +
$"Error {result}: {FFInterop.DecodeMessage(result)}");
}
// Create the sink filter
result = ffmpeg.avfilter_graph_create_filter(
&sinkFilterRef, ffmpeg.avfilter_get_by_name(SinkFilterName), SinkFilterInstance, null, null, FilterGraph);
// Check filter creation
if (result != 0)
{
throw new MediaContainerException(
$"{nameof(ffmpeg.avfilter_graph_create_filter)} ({SinkFilterName}) failed. " +
$"Error {result}: {FFInterop.DecodeMessage(result)}");
}
// Save the filter references
SourceFilter = sourceFilterRef;
SinkFilter = sinkFilterRef;
// TODO: from ffplay, ffmpeg.av_opt_set_int_list(sink, "pixel_formats", (byte*)&f0, 1, ffmpeg.AV_OPT_SEARCH_CHILDREN)
if (string.IsNullOrWhiteSpace(filterString))
{
result = ffmpeg.avfilter_link(SourceFilter, 0, SinkFilter, 0);
if (result != 0)
{
throw new MediaContainerException(
$"{nameof(ffmpeg.avfilter_link)} failed. " +
$"Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
else
{
var initFilterCount = FilterGraph->nb_filters;
SourceOutput = ffmpeg.avfilter_inout_alloc();
SourceOutput->name = ffmpeg.av_strdup("in");
SourceOutput->filter_ctx = SourceFilter;
SourceOutput->pad_idx = 0;
SourceOutput->next = null;
SinkInput = ffmpeg.avfilter_inout_alloc();
SinkInput->name = ffmpeg.av_strdup("out");
SinkInput->filter_ctx = SinkFilter;
SinkInput->pad_idx = 0;
SinkInput->next = null;
result = ffmpeg.avfilter_graph_parse(FilterGraph, filterString, SinkInput, SourceOutput, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_parse)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
// Reorder the filters to ensure that inputs of the custom filters are merged first
for (var i = 0; i < FilterGraph->nb_filters - initFilterCount; i++)
{
var sourceAddress = FilterGraph->filters[i];
var targetAddress = FilterGraph->filters[i + initFilterCount];
FilterGraph->filters[i] = targetAddress;
FilterGraph->filters[i + initFilterCount] = sourceAddress;
}
}
result = ffmpeg.avfilter_graph_config(FilterGraph, null);
if (result != 0)
{
throw new MediaContainerException($"{nameof(ffmpeg.avfilter_graph_config)} failed. Error {result}: {FFInterop.DecodeMessage(result)}");
}
}
catch (Exception ex)
{
this.LogError(Aspects.Component, $"Video filter graph could not be built: {filterString}.", ex);
DestroyFilterGraph();
}
finally
{
CurrentFilterArguments = filterArguments;
AppliedFilterString = filterString;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="MediaComponent"/> class.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="streamIndex">Index of the stream.</param>
/// <exception cref="ArgumentNullException">container</exception>
/// <exception cref="MediaContainerException">The container exception.</exception>
protected MediaComponent(MediaContainer container, int streamIndex)
{
// Ported from: https://github.com/FFmpeg/FFmpeg/blob/master/fftools/ffplay.c#L2559
Container = container ?? throw new ArgumentNullException(nameof(container));
m_LoggingHandler = ((ILoggingSource)Container).LoggingHandler;
m_CodecContext = new IntPtr(ffmpeg.avcodec_alloc_context3(null));
RC.Current.Add(CodecContext);
StreamIndex = streamIndex;
m_Stream = new IntPtr(container.InputContext->streams[streamIndex]);
StreamInfo = container.MediaInfo.Streams[streamIndex];
// Set default codec context options from probed stream
var setCodecParamsResult = ffmpeg.avcodec_parameters_to_context(CodecContext, Stream->codecpar);
if (setCodecParamsResult < 0)
{
this.LogWarning(Aspects.Component,
$"Could not set codec parameters. Error code: {setCodecParamsResult}");
}
// We set the packet timebase in the same timebase as the stream as opposed to the typical AV_TIME_BASE
if (this is VideoComponent && Container.MediaOptions.VideoForcedFps > 0)
{
var fpsRational = ffmpeg.av_d2q(Container.MediaOptions.VideoForcedFps, 1000000);
Stream->r_frame_rate = fpsRational;
CodecContext->pkt_timebase = new AVRational {
num = fpsRational.den, den = fpsRational.num
};
}
else
{
CodecContext->pkt_timebase = Stream->time_base;
}
// Find the default decoder codec from the stream and set it.
var defaultCodec = ffmpeg.avcodec_find_decoder(Stream->codec->codec_id);
AVCodec *forcedCodec = null;
// If set, change the codec to the forced codec.
if (Container.MediaOptions.DecoderCodec.ContainsKey(StreamIndex) &&
string.IsNullOrWhiteSpace(Container.MediaOptions.DecoderCodec[StreamIndex]) == false)
{
var forcedCodecName = Container.MediaOptions.DecoderCodec[StreamIndex];
forcedCodec = ffmpeg.avcodec_find_decoder_by_name(forcedCodecName);
if (forcedCodec == null)
{
this.LogWarning(Aspects.Component,
$"COMP {MediaType.ToString().ToUpperInvariant()}: " +
$"Unable to set decoder codec to '{forcedCodecName}' on stream index {StreamIndex}");
}
}
// Check we have a valid codec to open and process the stream.
if (defaultCodec == null && forcedCodec == null)
{
var errorMessage = $"Fatal error. Unable to find suitable decoder for {Stream->codec->codec_id.ToString()}";
CloseComponent();
throw new MediaContainerException(errorMessage);
}
var codecCandidates = new[] { forcedCodec, defaultCodec };
AVCodec *selectedCodec = null;
var codecOpenResult = 0;
foreach (var codec in codecCandidates)
{
if (codec == null)
{
continue;
}
// Pass default codec stuff to the codec context
CodecContext->codec_id = codec->id;
// Process the decoder options
{
var decoderOptions = Container.MediaOptions.DecoderParams;
// Configure the codec context flags
if (decoderOptions.EnableFastDecoding)
{
CodecContext->flags2 |= ffmpeg.AV_CODEC_FLAG2_FAST;
}
if (decoderOptions.EnableLowDelayDecoding)
{
CodecContext->flags |= ffmpeg.AV_CODEC_FLAG_LOW_DELAY;
}
// process the low res option
if (decoderOptions.LowResolutionIndex != ResolutionDivider.Full && codec->max_lowres > 0)
{
var lowResOption = Math.Min((byte)decoderOptions.LowResolutionIndex, codec->max_lowres)
.ToString(CultureInfo.InvariantCulture);
decoderOptions.LowResIndexOption = lowResOption;
}
// Ensure ref counted frames for audio and video decoding
if (CodecContext->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO || CodecContext->codec_type == AVMediaType.AVMEDIA_TYPE_AUDIO)
{
decoderOptions.RefCountedFrames = "1";
}
}
// Setup additional settings. The most important one is Threads -- Setting it to 1 decoding is very slow. Setting it to auto
// decoding is very fast in most scenarios.
var codecOptions = Container.MediaOptions.DecoderParams.GetStreamCodecOptions(Stream->index);
// Enable Hardware acceleration if requested
(this as VideoComponent)?.AttachHardwareDevice(container.MediaOptions.VideoHardwareDevice);
// Open the CodecContext. This requires exclusive FFmpeg access
lock (CodecLock)
{
var codecOptionsRef = codecOptions.Pointer;
codecOpenResult = ffmpeg.avcodec_open2(CodecContext, codec, &codecOptionsRef);
codecOptions.UpdateReference(codecOptionsRef);
}
// Check if the codec opened successfully
if (codecOpenResult < 0)
{
this.LogWarning(Aspects.Component,
$"Unable to open codec '{FFInterop.PtrToStringUTF8(codec->name)}' on stream {streamIndex}");
continue;
}
// If there are any codec options left over from passing them, it means they were not consumed
var currentEntry = codecOptions.First();
while (currentEntry?.Key != null)
{
this.LogWarning(Aspects.Component,
$"Invalid codec option: '{currentEntry.Key}' for codec '{FFInterop.PtrToStringUTF8(codec->name)}', stream {streamIndex}");
currentEntry = codecOptions.Next(currentEntry);
}
selectedCodec = codec;
break;
}
if (selectedCodec == null)
{
CloseComponent();
throw new MediaContainerException($"Unable to find suitable decoder codec for stream {streamIndex}. Error code {codecOpenResult}");
}
// Startup done. Set some options.
Stream->discard = AVDiscard.AVDISCARD_DEFAULT;
MediaType = (MediaType)CodecContext->codec_type;
switch (MediaType)
{
case MediaType.Audio:
case MediaType.Video:
BufferCountThreshold = 25;
BufferDurationThreshold = TimeSpan.FromSeconds(1);
DecodePacketFunction = DecodeNextAVFrame;
break;
case MediaType.Subtitle:
BufferCountThreshold = 0;
BufferDurationThreshold = TimeSpan.Zero;
DecodePacketFunction = DecodeNextAVSubtitle;
break;
default:
throw new NotSupportedException($"A component of MediaType '{MediaType}' is not supported");
}
if (StreamInfo.IsAttachedPictureDisposition)
{
BufferCountThreshold = 0;
BufferDurationThreshold = TimeSpan.Zero;
}
// Compute the start time
StartTime = Stream->start_time == ffmpeg.AV_NOPTS_VALUE ?
Container.MediaStartTime :
Stream->start_time.ToTimeSpan(Stream->time_base);
// compute the duration
Duration = (Stream->duration == ffmpeg.AV_NOPTS_VALUE || Stream->duration <= 0) ?
Container.MediaDuration :
Stream->duration.ToTimeSpan(Stream->time_base);
CodecId = Stream->codec->codec_id;
CodecName = FFInterop.PtrToStringUTF8(selectedCodec->name);
BitRate = Stream->codec->bit_rate < 0 ? 0 : Stream->codec->bit_rate;
this.LogDebug(Aspects.Component,
$"{MediaType.ToString().ToUpperInvariant()}: Start Offset: {StartTime.Format()}; Duration: {Duration.Format()}");
// Begin processing with a flush packet
SendFlushPacket();
}
