/// <summary>
/// a red cheomekey filter for .png image example.
/// <para>
/// ffmpeg -i <paramref name="input"/> -vf chromakey=red:0.1:0.0 <paramref name="output"/>
/// </para>
/// </summary>
/// <param name="input"></param>
/// <param name="output"></param>
public unsafe PngChromekeyFilter(string input, string output)
{
using (MediaReader reader = new MediaReader(input))
using (MediaWriter writer = new MediaWriter(output))
{
var videoIndex = reader.Where(_ => _.Codec.AVCodecContext.codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO).First().Index;
// init filter
int height = reader[videoIndex].Codec.AVCodecContext.height;
int width = reader[videoIndex].Codec.AVCodecContext.width;
int format = (int)reader[videoIndex].Codec.AVCodecContext.pix_fmt;
AVRational time_base = reader[videoIndex].TimeBase;
AVRational sample_aspect_ratio = reader[videoIndex].Codec.AVCodecContext.sample_aspect_ratio;
MediaFilterGraph filterGraph = new MediaFilterGraph();
filterGraph.AddVideoSrcFilter(new MediaFilter(MediaFilter.VideoSources.Buffer), width, height, (AVPixelFormat)format, time_base, sample_aspect_ratio).LinkTo(0,
filterGraph.AddFilter(new MediaFilter("chromakey"), "red:0.1:0.0")).LinkTo(0,
filterGraph.AddVideoSinkFilter(new MediaFilter(MediaFilter.VideoSinks.Buffersink)));
filterGraph.Initialize();
// add stream by reader and init writer
writer.AddStream(reader[videoIndex]);
writer.Initialize();
// init video frame format converter by dstcodec
PixelConverter pixelConverter = new PixelConverter(writer[0].Codec);
foreach (var srcPacket in reader.ReadPacket())
{
foreach (var srcFrame in reader[videoIndex].ReadFrame(srcPacket))
{
filterGraph.Inputs.First().WriteFrame(srcFrame);
foreach (var filterFrame in filterGraph.Outputs.First().ReadFrame())
{
// can use filterFrame.ToMat() gets the output image directly without the need for a writer.
//using EmguFFmpeg.EmguCV;
//using (var mat = filterFrame.ToMat())
//{
// mat.Save(output);
//}
foreach (var dstFrame in pixelConverter.Convert(filterFrame))
{
foreach (var dstPacket in writer[0].WriteFrame(dstFrame))
{
writer.WritePacket(dstPacket);
}
}
}
}
}
// flush codec cache
writer.FlushMuxer();
}
}
/// <summary>
/// decode video to image
/// filter graph:
/// ┌──────┐ ┌──────┐ ┌─────┐ ┌──────────┐ ┌──────┐
/// │input0│---->│buffer│---->│scale│---->│buffersink│---->│output│
/// └──────┘ └──────┘ └─────┘ └──────────┘ └──────┘
/// </summary>
/// <param name="inputFile">input video file</param>
/// <param name="outDirectory">folder for output image files</param>
/// <param name="scaleOptions">scale options <see cref="http://ffmpeg.org/ffmpeg-filters.html#scale-1"/></param>
public DecodeVideoWithCustomCodecScaledToMat(string inputFile, string outDirectory, string scaleOptions = "512:288")
{
using (MediaReader reader = new MediaReader(inputFile, null, null))
{
var videoIndex = reader.First(_ => _.Codec.AVCodecContext.codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO).Index;
unsafe
{
// relpace the default vide decode
// !!! IMPORTANT NOTE: This sample won't work, if you haven't downloaded ffmpeg (GPL license, as it is more complete), and you don't have NVIDIA hardware (CUDA) !!!
reader[videoIndex].Codec = MediaDecode.CreateDecode("h264_cuvid", _ => ffmpeg.avcodec_parameters_to_context(_, reader[videoIndex].Stream.codecpar));
}
int height = reader[videoIndex].Codec.AVCodecContext.height;
int width = reader[videoIndex].Codec.AVCodecContext.width;
int format = (int)reader[videoIndex].Codec.AVCodecContext.pix_fmt;
AVRational time_base = reader[videoIndex].TimeBase;
AVRational sample_aspect_ratio = reader[videoIndex].Codec.AVCodecContext.sample_aspect_ratio;
/* We are moving the packet to CUDA to perform the scaling.
* We can then:
* - remove hwdownload and format to leave it in CUDA, and forward the pointer to any other function, or write the frame to the output video
* - convert it to MAT whereas converting speed depends on the size of the scaled frame.
*/
MediaFilterGraph filterGraph = new MediaFilterGraph();
filterGraph.AddVideoSrcFilter(new MediaFilter(MediaFilter.VideoSources.Buffer), width, height, (AVPixelFormat)format, time_base, sample_aspect_ratio)
.LinkTo(0, filterGraph.AddFilter(new MediaFilter("scale"), scaleOptions))
.LinkTo(0, filterGraph.AddVideoSinkFilter(new MediaFilter(MediaFilter.VideoSinks.Buffersink)));
filterGraph.Initialize();
var sw = Stopwatch.StartNew();
foreach (var packet in reader.ReadPacket())
{
foreach (var frame in reader[videoIndex].ReadFrame(packet))
{
filterGraph.Inputs.First().WriteFrame(frame);
foreach (var filterFrame in filterGraph.Outputs.First().ReadFrame())
{
using (var image = filterFrame.ToMat())
{
image.Save(Path.Combine(Directory.CreateDirectory(outDirectory).FullName, $"{DateTime.Now.Ticks}.jpg"));
}
}
}
}
Console.WriteLine($"Converting to MAT [ processed in {sw.Elapsed.TotalMilliseconds:0} ms ]");
}
}
/// <summary>
/// Make the specified color of <paramref name="input0"/> transparent and overlay it on the <paramref name="input1"/> video to <paramref name="output"/>
/// <para>
/// NOTE: green [R:0 G:128 B:0]
/// </para>
/// <para>
/// ffmpeg -i <paramref name="input0"/> -i <paramref name="input1"/> -filter_complex "[1:v]chromakey=green:0.1:0.0[ckout];[0:v][ckout]overlay[out]" -map "[out]" <paramref name="output"/>
/// </para>
/// filter graph:
/// ┌──────┐ ┌──────┐ ┌─────────┐ ┌─────────┐
/// │input0│---->│buffer│---->│chromakey│---->│ │
/// └──────┘ └──────┘ └─────────┘ │ │ ┌──────────┐ ┌──────┐
/// │ overlay │---->│buffersink│---->│output│
/// ┌──────┐ ┌──────┐ │ │ └──────────┘ └──────┘
/// │input1│-----│buffer│-------------------->│ │
/// └──────┘ └──────┘ └─────────┘
/// </summary>
/// <param name="input0">foreground</param>
/// <param name="input1">background</param>
/// <param name="output">output</param>
/// <param name="chromakeyOptions">rgb(green or 0x008000):similarity:blend, see http://ffmpeg.org/ffmpeg-filters.html#chromakey </param>
public VideoChromekeyFilter(string input0, string input1, string output, string chromakeyOptions = "green:0.1:0.0")
{
using (MediaReader reader0 = new MediaReader(input0))
using (MediaReader reader1 = new MediaReader(input1))
using (MediaWriter writer = new MediaWriter(output))
{
var videoIndex0 = reader0.Where(_ => _.Codec.AVCodecContext.codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO).First().Index;
var videoIndex1 = reader1.Where(_ => _.Codec.AVCodecContext.codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO).First().Index;
// init complex filter graph
int height0 = reader0[videoIndex0].Codec.AVCodecContext.height;
int width0 = reader0[videoIndex0].Codec.AVCodecContext.width;
int format0 = (int)reader0[videoIndex0].Codec.AVCodecContext.pix_fmt;
AVRational time_base0 = reader0[videoIndex0].TimeBase;
AVRational sample_aspect_ratio0 = reader0[videoIndex0].Codec.AVCodecContext.sample_aspect_ratio;
int height1 = reader1[videoIndex1].Codec.AVCodecContext.height;
int width1 = reader1[videoIndex1].Codec.AVCodecContext.width;
int format1 = (int)reader1[videoIndex1].Codec.AVCodecContext.pix_fmt;
AVRational time_base1 = reader1[videoIndex1].TimeBase;
AVRational sample_aspect_ratio1 = reader1[videoIndex1].Codec.AVCodecContext.sample_aspect_ratio;
MediaFilterGraph filterGraph = new MediaFilterGraph();
var in0 = filterGraph.AddVideoSrcFilter(new MediaFilter(MediaFilter.VideoSources.Buffer), width0, height0, (AVPixelFormat)format0, time_base0, sample_aspect_ratio0);
var in1 = filterGraph.AddVideoSrcFilter(new MediaFilter(MediaFilter.VideoSources.Buffer), width1, height1, (AVPixelFormat)format1, time_base1, sample_aspect_ratio1);
var chromakey = filterGraph.AddFilter(new MediaFilter("chromakey"), chromakeyOptions);
var overlay = filterGraph.AddFilter(new MediaFilter("overlay"));
var out0 = filterGraph.AddVideoSinkFilter(new MediaFilter(MediaFilter.VideoSinks.Buffersink));
in0.LinkTo(0, chromakey, 0).LinkTo(0, overlay, 1).LinkTo(0, out0, 0);
in1.LinkTo(0, overlay, 0);
filterGraph.Initialize();
// add stream by reader and init writer
writer.AddStream(reader0[videoIndex0]);
writer.Initialize();
// init video frame format converter by dstcodec
PixelConverter pixelConverter = new PixelConverter(writer[0].Codec);
long pts = 0;
MediaReader[] readers = new MediaReader[] { reader0, reader1 };
int[] index = new int[] { videoIndex0, videoIndex1 };
for (int i = 0; i < readers.Length; i++)
{
var reader = readers[i];
foreach (var srcPacket in reader.ReadPacket())
{
foreach (var srcFrame in reader[index[i]].ReadFrame(srcPacket))
{
filterGraph.Inputs[i].WriteFrame(srcFrame);
foreach (var filterFrame in filterGraph.Outputs.First().ReadFrame())
{
foreach (var dstFrame in pixelConverter.Convert(filterFrame))
{
dstFrame.Pts = pts++;
foreach (var dstPacket in writer[0].WriteFrame(dstFrame))
{
writer.WritePacket(dstPacket);
}
}
}
}
}
}
// flush codec cache
writer.FlushMuxer();
}
}