private void Media_MediaOpening(object sender, MediaOpeningRoutedEventArgs e) { if (VIDTextBox.Text.StartsWith("udp://")) { e.Options.VideoFilter = "yadif"; } var videoStream = e.Options.VideoStream; if (videoStream != null) { // Check if the video requires deinterlacing var requiresDeinterlace = videoStream.FieldOrder != AVFieldOrder.AV_FIELD_PROGRESSIVE && videoStream.FieldOrder != AVFieldOrder.AV_FIELD_UNKNOWN; // Hardwrae device priorities var deviceCandidates = new AVHWDeviceType[] { AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2, AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA, AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA }; // Hardware device selection if (videoStream.FPS <= 30) { foreach (var deviceType in deviceCandidates) { var accelerator = videoStream.HardwareDevices.FirstOrDefault(d => d.DeviceType == deviceType); if (accelerator != null) { if (Platform.GuiContext.Current.IsInDebugMode) { e.Options.VideoHardwareDevice = accelerator; } break; } } } var videoFilter = new StringBuilder(); // The yadif filter deinterlaces the video; we check the field order if we need // to deinterlace the video automatically if (requiresDeinterlace) { videoFilter.Append("yadif,"); } // Scale down to maximum 1080p screen resolution. if (videoStream.PixelHeight > 1080) { // e.Options.VideoHardwareDevice = null; videoFilter.Append($"scale=-1:1080,"); } e.Options.VideoFilter = videoFilter.ToString().TrimEnd(','); } }
private static unsafe void DecodeAllFramesToImages(AVHWDeviceType HWDevice) { // decode all frames from url, please not it might local resorce, e.g. string url = "../../sample_mpeg4.mp4"; var url = "http://clips.vorwaerts-gmbh.de/big_buck_bunny.mp4"; // be advised this file holds 1440 frames using (var vsd = new VideoStreamDecoder(url, HWDevice)) { Console.WriteLine($"codec name: {vsd.CodecName}"); var info = vsd.GetContextInfo(); info.ToList().ForEach(x => Console.WriteLine($"{x.Key} = {x.Value}")); var sourceSize = vsd.FrameSize; var sourcePixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); var destinationSize = sourceSize; var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; using (var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat)) { var frameNumber = 0; while (vsd.TryDecodeNextFrame(out var frame)) { var convertedFrame = vfc.Convert(frame); using (var bitmap = new Bitmap(convertedFrame.width, convertedFrame.height, convertedFrame.linesize[0], PixelFormat.Format24bppRgb, (IntPtr)convertedFrame.data[0])) bitmap.Save($"frame.{frameNumber:D8}.jpg", ImageFormat.Jpeg); Console.WriteLine($"frame: {frameNumber}"); frameNumber++; } } } }
public static HardwareVideoDecoder?ToHardwareVideoDecoder(this AVHWDeviceType hwDeviceType) { switch (hwDeviceType) { case AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA: return(HardwareVideoDecoder.NVDEC); case AVHWDeviceType.AV_HWDEVICE_TYPE_QSV: return(HardwareVideoDecoder.QuickSyncVideo); case AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2: return(HardwareVideoDecoder.DXVA2); case AVHWDeviceType.AV_HWDEVICE_TYPE_VDPAU: return(HardwareVideoDecoder.VDPAU); case AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI: return(HardwareVideoDecoder.VAAPI); case AVHWDeviceType.AV_HWDEVICE_TYPE_MEDIACODEC: return(HardwareVideoDecoder.MediaCodec); case AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX: return(HardwareVideoDecoder.VideoToolbox); default: return(null); } }
private static AVPixelFormat GetHWPixelFormat(AVHWDeviceType hWDevice) { switch (hWDevice) { case AVHWDeviceType.AV_HWDEVICE_TYPE_NONE: return(AVPixelFormat.AV_PIX_FMT_NONE); case AVHWDeviceType.AV_HWDEVICE_TYPE_VDPAU: return(AVPixelFormat.AV_PIX_FMT_VDPAU); case AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA: return(AVPixelFormat.AV_PIX_FMT_CUDA); case AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI: return(AVPixelFormat.AV_PIX_FMT_VAAPI); case AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2: return(AVPixelFormat.AV_PIX_FMT_NV12); case AVHWDeviceType.AV_HWDEVICE_TYPE_QSV: return(AVPixelFormat.AV_PIX_FMT_QSV); case AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX: return(AVPixelFormat.AV_PIX_FMT_VIDEOTOOLBOX); case AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA: return(AVPixelFormat.AV_PIX_FMT_NV12); case AVHWDeviceType.AV_HWDEVICE_TYPE_DRM: return(AVPixelFormat.AV_PIX_FMT_DRM_PRIME); case AVHWDeviceType.AV_HWDEVICE_TYPE_OPENCL: return(AVPixelFormat.AV_PIX_FMT_OPENCL); case AVHWDeviceType.AV_HWDEVICE_TYPE_MEDIACODEC: return(AVPixelFormat.AV_PIX_FMT_MEDIACODEC); default: return(AVPixelFormat.AV_PIX_FMT_NONE); } }
public StreamDecoder( AVFormatContext *formatContext, int streamIndex, AVCodec *codec, AVHWDeviceType hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) : base(null) { _streamIndex = streamIndex; _formatContext = formatContext; _stream = formatContext->streams[streamIndex]; _codecContext = ffmpeg.avcodec_alloc_context3(codec); if (hwDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { ffmpeg.av_hwdevice_ctx_create(&_codecContext->hw_device_ctx, hwDeviceType, null, null, 0) .ThrowExceptionIfError(); } ffmpeg.avcodec_parameters_to_context(_codecContext, _stream->codecpar) .ThrowExceptionIfError(); ffmpeg.avcodec_open2(_codecContext, codec, null).ThrowExceptionIfError(); _codecContext->pkt_timebase = _stream->time_base; _codec = codec; _packet = ffmpeg.av_packet_alloc(); _frame = ffmpeg.av_frame_alloc(); _receivedFrame = ffmpeg.av_frame_alloc(); }
public VideoStreamDecoder(string url, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { _pFormatContext = ffmpeg.avformat_alloc_context(); _receivedFrame = ffmpeg.av_frame_alloc(); var pFormatContext = _pFormatContext; ffmpeg.avformat_open_input(&pFormatContext, url, null, null).ThrowExceptionIfError(); ffmpeg.avformat_find_stream_info(_pFormatContext, null).ThrowExceptionIfError(); AVCodec *codec = null; _streamIndex = ffmpeg .av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0) .ThrowExceptionIfError(); _pCodecContext = ffmpeg.avcodec_alloc_context3(codec); if (HWDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { ffmpeg.av_hwdevice_ctx_create(&_pCodecContext->hw_device_ctx, HWDeviceType, null, null, 0) .ThrowExceptionIfError(); } ffmpeg.avcodec_parameters_to_context(_pCodecContext, _pFormatContext->streams[_streamIndex]->codecpar) .ThrowExceptionIfError(); ffmpeg.avcodec_open2(_pCodecContext, codec, null).ThrowExceptionIfError(); CodecName = ffmpeg.avcodec_get_name(codec->id); FrameSize = new Size(_pCodecContext->width, _pCodecContext->height); PixelFormat = _pCodecContext->pix_fmt; _pPacket = ffmpeg.av_packet_alloc(); _pFrame = ffmpeg.av_frame_alloc(); }
//public AVFormatContext* PFormatContext //{ // get // { // return _pFormatContext; // } //} public VideoStreamDecoder(string url, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { _pFormatContext = ffmpeg.avformat_alloc_context(); _receivedFrame = ffmpeg.av_frame_alloc(); var pFormatContext = _pFormatContext; ffmpeg.avformat_open_input(&pFormatContext, url, null, null).ThrowExceptionIfError(); ffmpeg.avformat_find_stream_info(_pFormatContext, null).ThrowExceptionIfError(); AVCodec *codec = null; // TODO: Why is ffmpeg.AV_CODEC_FLAG_LOW_DELAY ignored! _streamIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &codec, ffmpeg.AV_CODEC_FLAG_LOW_DELAY).ThrowExceptionIfError(); _pCodecContext = ffmpeg.avcodec_alloc_context3(codec); if (HWDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { ffmpeg.av_hwdevice_ctx_create(&_pCodecContext->hw_device_ctx, HWDeviceType, null, null, 0).ThrowExceptionIfError(); } ffmpeg.avcodec_parameters_to_context(_pCodecContext, _pFormatContext->streams[_streamIndex]->codecpar).ThrowExceptionIfError(); ffmpeg.avcodec_open2(_pCodecContext, codec, null).ThrowExceptionIfError(); CodecName = ffmpeg.avcodec_get_name(codec->id); FrameSize = new Size(_pCodecContext->width, _pCodecContext->height); PixelFormat = _pCodecContext->pix_fmt; DurationMilliseconds = _pFormatContext->duration; Framerate = 25;// _pFormatContext->video_codec->supported_framerates[0].num / _pFormatContext->video_codec->supported_framerates[0].den; _pPacket = ffmpeg.av_packet_alloc(); _pFrame = ffmpeg.av_frame_alloc(); }
private static void ConfigureHWDecoder(out AVHWDeviceType HWtype) { HWtype = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; Console.WriteLine("Use hardware acceleration for decoding?[n]"); var key = Console.ReadLine(); var availableHWDecoders = new Dictionary <int, AVHWDeviceType>(); if (key == "y") { Console.WriteLine("Select hardware decoder:"); var type = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; var number = 0; while ((type = ffmpeg.av_hwdevice_iterate_types(type)) != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { Console.WriteLine($"{++number}. {type}"); availableHWDecoders.Add(number, type); } if (availableHWDecoders.Count == 0) { Console.WriteLine("Your system have no hardware decoders."); HWtype = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; return; } int decoderNumber = availableHWDecoders.SingleOrDefault(t => t.Value == AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2).Key; if (decoderNumber == 0) { decoderNumber = availableHWDecoders.First().Key; } Console.WriteLine($"Selected [{decoderNumber}]"); int.TryParse(Console.ReadLine(), out var inputDecoderNumber); availableHWDecoders.TryGetValue(inputDecoderNumber == 0 ? decoderNumber : inputDecoderNumber, out HWtype); } }
private void ConfigureHWDecoder(out AVHWDeviceType HWtype) { HWtype = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; if (!Hw) { return; } Debug.WriteLine("Use hardware acceleration for decoding?[n]"); var availableHWDecoders = new Dictionary <int, AVHWDeviceType>(); Debug.WriteLine("Select hardware decoder:"); var type = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; var number = 0; while ((type = ffmpeg.av_hwdevice_iterate_types(type)) != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { Debug.WriteLine($"{++number}. {type}"); availableHWDecoders.Add(number, type); } if (availableHWDecoders.Count == 0) { Debug.WriteLine("Your system have no hardware decoders."); HWtype = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; return; } int decoderNumber = availableHWDecoders.SingleOrDefault(t => t.Value == AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2).Key; if (decoderNumber == 0) { decoderNumber = availableHWDecoders.First().Key; } availableHWDecoders.TryGetValue(decoderNumber, out HWtype); }
public AudioStreamDecoder(string url, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { _pFormatContext = ffmpeg.avformat_alloc_context(); _receivedFrame = ffmpeg.av_frame_alloc(); var pFormatContext = _pFormatContext; ffmpeg.avformat_open_input(&pFormatContext, url, null, null).ThrowExceptionIfError(); ffmpeg.avformat_find_stream_info(_pFormatContext, null).ThrowExceptionIfError(); AVCodec *videoCodec = null; _streamVideoIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0).ThrowExceptionIfError(); _pVideoCodecContext = ffmpeg.avcodec_alloc_context3(videoCodec); if (HWDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { ffmpeg.av_hwdevice_ctx_create(&_pVideoCodecContext->hw_device_ctx, HWDeviceType, null, null, 0).ThrowExceptionIfError(); } ffmpeg.avcodec_parameters_to_context(_pVideoCodecContext, _pFormatContext->streams[_streamVideoIndex]->codecpar).ThrowExceptionIfError(); if (_pFormatContext->streams[_streamVideoIndex]->avg_frame_rate.den != 0) { Fps = _pFormatContext->streams[_streamVideoIndex]->avg_frame_rate.num / _pFormatContext->streams[_streamVideoIndex]->avg_frame_rate.den; Console.WriteLine("计算得到FPS"); } else { Console.WriteLine("默认FPS"); Fps = 25; } ffmpeg.avcodec_open2(_pVideoCodecContext, videoCodec, null).ThrowExceptionIfError(); CodecName = ffmpeg.avcodec_get_name(videoCodec->id); FrameSize = new Size(_pVideoCodecContext->width, _pVideoCodecContext->height); PixelFormat = _pVideoCodecContext->pix_fmt; _pPacket = ffmpeg.av_packet_alloc(); _pFrame = ffmpeg.av_frame_alloc(); AVCodec *audioCodec = null; _streamAudioIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &audioCodec, 0).ThrowExceptionIfError(); _pAudioCodecContext = ffmpeg.avcodec_alloc_context3(audioCodec); ffmpeg.avcodec_parameters_to_context(_pAudioCodecContext, _pFormatContext->streams[_streamAudioIndex]->codecpar).ThrowExceptionIfError(); ffmpeg.avcodec_open2(_pAudioCodecContext, audioCodec, null).ThrowExceptionIfError(); if (_streamAudioIndex > 0) { AVStream *avs = _pFormatContext->streams[_streamAudioIndex]; Console.WriteLine($"codec_id:{avs->codecpar->codec_id}"); Console.WriteLine($"format:{avs->codecpar->format}"); Console.WriteLine($"sample_rate:{avs->codecpar->sample_rate}"); Console.WriteLine($"channels:{avs->codecpar->channels}"); Console.WriteLine($"frame_size:{avs->codecpar->frame_size}"); in_sample_fmt = _pAudioCodecContext->sample_fmt; in_sample_rate = _pAudioCodecContext->sample_rate; //输入的采样率 in_ch_layout = _pAudioCodecContext->channel_layout; //输入的声道布局 in_channels = _pAudioCodecContext->channels; in_start_time = avs->start_time; } }
public VideoStreamDecoder(string url, VideoInputType inputType, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { try { ffmpeg.avdevice_register_all(); iFormatContext = ffmpeg.avformat_alloc_context(); receivedFrame = ffmpeg.av_frame_alloc(); var _iFormatContext = iFormatContext; AVDictionary *avDict; ffmpeg.av_dict_set(&avDict, "reorder_queue_size", "1", 0); switch (inputType) { case VideoInputType.CAM_DEVICE: AVInputFormat *iformat = ffmpeg.av_find_input_format("dshow"); ffmpeg.avformat_open_input(&_iFormatContext, url, iformat, null).ThrowExceptionIfError(); break; case VideoInputType.RTP_RTSP: ffmpeg.avformat_open_input(&_iFormatContext, url, null, &avDict).ThrowExceptionIfError(); break; default: break; } ffmpeg.avformat_find_stream_info(iFormatContext, null).ThrowExceptionIfError(); AVCodec *codec; dec_stream_index = ffmpeg.av_find_best_stream(iFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0).ThrowExceptionIfError(); iCodecContext = ffmpeg.avcodec_alloc_context3(codec); if (HWDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { ffmpeg.av_hwdevice_ctx_create(&iCodecContext->hw_device_ctx, HWDeviceType, null, null, 0).ThrowExceptionIfError(); } ffmpeg.avcodec_parameters_to_context(iCodecContext, iFormatContext->streams[dec_stream_index]->codecpar).ThrowExceptionIfError(); ffmpeg.avcodec_open2(iCodecContext, codec, null).ThrowExceptionIfError(); CodecName = ffmpeg.avcodec_get_name(codec->id); FrameSize = new Size(iCodecContext->width, iCodecContext->height); PixelFormat = iCodecContext->pix_fmt; rawPacket = ffmpeg.av_packet_alloc(); decodedFrame = ffmpeg.av_frame_alloc(); } catch (AccessViolationException ex) { throw new AccessViolationException("Access Violation Exception", ex); } }
private static AVPixelFormat FindPixelFormat(AVHWDeviceType deviceType) { if (deviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA) { return(AVPixelFormat.AV_PIX_FMT_D3D11); } throw new NotImplementedException($"Hardware device type '{deviceType}' not supported"); }
public void AutoSelectHardwareDevice() { var types = MediaHelper.GetHardwareDeviceTypes(); if (types.Length > 0) { HardwareDevice = types[0]; } }
private void Form1_Load(object sender, EventArgs e) { Image screenboard = ScreenBoradImage.Image; status = "정상 운행중"; end = "프로그램을 종료합니다."; start = "프로그램을 시작합니다."; connected = "연결이 되었습니다."; disconnected = "연결이 끊겼습니다."; camerastatus1 = "전방 카메라 "; camerastatus2 = "후방 카메라 "; firstlcs = "1차로 "; secondlcs = "2차로 "; thirdlcs = "3차로 "; alllcs = "전차로 "; stop = "진입금지"; forward = "진입가능"; this.lgm = new LogManager(start); enterhold = true; enterhold2 = true; enterhold3 = true; camera1cnt = 0; camera2cnt = 0; video1cnt = 0; video2cnt = 0; gridcnt = 0; imagemax1 = false; imagemax2 = false; camera1record = 0; camera2record = 0; this.dataGridView1.ColumnHeadersDefaultCellStyle.Font = new Font("Courier New", 13, FontStyle.Bold); this.dataGridView1.DefaultCellStyle.Font = new Font("Courier New", 13); this.dataGridView1.RowsDefaultCellStyle.BackColor = Color.DarkBlue; lcsgo = "http://192.168.10.105/relay_cgi.cgi?type=0&relay=0&on=1&time=0&pwd=0&"; lcsgostop = "http://192.168.10.105/relay_cgi.cgi?type=0&relay=0&on=0&time=0&pwd=0&"; lcsstop = "http://192.168.10.105/relay_cgi.cgi?type=0&relay=1&on=1&time=0&pwd=0&"; lcsstopstop = "http://192.168.10.105/relay_cgi.cgi?type=0&relay=1&on=0&time=0&pwd=0&"; lcsflash = "http://192.168.10.105/relay_cgi.cgi?type=0&relay=2&on=1&time=0&pwd=0&"; lcsflashstop = "http://192.168.10.105/relay_cgi.cgi?type=0&relay=2&on=0&time=0&pwd=0&"; //SendLcsData(lcsgo); PlayCamera1(); PlayCamera2(); hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; hwDeviceType2 = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; //temp FFmpegBinariesHelper.RegisterFFmpegBinaries(); //ConfigureHWDecoder(out hwDeviceType); isInit = false; isInit2 = false; }
public EasyFFmpegManager() { hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; //temp FFmpegBinariesHelper.RegisterFFmpegBinaries(); //ConfigureHWDecoder(out hwDeviceType); isInit = false; }
private List <AVHWDeviceType> GetHWDevices() { List <AVHWDeviceType> hwDevices = new List <AVHWDeviceType>(); AVHWDeviceType type = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; while ((type = av_hwdevice_iterate_types(type)) != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { hwDevices.Add(type); } return(hwDevices); }
public static AVPixelFormat GetHWPixelFormat(AVHWDeviceType hWDevice) { return(hWDevice switch { AVHWDeviceType.AV_HWDEVICE_TYPE_NONE => AVPixelFormat.AV_PIX_FMT_NONE, AVHWDeviceType.AV_HWDEVICE_TYPE_VDPAU => AVPixelFormat.AV_PIX_FMT_VDPAU, AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA => AVPixelFormat.AV_PIX_FMT_CUDA, AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI => AVPixelFormat.AV_PIX_FMT_VAAPI, AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2 => AVPixelFormat.AV_PIX_FMT_NV12, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV => AVPixelFormat.AV_PIX_FMT_QSV, AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX => AVPixelFormat.AV_PIX_FMT_VIDEOTOOLBOX, AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA => AVPixelFormat.AV_PIX_FMT_NV12, AVHWDeviceType.AV_HWDEVICE_TYPE_DRM => AVPixelFormat.AV_PIX_FMT_DRM_PRIME, AVHWDeviceType.AV_HWDEVICE_TYPE_OPENCL => AVPixelFormat.AV_PIX_FMT_OPENCL, AVHWDeviceType.AV_HWDEVICE_TYPE_MEDIACODEC => AVPixelFormat.AV_PIX_FMT_MEDIACODEC, _ => AVPixelFormat.AV_PIX_FMT_NONE });
public VideoStreamDecoder(string url, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { source = url; _pFormatContext = ffmpeg.avformat_alloc_context(); _receivedFrame = ffmpeg.av_frame_alloc(); var pFormatContext = _pFormatContext; ffmpeg.avformat_open_input(&pFormatContext, source, null, null).ThrowExceptionIfError(); ffmpeg.avformat_find_stream_info(_pFormatContext, null).ThrowExceptionIfError(); AVCodec *codec = null; _streamIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0).ThrowExceptionIfError(); _pCodecContext = ffmpeg.avcodec_alloc_context3(codec); if (HWDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { ffmpeg.av_hwdevice_ctx_create(&_pCodecContext->hw_device_ctx, HWDeviceType, null, null, 0).ThrowExceptionIfError(); } pStream = _pFormatContext->streams[_streamIndex]; ffmpeg.avcodec_parameters_to_context(_pCodecContext, _pFormatContext->streams[_streamIndex]->codecpar).ThrowExceptionIfError(); ffmpeg.avcodec_open2(_pCodecContext, codec, null).ThrowExceptionIfError(); CodecName = ffmpeg.avcodec_get_name(codec->id); FrameSize = new Size(_pCodecContext->width, _pCodecContext->height); PixelFormat = _pCodecContext->pix_fmt; _pPacket = ffmpeg.av_packet_alloc(); _pFrame = ffmpeg.av_frame_alloc(); PConvertContext = ffmpeg.sws_getContext(_pCodecContext->width, _pCodecContext->height, _pCodecContext->pix_fmt, _pCodecContext->width, _pCodecContext->height, AVPixelFormat.AV_PIX_FMT_BGRA, ffmpeg.SWS_FAST_BILINEAR, null, null, null); if (PConvertContext == null) { throw new ApplicationException(@"Could not initialize the conversion context."); } dstData = new byte_ptrArray4(); dstLinesize = new int_array4(); convertedFrameBufferPtr = Marshal.AllocHGlobal(ffmpeg.av_image_get_buffer_size(AVPixelFormat.AV_PIX_FMT_BGRA, _pCodecContext->width, _pCodecContext->height, 1)); ffmpeg.av_image_fill_arrays(ref dstData, ref dstLinesize, (byte *)convertedFrameBufferPtr, AVPixelFormat.AV_PIX_FMT_BGRA, _pCodecContext->width, _pCodecContext->height, 1); }
private Texture2D DecodeFrameToTexture2D(String filename, int frameIndex = 10, AVHWDeviceType HWDevice = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { using (var vsd = new VideoStreamDecoder(filename, HWDevice)) { Debug.Log($"codec name: {vsd.CodecName}"); var info = vsd.GetContextInfo(); info.ToList().ForEach(x => Debug.Log($"{x.Key} = {x.Value}")); var sourceSize = vsd.FrameSize; var sourcePixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); var destinationSize = sourceSize; var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; using (var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat)) { var currentFrame = 0; while (vsd.TryDecodeNextFrame(out var frame) && _isRunning) { Debug.Log($"Processing frame: {currentFrame}"); var avframe = vfc.Convert(frame); if (OnFrameRendered != null) { byte[] imageData; vsd.AvFrameToImageByteArray(avframe, out imageData); OnFrameRendered(imageData); } if (currentFrame == frameIndex) { Debug.Log($"Saving frame: {frameIndex}"); return(vsd.AVFrameToTexture2D(avframe)); } currentFrame++; } return(new Texture2D(4, 4)); } } }
private unsafe AVPixelFormat GetHWPixelFormat(AVHWDeviceType hwDevice, AVCodec *codec) { const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 1; AVPixelFormat pixelFormat = AVPixelFormat.AV_PIX_FMT_NONE; for (int i = 0; ; i++) { AVCodecHWConfig *hwConfig = ffmpeg.avcodec_get_hw_config(codec, i); if (hwConfig == null) { throw new Exception($"Failed to find compatible pixel format for {hwDevice}"); } if ((hwConfig->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) == 0 || hwConfig->device_type != hwDevice) { continue; } AVHWFramesConstraints *hwConstraints = ffmpeg.av_hwdevice_get_hwframe_constraints(_pCodecContext->hw_device_ctx, hwConfig); if (hwConstraints != null) { for (AVPixelFormat *p = hwConstraints->valid_sw_formats; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++) { pixelFormat = *p; if (ffmpeg.sws_isSupportedInput(pixelFormat) > 0) { break; } else { pixelFormat = AVPixelFormat.AV_PIX_FMT_NONE; } } ffmpeg.av_hwframe_constraints_free(&hwConstraints); } if (pixelFormat != AVPixelFormat.AV_PIX_FMT_NONE) { return(pixelFormat); } } }
public static unsafe byte[]? GetThumbnail(FfmpegSettings settings, bool extendedLogging) { try { if (UseNativeBinding) { bool isGrayByte = settings.GrayScale == 1; AVHWDeviceType HWDevice = HardwareAccelerationMode switch { FFHardwareAccelerationMode.vdpau => AVHWDeviceType.AV_HWDEVICE_TYPE_VDPAU, FFHardwareAccelerationMode.dxva2 => AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2, FFHardwareAccelerationMode.vaapi => AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI, FFHardwareAccelerationMode.qsv => AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, FFHardwareAccelerationMode.cuda => AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA, _ => AVHWDeviceType.AV_HWDEVICE_TYPE_NONE }; using var vsd = new VideoStreamDecoder(settings.File, HWDevice); if (vsd.PixelFormat < 0 || vsd.PixelFormat >= AVPixelFormat.AV_PIX_FMT_NB) { throw new Exception($"Invalid source pixel format"); } Size sourceSize = vsd.FrameSize; Size destinationSize = isGrayByte ? new Size(16, 16) : new Size(100, Convert.ToInt32(sourceSize.Height * (100 / (double)sourceSize.Width))); AVPixelFormat destinationPixelFormat = isGrayByte ? AVPixelFormat.AV_PIX_FMT_GRAY8 : AVPixelFormat.AV_PIX_FMT_BGRA; using var vfc = new VideoFrameConverter(sourceSize, vsd.PixelFormat, destinationSize, destinationPixelFormat); if (!vsd.TryDecodeFrame(out var srcFrame, settings.Position)) { throw new Exception($"Failed decoding frame at {settings.Position}"); } AVFrame convertedFrame = vfc.Convert(srcFrame); if (isGrayByte) { int length = ffmpeg.av_image_get_buffer_size(destinationPixelFormat, convertedFrame.width, convertedFrame.height, 1).ThrowExceptionIfError(); byte[] data = new byte[length]; Marshal.Copy((IntPtr)convertedFrame.data[0], data, 0, length); return(data); } else { int width = convertedFrame.width; int height = convertedFrame.height; var totalBytes = width * height * 4; var rgbaBytes = new byte[totalBytes]; int stride = convertedFrame.linesize[0]; if (stride == width * 4) { Marshal.Copy((IntPtr)convertedFrame.data[0], rgbaBytes, 0, totalBytes); } else { var sourceOffset = 0; var destOffset = 0; var byteWidth = width * 4; for (var y = 0; y < height; y++) { Marshal.Copy((IntPtr)convertedFrame.data[0] + sourceOffset, rgbaBytes, destOffset, byteWidth); sourceOffset += stride; destOffset += byteWidth; } } var image = Image.LoadPixelData <SixLabors.ImageSharp.PixelFormats.Bgra32>(rgbaBytes, width, height); using MemoryStream stream = new(); image.Save(stream, new SixLabors.ImageSharp.Formats.Jpeg.JpegEncoder()); bool equal = rgbaBytes.SequenceEqual(stream.ToArray()); return(stream.ToArray()); } } } catch (Exception e) { Logger.Instance.Info($"Failed using native FFmpeg binding on '{settings.File}', try switching to process mode. Exception: {e}"); } //https://docs.microsoft.com/en-us/dotnet/csharp/how-to/concatenate-multiple-strings#string-literals string ffmpegArguments = $" -hide_banner -loglevel {(extendedLogging ? "error" : "quiet")}" + $" -y -hwaccel {HardwareAccelerationMode} -ss {settings.Position} -i \"{settings.File}\"" + $" -t 1 -f {(settings.GrayScale == 1 ? "rawvideo -pix_fmt gray" : "mjpeg")} -vframes 1" + $" {(settings.GrayScale == 1 ? "-s 16x16" : "-vf scale=100:-1")} {CustomFFArguments} \"-\""; using var process = new Process { StartInfo = new ProcessStartInfo { Arguments = ffmpegArguments, FileName = FFmpegPath, CreateNoWindow = true, RedirectStandardInput = false, RedirectStandardOutput = true, WorkingDirectory = Path.GetDirectoryName(FFmpegPath) !, RedirectStandardError = extendedLogging, WindowStyle = ProcessWindowStyle.Hidden } }; string errOut = string.Empty; byte[]? bytes = null; try { process.EnableRaisingEvents = true; process.Start(); if (extendedLogging) { process.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => { if (e.Data?.Length > 0) { errOut += Environment.NewLine + e.Data; } }); process.BeginErrorReadLine(); } using var ms = new MemoryStream(); process.StandardOutput.BaseStream.CopyTo(ms); if (!process.WaitForExit(TimeoutDuration)) { throw new TimeoutException($"FFmpeg timed out on file: {settings.File}"); } else if (extendedLogging) { process.WaitForExit(); // Because of asynchronous event handlers, see: https://github.com/dotnet/runtime/issues/18789 } if (process.ExitCode != 0) { throw new FFInvalidExitCodeException($"FFmpeg exited with: {process.ExitCode}"); } bytes = ms.ToArray(); if (bytes.Length == 0) { bytes = null; // Makes subsequent checks easier } else if (settings.GrayScale == 1 && bytes.Length != 256) { bytes = null; errOut += $"{Environment.NewLine}graybytes length != 256"; } } catch (Exception e) { errOut += $"{Environment.NewLine}{e.Message}"; try { if (process.HasExited == false) { process.Kill(); } } catch { } bytes = null; } if (bytes == null || errOut.Length > 0) { string message = $"{((bytes == null) ? "ERROR: Failed to retrieve" : "WARNING: Problems while retrieving")} {(settings.GrayScale == 1 ? "graybytes" : "thumbnail")} from: {settings.File}"; if (extendedLogging) { message += $":{Environment.NewLine}{FFmpegPath} {ffmpegArguments}"; } Logger.Instance.Info($"{message}{errOut}"); } return(bytes); }
private static extern int av_hwdevice_ctx_create(AVBufferRef **device_ctx, AVHWDeviceType type, [MarshalAs(UnmanagedType.LPUTF8Str)] string device, AVDictionary *opts, int flags);
public VideoStreamDecoder(string url, VIDEO_INPUT_TYPE inputType = VIDEO_INPUT_TYPE.RTP_RTSP, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { ffmpeg.avdevice_register_all(); AVFormatContext *pFormatCtx = ffmpeg.avformat_alloc_context(); AVDictionary * options = null; ffmpeg.av_dict_set(&options, "list_devices", "true", 0); AVInputFormat *iformat = ffmpeg.av_find_input_format("dshow"); Console.WriteLine("========Device Info=============\n"); ffmpeg.avformat_open_input(&pFormatCtx, null, iformat, &options); Console.WriteLine("===============================\n"); AVDeviceInfoList *device_list = null; int result = ffmpeg.avdevice_list_input_sources(iformat, null, options, &device_list); Console.WriteLine(result); //iFormatContext = ffmpeg.avform at_alloc_context(); //receivedFrame = ffmpeg.av_frame_alloc(); //var _iFormatContext = iFormatContext; //int i; //AVDictionary* avDict; //ffmpeg.av_dict_set(&avDict, "reorder_queue_size", "1", 0); //switch (inputType) //{ // case VIDEO_INPUT_TYPE.CAM_DEVICE: // AVInputFormat* iformat = ffmpeg.av_find_input_format("dshow"); // AVDeviceInfoList* listdevice = null; // ffmpeg.avdevice_list_devices(_iFormatContext, (AVDeviceInfoList**)listdevice); // Console.WriteLine(listdevice->devices[0]->ToString()); // //ffmpeg.avformat_open_input(&_iFormatContext, url, iformat, null).ThrowExceptionIfError(); // break; // case VIDEO_INPUT_TYPE.RTP_RTSP: // ffmpeg.avformat_open_input(&_iFormatContext, @"C:\Users\admin\Desktop\result1.avi", null, null); // break; // default: // break; //} Console.ReadLine(); //_iFormatContext->streams[0]->time_base = new AVRational { num = 1, den = 30 }; //_iFormatContext->streams[0]->avg_frame_rate = new AVRational { num = 30, den = 1 }; //AVCodec* videoCodec = null; //AVCodec* audioCodec = null; //for (i = 0; i < _iFormatContext->nb_streams; i++) //{ // if (_iFormatContext->streams[i]->codec->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO) // { // videoIndex = i; // videoCodecContext = _iFormatContext->streams[i]->codec; // videoCodec = ffmpeg.avcodec_find_decoder(videoCodecContext->codec_id); // } // else if (_iFormatContext->streams[i]->codec->codec_type == AVMediaType.AVMEDIA_TYPE_AUDIO) // { // audioCodeContext = _iFormatContext->streams[i]->codec; // audioCodec = ffmpeg.avcodec_find_decoder(audioCodeContext->codec_id); // audioIndex = i; // } //} //ffmpeg.avformat_find_stream_info(_iFormatContext, null).ThrowExceptionIfError(); //Stream에 접근하기 위해서는 미디어로부터 데이터 읽어야함. //videoStreamIndex = ffmpeg.av_find_best_stream(_iFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0).ThrowExceptionIfError(); //audioStreamIndex = ffmpeg.av_find_best_stream(_iFormatContext, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &audioCodec, 0); //Console.WriteLine($"VideoStreamIndex : {videoIndex} AudioStreamIndex : {audioIndex}"); //Console.WriteLine($"VideoCodec : {videoCodec->id} AudioCodec : {audioCodec->id}"); //videoCodecContext = ffmpeg.avcodec_alloc_context3(videoCodec); //audioCodeContext = ffmpeg.avcodec_alloc_context3(audioCodec); //if (HWDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) //{ // ffmpeg.av_hwdevice_ctx_create(&videoCodecContext->hw_device_ctx, HWDeviceType, null, null, 0).ThrowExceptionIfError(); //} //ffmpeg.avcodec_parameters_to_context(videoCodecContext, _iFormatContext->streams[videoStreamIndex]->codecpar).ThrowExceptionIfError(); // 동영상 파일에 있는 정보가 컨텍스트에 복사되고 없는 정보는 코덱의 원래 정보가 유지된다. 간단한 코덱은 별도의 옵션이 없지만 고성능 코덱은 동작에 필요한 필수 옵션이 있다. 이 정보를 복사하지 않으면 코덱이 제대로 동작하지 않아 일부 파일이 열리지 않는다. 다음 함수는 코덱을 열어 사용할 준비를 하고 컨텍스트도 코덱에 맞게 초기화한다. //ffmpeg.avcodec_parameters_to_context(audioCodeContext, _iFormatContext->streams[audioStreamIndex]->codecpar).ThrowExceptionIfError(); //ffmpeg.avcodec_open2(videoCodecContext, videoCodec, null).ThrowExceptionIfError(); // 세번째 인수는 코덱으로 전달할 옵션값이며 필요 없으면 NULL로 지정한다.여기까지 진행하면 코덱과 컨텍스트가 모두 완비되어 패킷의 압축을 풀어 프레임 정보를 만들 준비가 되었다. 코덱을 다 사용한 후 다음 함수로 컨텍스트와 관련 메모리를 모두 해제한다. //ffmpeg.avcodec_open2(audioCodeContext, audioCodec, null).ThrowExceptionIfError(); // 세번째 인수는 코덱으로 전달할 옵션값이며 필요 없으면 NULL로 지정한다.여기까지 진행하면 코덱과 컨텍스트가 모두 완비되어 패킷의 압축을 풀어 프레임 정보를 만들 준비가 되었다. 코덱을 다 사용한 후 다음 함수로 컨텍스트와 관련 메모리를 모두 해제한다. //CodecName = ffmpeg.avcodec_get_name(videoCodec->id); //AudioCodecName = ffmpeg.avcodec_get_name(audioCodec->id); //swrCtx = ffmpeg.swr_alloc(); //FrameSize = new Size(videoCodecContext->width, videoCodecContext->height); //PixelFormat = videoCodecContext->pix_fmt; ////Console.WriteLine(audioCodecName); //swrCtx_Audio = ffmpeg.swr_alloc(); //AVSampleFormat in_sample_fmt = audioCodeContext->sample_fmt; //int in_sample_rate = audioCodeContext->sample_rate; //long in_ch_layout = (long)audioCodeContext->channel_layout; //out_sample_fmt = AVSampleFormat.AV_SAMPLE_FMT_FLTP; //int out_sample_rate = 44100; //int out_ch_layout = ffmpeg.AV_CH_LAYOUT_MONO; //ffmpeg.swr_alloc_set_opts(swrCtx_Audio, out_ch_layout, out_sample_fmt, out_sample_rate, in_ch_layout, in_sample_fmt, in_sample_rate, 0, null); //ffmpeg.swr_init(swrCtx_Audio); ////Resampling setting options-------------------------------------------- ---------------end ////Get the number of output channels //out_channel_nb = ffmpeg.av_get_channel_layout_nb_channels((ulong)out_ch_layout); ////Store pcm data //out_buffer_audio = (byte*)ffmpeg.av_malloc(2 * 8000); //rawPacket = ffmpeg.av_packet_alloc(); //decodedFrame = ffmpeg.av_frame_alloc(); }
public static IEnumerable <(PointerBitmap bitmap, VideoFrameState state)> DecodeFrames(string url, AVHWDeviceType HWDevice) { _EnsureBinariesAreSet(); using (var vsd = new VideoStreamDecoder(url, HWDevice)) { var info = GetDecoderInfo(vsd); var state = new Dictionary <string, long>(); var context = new VideoFrameState(info, state); var sourceSize = vsd.FrameSize; var sourcePixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); var destinationSize = sourceSize; var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; long index = 0; using (var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat)) { while (vsd.TryDecodeNextFrame(out var frame)) { var convertedFrame = vfc.Convert(frame); state["index"] = index; state["pts"] = frame.pts; // state["pkt_pts"] = frame.pkt_pts; state["pkt_dts"] = frame.pkt_dts; state["best_effort_timestamp"] = frame.best_effort_timestamp; state["display_picture_number"] = frame.display_picture_number; state["coded_picture_number"] = frame.coded_picture_number; state["decode_error_flags"] = frame.decode_error_flags; yield return(AsPointerBitmap(convertedFrame), context); ++index; } } } }
private unsafe void DecodeAllFramesToImages(AVHWDeviceType HWDevice) { isWait = true; frames.Clear(); audioFrames.Clear(); // decode all frames from url, please not it might local resorce, e.g. string url = "../../sample_mpeg4.mp4"; //var url = "http://clips.vorwaerts-gmbh.de/big_buck_bunny.mp4"; // be advised this file holds 1440 frames //var url = "rtmp://58.200.131.2:1935/livetv/hunantv"; int curGuid = currentGuid; //FileStream fs = new FileStream("E://tt.mp3", FileMode.OpenOrCreate); using (var vsd = new AudioStreamDecoder(url, HWDevice)) { TempBufferSize = vsd.Fps / 2; StartVideoThread(vsd.Fps); StartAudioThread(); Console.WriteLine($"FPS:{vsd.Fps}"); Console.WriteLine($"codec name: {vsd.CodecName}"); var info = vsd.GetContextInfo(); info.ToList().ForEach(x => Console.WriteLine($"{x.Key} = {x.Value}")); var sourceSize = vsd.FrameSize; var sourcePixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); //var destinationSize = sourceSize; var destinationSize = new Size(1920, 1080); var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; using (VideoFrameConverter vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat)) { using (AudioFrameConverter afc = new AudioFrameConverter(vsd.in_sample_fmt, vsd.in_sample_rate, vsd.in_channels)) { naudioInit(vsd.in_sample_rate, vsd.in_channels); Stopwatch stopwatch = new Stopwatch(); while (vsd.TryDecodeNextFrame(out var frame, out bool isVideo)) { if (curGuid != currentGuid) { break; } if (isVideo) { if (!isLastMode) { while (frames.Count >= TempBufferSize) { if (curGuid != currentGuid) { break; } Thread.Sleep(1); } } //stopwatch.Start(); AVFrame convertedFrame = vfc.Convert(frame); int length = convertedFrame.height * convertedFrame.linesize[0]; byte[] managedArray = IntPrtToBytes((IntPtr)convertedFrame.data[0], length); VideoSt st = new VideoSt() { data = managedArray, width = convertedFrame.width, height = convertedFrame.height, stride = convertedFrame.linesize[0] }; frames.Add(st); if (frames.Count >= TempBufferSize) { isWait = false; } //stopwatch.Stop(); //Console.WriteLine($"解析时间:{stopwatch.ElapsedMilliseconds}毫秒"); //stopwatch.Reset(); } else { var convertedFrame = afc.Convert(frame); int length = convertedFrame.pkt_size; byte[] managedArray = new byte[0]; if (managedArray.Length != length) { managedArray = new byte[length]; } Marshal.Copy((IntPtr)convertedFrame.data[0], managedArray, 0, managedArray.Length); audioFrames.Add(new AudioSt() { data = managedArray }); } } } } } //fs.Close(); }
public MediaDecoder(Uri source, AVHWDeviceType hardwareDevice) { Source = source; HardwareDevice = hardwareDevice; Open(); }
public BGBuilder(AVHWDeviceType HWDevice, string url) { var mediaInfo = new MediaInfoDotNet.MediaFile(url); var videoInfo = mediaInfo.Video[0]; var videoFrameRate = videoInfo.frameRate; // Hz (fps) var videoFrameCount = videoInfo.frameCount; int bgFrameCount = 0; long bgFrameInterval = 5 * (long)videoInfo.frameRate; // in frames long frameCount = 0L; using (var vsd = new VideoStreamDecoder(url, HWDevice)) { var srcSize = vsd.FrameSize; var srcPixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); var dstSize = srcSize; var dstPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; var width = dstSize.Width; var height = dstSize.Height; List <Bitmap> frames = new List <Bitmap>(); using (var vfc = new VideoFrameConverter(srcSize, srcPixelFormat, dstSize, dstPixelFormat)) { // Every 30s, vote on most "unchanged" pixel colours (every 5 seconds = 6 images) and assemble the BG // Store the voted-unchanged-colour images while (vsd.TryDecodeNextFrame(out var frame)) { if (frameCount % bgFrameInterval == 0) { var convertedFrame = vfc.Convert(frame); Bitmap currImage = new Bitmap(width, height, convertedFrame.linesize[0], PixelFormat.Format24bppRgb, (IntPtr)convertedFrame.data[0]); //currImage.Save($"bg{bgFrameCount:D6}.jpg", ImageFormat.Jpeg); frames.Add(new Bitmap(currImage)); int numFrames = 20; if (frames.Count == numFrames) { Rectangle rect = new Rectangle(0, 0, currImage.Width, currImage.Height); Bitmap bgDst = new Bitmap(rect.Width, rect.Height, PixelFormat.Format24bppRgb); BitmapData bgDstData = bgDst.LockBits(rect, ImageLockMode.WriteOnly, PixelFormat.Format24bppRgb); BitmapData[] bmpDatas = new BitmapData[numFrames]; byte *[] p = new byte *[numFrames]; int colourDen = 12; int colourDim = 256 / colourDen; for (int i = 0; i < numFrames; i++) { bmpDatas[i] = frames[i].LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb); p[i] = (byte *)bmpDatas[i].Scan0.ToPointer(); } byte *d = (byte *)bgDstData.Scan0.ToPointer(); for (int yi = 0; yi < rect.Height; yi++) { for (int xi = 0; xi < rect.Width; xi++, d += 3) { SortedDictionary <int, List <Color> > votes = new SortedDictionary <int, List <Color> >(); for (int k = 0; k < numFrames; k++) { Color c = Color.FromArgb(*(p[k] + 0), *(p[k] + 1), *(p[k] + 2)); // 3D Color-space 256/3 ^3 int colorKey = (c.R / colourDen) + (c.G / colourDen) * colourDim + (c.B / colourDen) * colourDim * colourDim; if (!votes.ContainsKey(colorKey)) { votes[colorKey] = new List <Color>(); } votes[colorKey].Add(c); p[k] += 3; } int maxVotes = int.MinValue; int keyOfMaxVotes = -1; foreach (var key in votes.Keys) { if (votes[key].Count > maxVotes) { maxVotes = votes[key].Count; keyOfMaxVotes = key; } } int[] sum = new int[3]; for (int k = 0; k < 3; k++) { sum[k] = 0; } for (int k = 0; k < votes[keyOfMaxVotes].Count; k++) { sum[0] += votes[keyOfMaxVotes][k].R; sum[1] += votes[keyOfMaxVotes][k].G; sum[2] += votes[keyOfMaxVotes][k].B; } //if (votes[keyOfMaxVotes].Count != numFrames) // Console.WriteLine("Interesting"); *(d + 0) = (byte)((float)sum[0] / (float)votes[keyOfMaxVotes].Count); *(d + 1) = (byte)((float)sum[1] / (float)votes[keyOfMaxVotes].Count); *(d + 2) = (byte)((float)sum[2] / (float)votes[keyOfMaxVotes].Count); } } for (int i = 0; i < 6; i++) { frames[i].UnlockBits(bmpDatas[i]); } bgDst.UnlockBits(bgDstData); bgDst.Save($"bgc{frameCount:D06}.jpg", ImageFormat.Jpeg); frames.Clear(); } bgFrameCount++; } frameCount++; } } } }
private unsafe void ProcessFrames(AVHWDeviceType HWDevice, string url, long startFrame = 0, long endFrame = long.MaxValue) { _videoFilepath = url; _processingEnabled = true; _perfTimer = new FPSTimer((int)_videoFrameRate); long leadInFrames = 5; if (!_processMultithreaded && leadInFrames < startFrame) // dont't do leadIn if we're too close to the start of the video { startFrame = startFrame - leadInFrames; } else { leadInFrames = 0L; } // Don't set endFrame in most cases since we want to be able to seek to end and not have this loop exit //if (endFrame > _videoFrameCount) // endFrame = _videoFrameCount - 1; using (var vsd = new VideoStreamDecoder(url, HWDevice)) { Console.WriteLine($"codec name: {vsd.CodecName}"); var info = vsd.GetContextInfo(); info.ToList().ForEach(x => Console.WriteLine($"{x.Key} = {x.Value}")); var sourceSize = vsd.FrameSize; var sourcePixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); var destinationSize = sourceSize; var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; int framesSinceSeek = 0; int framesSinceSeekThresh = 5; using (var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat)) { _frameNumber = startFrame; if (startFrame != 0) { vsd.Seek(startFrame); } //byte[] currFrameData = new byte[destinationSize.Width * destinationSize.Height * 3]; //byte[] prevFrameData = new byte[destinationSize.Width * destinationSize.Height * 3]; var width = destinationSize.Width; var height = destinationSize.Height; Image <Bgr, byte> prevImage = new Image <Bgr, byte>(width, height); //Image Class from Emgu.CV //FrameBlender[] backgroundBuilders = new FrameBlender[processSettings.backgroundFrameBlendInterval]; //Bitmap[] bgs = new Bitmap[processSettings.backgroundFrameBlendInterval]; //for (int i = 0; i < backgroundBuilders.Length; i++) { // backgroundBuilders[i] = new FrameBlender(width, height, processSettings.backgroundFrameBlendCount); // bgs[i] = new Bitmap(width, height, PixelFormat.Format32bppArgb); //} //FrameBlender frameSmoother = new FrameBlender(width, height, _processSettings.frameBlendCount); Image <Bgra, float> background = null; //var bgBuilder = new Emgu.CV.BackgroundSubtractorMOG2(500, 16, false); var bgBuilder = new Emgu.CV.BackgroundSubtractorKNN(500, 4.0, false); Image <Gray, byte> foregroundMask = new Image <Gray, byte>(width, height); var currForeground = new Image <Bgr, byte>(width, height); var prevForeground = new Image <Bgr, byte>(width, height); //var movement = new Image<Gray, byte>(width, height); //var movementHist = new Image<Gray, byte>(width, height); bool decoderActive = true; AVFrame frame; while (decoderActive && (_frameNumber < endFrame) && _processingEnabled) { int seekFrameNum = _trackBarPos; if (!_processMultithreaded) { if (Math.Abs(_frameNumber - seekFrameNum) > 250) { vsd.Seek(seekFrameNum); _frameNumber = seekFrameNum; framesSinceSeek = 0; } var trackBarSetMI = new MethodInvoker(() => trackBar1.Value = Math.Min((int)_frameNumber, trackBar1.Maximum - 1)); trackBar1.Invoke(trackBarSetMI); } try { decoderActive = vsd.TryDecodeNextFrame(out frame); } catch { continue; } if (framesSinceSeek < framesSinceSeekThresh) { ++framesSinceSeek; } while (_processingSleep) { Thread.Sleep(500); } var convertedFrame = vfc.Convert(frame); Image <Bgr, byte> currImage = new Image <Bgr, byte>(width, height, convertedFrame.linesize[0], (IntPtr)convertedFrame.data[0]); // Shadow reduction: Shadows are lindear and less vertical, so stretch the image wider // and then resize back to original aspect-ratio to discard some horizontal detail. // Also, people are taller than bikes & balls if (_settingsControl.EnableShadowReduction) { currImage = currImage.Resize(width, height / 8, Emgu.CV.CvEnum.Inter.Area).Resize(width, height, Emgu.CV.CvEnum.Inter.Area); } // Smooth using multiple frames // Use 10% of previous frame for some speckle-noise & flicker reduction if (_settingsControl.EnableDespeckle) { currImage = currImage.SmoothGaussian(3); } currImage = (0.3 * currImage.Mat + 0.7 * prevImage.Mat).ToImage <Bgr, byte>(); //currImage = frameSmoother.Update(currImage.ToBitmap()).ToImage<Bgr, byte>(); if (!_maskSet) { using (Bitmap bmp = ShowEditMaskForm(currImage.ToBitmap(), _mask)) { _maskSet = true; if (bmp == null) { continue; } _mask = GetMatFromSDImage(bmp).ToImage <Bgra, byte>()[2]; // mask is red-channel // Clean-up and invert to form the correct mask var whiteImg = new Image <Gray, byte>(width, height); whiteImg.SetValue(255); _mask = whiteImg.Copy(_mask).Not(); } } bgBuilder.Apply(currImage, foregroundMask); if (_frameNumber == 0L) { background = currImage.Convert <Bgra, float>(); } else { Image <Bgra, float> newBg = currImage.Convert <Bgra, float>(); newBg[3].SetValue(1, foregroundMask.Not()); background[3].SetValue(1); background = (background.Mat * .97 + newBg.Mat * .03).ToImage <Bgra, float>(); //var picMI = new MethodInvoker(() => pictureBox1.Image = background.ToBitmap()); //pictureBox1.Invoke(picMI); } if (leadInFrames > 4L) { leadInFrames--; _frameNumber++; continue; // skip further processing until the lead-in is complete } else if (leadInFrames > 0L) { leadInFrames--; } Mat foregroundMat = background.Convert <Bgr, byte>().Not().Mat + currImage.Mat; currForeground = foregroundMat.ToImage <Bgr, byte>(); if (_settingsControl.EnableDespeckle) { currForeground = currForeground.SmoothGaussian(3); // remove speckle (video low-light noise, small birds, insects, etc) } //currForeground = currImage.Copy(foregroundMask.Not()); Mat moveMat = currForeground.Mat - prevForeground.Mat; _movement = (1.02 * (_movement.Mat * 0.85 + 0.15 * ((moveMat - _settingsControl.MovementNoiseFloor) * _settingsControl.MovementPixMul).ToImage <Bgr, byte>().Convert <Gray, float>().Mat)).ToImage <Gray, float>(); //var picMI = new MethodInvoker(() => pictureBox1.Image = _movement.ToBitmap()); //pictureBox1.Invoke(picMI); if (_mask != null) { _movement = _movement.Copy(_mask); } currImage = new Image <Bgr, byte>(width, height, convertedFrame.linesize[0], (IntPtr)convertedFrame.data[0]); prevImage.Bytes = currImage.Bytes; prevForeground.Bytes = currForeground.Bytes; if (leadInFrames == 0) { int currentFps = _perfTimer.Update(); double processingRate = (double)currentFps / (double)_videoFrameRate; var time = TimeSpan.FromSeconds((double)_frameNumber / (double)_videoFrameRate); // https://docs.microsoft.com/en-us/dotnet/standard/base-types/custom-timespan-format-strings statusLabel.Text = $"{time:hh\\:mm\\:ss}"; statusProcessingRate.Text = $"Processing@{ processingRate: 0.0}x"; //var moveScore = _movement.GetSum().Intensity * _settingsControl.MovementScoreMul; // *** Do perspective correction to movement score var moveScore = 0.0; for (int yi = 0; yi != _movement.Height; ++yi) { // Assume that a foreground person is 2x taller than a background person double pc = 1.0 - ((double)yi / (double)_movement.Height); // 0..1, bottom..top pc = 0.5 + 0.5 * pc; // 0.5..1.0, bottom..top pc *= 2.0; // foreground person is 2x the height of background person pc *= pc; // squared since movement~area Rectangle rowRect = new Rectangle(0, yi, _movement.Width, 1); Image <Gray, float> row = _movement.GetSubRect(rowRect); moveScore += row.GetSum().Intensity *_settingsControl.MovementScoreMul *pc; } if (framesSinceSeek == framesSinceSeekThresh) { if (_frameNumber < _movementScores.Length) { _movementScores[_frameNumber] = (float)moveScore; } _movementScoreMax = Math.Max(_movementScoreMax, (float)moveScore); UpdateChart(); // decay the maximum slowly to ensure early "noise" peaks don't destroy scaling forever _movementScoreMax *= 0.9999f; } } _processedFrameCount++; _frameNumber++; if (!_processMultithreaded && (_frameNumber < endFrame)) { // Off-load some processing to another thread to allow faster updates on the main processing thread Task.Run(() => { try { UpdateProcessMainView(currImage.Mat); } catch { } }); } if (_frameNumber == endFrame) { var mi = new MethodInvoker(() => ProcessingCompleteAndExport()); this.Invoke(mi); } } } } }
public VideoStreamDecoder(string url, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { _pFormatContext = ffmpeg.avformat_alloc_context(); _receivedFrame = ffmpeg.av_frame_alloc(); var pFormatContext = _pFormatContext; ffmpeg.avformat_open_input(&pFormatContext, url, null, null).ThrowExceptionIfError(); AVCodec * codec = null; AVBufferRef *codecBuff = null; if (HWDeviceType is AVHWDeviceType.AV_HWDEVICE_TYPE_QSV) { codec = ffmpeg.avcodec_find_decoder_by_name("h264_qsv"); codec->id = AVCodecID.AV_CODEC_ID_H264; for (int i = 0; i < _pFormatContext->nb_streams; i++) { AVStream *st = _pFormatContext->streams[i]; if (st->codecpar->codec_id == AVCodecID.AV_CODEC_ID_H264 && _videoStream == null) { _videoStream = st; Console.WriteLine("Stream founded!"); } else { st->discard = AVDiscard.AVDISCARD_ALL; } } _streamIndex = _videoStream->index; } else { _streamIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0); } _pCodecContext = ffmpeg.avcodec_alloc_context3(codec); if (HWDeviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) { if (ffmpeg.av_hwdevice_ctx_create(&_pCodecContext->hw_device_ctx, HWDeviceType, "auto", null, 0) < 0) { throw new Exception("HW device init ERROR!"); } else { Console.WriteLine("Device " + HWDeviceType + " init OK"); isHwAccelerate = true; } } if (_pCodecContext == null) { throw new Exception("Codec init error"); } ffmpeg.avformat_find_stream_info(_pFormatContext, null); //This is necessary to determine the parameters of online broadcasting if (_videoStream->codecpar->extradata != null) { int size = (int)(_videoStream->codecpar->extradata_size); _pCodecContext->extradata = (byte *)ffmpeg.av_mallocz((ulong)size + ffmpeg.AV_INPUT_BUFFER_PADDING_SIZE); _pCodecContext->extradata_size = (int)size + ffmpeg.AV_INPUT_BUFFER_PADDING_SIZE; FFmpegHelper.memcpy((IntPtr)_pCodecContext->extradata, (IntPtr)_videoStream->codecpar->extradata, size); //Or just /*for (int i = 0; i < size; i++) * _pCodecContext->extradata[i] = _videoStream->codecpar->extradata[i];*/ } if (HWDeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV) { _pCodecContext->get_format = get_fmt; } ffmpeg.avcodec_parameters_to_context(_pCodecContext, _videoStream->codecpar); ffmpeg.avcodec_open2(_pCodecContext, codec, null); CodecName = ffmpeg.avcodec_get_name(codec->id); FrameSize = new Size(_videoStream->codecpar->width, _videoStream->codecpar->height); PixelFormat = _pCodecContext->sw_pix_fmt; //It doesn't work (== AV_HWDEVICE_TYPE_NONE before the first frame) Console.WriteLine("Codec: " + CodecName.ToString()); Console.WriteLine("Size: " + FrameSize.ToString()); Console.WriteLine("PixelFormat: " + PixelFormat.ToString()); _pPacket = ffmpeg.av_packet_alloc(); _pFrame = ffmpeg.av_frame_alloc(); }
/// <summary> /// Handles the MediaOpening event of the Media control. /// </summary> /// <param name="sender">The source of the event.</param> /// <param name="e">The <see cref="MediaOpeningEventArgs"/> instance containing the event data.</param> private void OnMediaOpening(object sender, MediaOpeningEventArgs e) { // You can start off by adjusting subtitles delay // e.Options.SubtitlesDelay = TimeSpan.FromSeconds(7); // See issue #216 // Example of automatically side-loading SRT subs try { var inputUrl = e.Info.InputUrl; var url = new Uri(inputUrl); if (url.IsFile || url.IsUnc) { inputUrl = System.IO.Path.ChangeExtension(url.LocalPath, "srt"); if (System.IO.File.Exists(inputUrl)) { e.Options.SubtitlesUrl = inputUrl; } } } catch { } // You can force video FPS if necessary // see: https://github.com/unosquare/ffmediaelement/issues/212 // e.Options.VideoForcedFps = 25; // An example of specifcally selecting a subtitle stream var subtitleStreams = e.Info.Streams.Where(kvp => kvp.Value.CodecType == AVMediaType.AVMEDIA_TYPE_SUBTITLE).Select(kvp => kvp.Value); var englishSubtitleStream = subtitleStreams.FirstOrDefault(s => s.Language != null && s.Language.ToLowerInvariant().StartsWith("en")); if (englishSubtitleStream != null) { e.Options.SubtitleStream = englishSubtitleStream; } // An example of specifcally selecting an audio stream var audioStreams = e.Info.Streams.Where(kvp => kvp.Value.CodecType == AVMediaType.AVMEDIA_TYPE_AUDIO).Select(kvp => kvp.Value); var englishAudioStream = audioStreams.FirstOrDefault(s => s.Language != null && s.Language.ToLowerInvariant().StartsWith("en")); if (englishAudioStream != null) { e.Options.AudioStream = englishAudioStream; } // Setting Advanced Video Stream Options is also possible var videoStream = e.Options.VideoStream; if (videoStream != null) { // Check if the video requires deinterlacing var requiresDeinterlace = videoStream.FieldOrder != AVFieldOrder.AV_FIELD_PROGRESSIVE && videoStream.FieldOrder != AVFieldOrder.AV_FIELD_UNKNOWN; // Hardwrae device priorities var deviceCandidates = new AVHWDeviceType[] { AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA, AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA, AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2 }; // Hardware device selection if (videoStream.FPS <= 30) { foreach (var deviceType in deviceCandidates) { var accelerator = videoStream.HardwareDevices.FirstOrDefault(d => d.DeviceType == deviceType); if (accelerator != null) { if (GuiContext.Current.IsInDebugMode) { e.Options.VideoHardwareDevice = accelerator; } break; } } } // Start building a video filter var videoFilter = new StringBuilder(); // The yadif filter deinterlaces the video; we check the field order if we need // to deinterlace the video automatically if (requiresDeinterlace) { videoFilter.Append("yadif,"); } // Scale down to maximum 1080p screen resolution. if (videoStream.PixelHeight > 1080) { // e.Options.VideoHardwareDevice = null; videoFilter.Append($"scale=-1:1080,"); } e.Options.VideoFilter = videoFilter.ToString().TrimEnd(','); // Since the MediaElement control belongs to a different thread // we have to set properties on its UI thread. GuiContext.Current.EnqueueInvoke(() => { Media.ClosedCaptionsChannel = videoStream.HasClosedCaptions ? CaptionsChannel.CC1 : CaptionsChannel.CCP; }); } // e.Options.AudioFilter = "aecho=0.8:0.9:1000:0.3"; // e.Options.AudioFilter = "chorus=0.5:0.9:50|60|40:0.4|0.32|0.3:0.25|0.4|0.3:2|2.3|1.3"; }