public unsafe IDecodedVideoFrame TryDecode(RawVideoFrame rawVideoFrame)
{
fixed(byte *rawBufferPtr = &rawVideoFrame.FrameSegment.Array[rawVideoFrame.FrameSegment.Offset])
{
int resultCode;
if (rawVideoFrame is RawH264IFrame rawH264IFrame)
{
if (rawH264IFrame.SpsPpsSegment.Array != null &&
!_extraData.SequenceEqual(rawH264IFrame.SpsPpsSegment))
{
if (_extraData.Length != rawH264IFrame.SpsPpsSegment.Count)
{
_extraData = new byte[rawH264IFrame.SpsPpsSegment.Count];
}
Buffer.BlockCopy(rawH264IFrame.SpsPpsSegment.Array, rawH264IFrame.SpsPpsSegment.Offset,
_extraData, 0, rawH264IFrame.SpsPpsSegment.Count);
fixed(byte *initDataPtr = &_extraData[0])
{
resultCode = FFmpegVideoPInvoke.SetVideoDecoderExtraData(_decoderHandle,
(IntPtr)initDataPtr, _extraData.Length);
if (resultCode != 0)
{
throw new DecoderException(
$"An error occurred while setting video extra data, {_videoCodecId} codec, code: {resultCode}");
}
}
}
}
lock (disposalLock) {
if (_disposed)
{
Console.WriteLine("Skipped decoding video frame, as decoder was disposed. (Therefore the frame probably wasn't wanted)");
return(null);
}
resultCode = FFmpegVideoPInvoke.DecodeFrame(_decoderHandle, (IntPtr)rawBufferPtr,
rawVideoFrame.FrameSegment.Count,
out int width, out int height, out FFmpegPixelFormat pixelFormat);
if (resultCode != 0)
{
return(null);
}
if (_currentFrameParameters.Width != width || _currentFrameParameters.Height != height ||
_currentFrameParameters.PixelFormat != pixelFormat)
{
_currentFrameParameters = new DecodedVideoFrameParameters(width, height, pixelFormat);
DropAllVideoScalers();
}
}
return(new DecodedVideoFrame(TransformTo));
}
}
public unsafe bool TryDecode(RawVideoFrame rawVideoFrame, out DecodedVideoFrameParameters frameParameters)
{
Debug.Assert(rawVideoFrame.FrameSegment.Array != null, "rawVideoFrame.FrameSegment.Array != null");
fixed(byte *rawBufferPtr = &rawVideoFrame.FrameSegment.Array[rawVideoFrame.FrameSegment.Offset])
{
int resultCode;
if (rawVideoFrame is RawH264IFrame rawH264IFrame)
{
if (rawH264IFrame.SpsPpsSegment.Array != null &&
!_extraData.SequenceEqual(rawH264IFrame.SpsPpsSegment))
{
if (_extraData.Length != rawH264IFrame.SpsPpsSegment.Count)
{
_extraData = new byte[rawH264IFrame.SpsPpsSegment.Count];
}
Buffer.BlockCopy(rawH264IFrame.SpsPpsSegment.Array, rawH264IFrame.SpsPpsSegment.Offset,
_extraData, 0, rawH264IFrame.SpsPpsSegment.Count);
fixed(byte *initDataPtr = &_extraData[0])
{
resultCode = FFmpegVideoPInvoke.SetVideoDecoderExtraData(_decoderHandle,
(IntPtr)initDataPtr, _extraData.Length);
if (resultCode != 0)
{
throw new DecoderException(
$"An error occurred while setting video extra data, {_videoCodecId} codec, code: {resultCode}");
}
}
}
}
resultCode = FFmpegVideoPInvoke.DecodeFrame(_decoderHandle, (IntPtr)rawBufferPtr,
rawVideoFrame.FrameSegment.Count,
out int width, out int height, out FFmpegPixelFormat pixelFormat);
if (resultCode != 0)
{
frameParameters = null;
return(false);
}
_currentDecodedFrameTimestamp = rawVideoFrame.Timestamp;
if (_currentFrameParameters.Width != width || _currentFrameParameters.Height != height ||
_currentFrameParameters.PixelFormat != pixelFormat)
{
_currentFrameParameters = new DecodedVideoFrameParameters(width, height, pixelFormat);
DropAllVideoScalers();
}
frameParameters = _currentFrameParameters;
return(true);
}
}
private void OnFrameReceived(object sender, IDecodedVideoFrame decodedFrame)
{
if (!decodedFrame.FrameParameters.Equals(_frameParameters))
{
_frameParameters = decodedFrame.FrameParameters;
_transformParameters = new TransformParameters(System.Drawing.RectangleF.Empty,
new System.Drawing.Size(_frameParameters.Width, _frameParameters.Height),
ScalingPolicy.Stretch, PixelFormat.Bgr24, ScalingQuality.FastBilinear);
_cvBitmap = new Image <Bgr, byte>(_frameParameters.Width, _frameParameters.Height);
FrameSizeChanged?.Invoke(this, new Tuple <double, double>(_frameParameters.Width, _frameParameters.Height));
}
if (_yoloWrapper.IsYoloReady())
{
decodedFrame.TransformTo(_cvBitmap.Mat.DataPointer, _cvBitmap.Mat.Cols * _cvBitmap.Mat.ElementSize, _transformParameters);
_yoloWrapper.FrameIn(this, _cvBitmap.Mat);
}
}
/// <exception cref="DecoderException"></exception>
public static FFmpegDecodedVideoScaler Create(DecodedVideoFrameParameters decodedVideoFrameParameters,
PostVideoDecodingParameters postVideoDecodingParameters)
{
if (decodedVideoFrameParameters == null)
{
throw new ArgumentNullException(nameof(decodedVideoFrameParameters));
}
if (postVideoDecodingParameters == null)
{
throw new ArgumentNullException(nameof(postVideoDecodingParameters));
}
int sourceLeft = 0;
int sourceTop = 0;
int sourceWidth = decodedVideoFrameParameters.Width;
int sourceHeight = decodedVideoFrameParameters.Height;
int scaledWidth = decodedVideoFrameParameters.Width;
int scaledHeight = decodedVideoFrameParameters.Height;
if (!postVideoDecodingParameters.RegionOfInterest.IsEmpty)
{
sourceLeft =
(int)(decodedVideoFrameParameters.Width * postVideoDecodingParameters.RegionOfInterest.Left);
sourceTop =
(int)(decodedVideoFrameParameters.Height * postVideoDecodingParameters.RegionOfInterest.Top);
sourceWidth =
(int)(decodedVideoFrameParameters.Width * postVideoDecodingParameters.RegionOfInterest.Width);
sourceHeight =
(int)(decodedVideoFrameParameters.Height * postVideoDecodingParameters.RegionOfInterest.Height);
}
if (!postVideoDecodingParameters.TargetFrameSize.IsEmpty)
{
scaledWidth = postVideoDecodingParameters.TargetFrameSize.Width;
scaledHeight = postVideoDecodingParameters.TargetFrameSize.Height;
ScalingPolicy scalingPolicy = postVideoDecodingParameters.ScalePolicy;
float srcAspectRatio = (float)sourceWidth / sourceHeight;
float destAspectRatio = (float)scaledWidth / scaledHeight;
if (scalingPolicy == ScalingPolicy.Auto)
{
float relativeChange = Math.Abs(srcAspectRatio - destAspectRatio) / srcAspectRatio;
scalingPolicy = relativeChange > MaxAspectRatioError
? ScalingPolicy.RespectAspectRatio
: ScalingPolicy.Stretch;
}
if (scalingPolicy == ScalingPolicy.RespectAspectRatio)
{
if (destAspectRatio < srcAspectRatio)
{
scaledHeight = sourceHeight * scaledWidth / sourceWidth;
}
else
{
scaledWidth = sourceWidth * scaledHeight / sourceHeight;
}
}
}
PixelFormat scaledPixelFormat = postVideoDecodingParameters.TargetFormat;
FFmpegPixelFormat scaledFFmpegPixelFormat = GetFFmpegPixelFormat(scaledPixelFormat);
FFmpegScalingQuality scaleQuality = GetFFmpegScaleQuality(postVideoDecodingParameters.ScaleQuality);
int resultCode = FFmpegVideoPInvoke.CreateVideoScaler(sourceLeft, sourceTop, sourceWidth, sourceHeight,
decodedVideoFrameParameters.PixelFormat,
scaledWidth, scaledHeight, scaledFFmpegPixelFormat, scaleQuality, out var handle);
if (resultCode != 0)
{
throw new DecoderException(@"An error is occurred while creating scaler, code: {resultCode}");
}
return(new FFmpegDecodedVideoScaler(handle, scaledWidth, scaledHeight, scaledPixelFormat));
}
protected bool Equals(DecodedVideoFrameParameters other)
{
return(Width == other.Width && Height == other.Height && PixelFormat == other.PixelFormat);
}
public DecodedVideoFrame(DateTime timestamp, DecodedVideoFrameParameters decodedVideoFrameParameters, Action <IntPtr, int, TransformParameters> transformAction)
{
this.FrameParameters = decodedVideoFrameParameters;
this.Timestamp = timestamp;
_transformAction = transformAction;
}
public unsafe IDecodedVideoFrame TryDecode(RawVideoFrame rawVideoFrame)
{
fixed(byte *rawBufferPtr = &rawVideoFrame.FrameSegment.Array[rawVideoFrame.FrameSegment.Offset])
{
int resultCode;
if (rawVideoFrame is RawH264IFrame rawH264IFrame)
{
if (rawH264IFrame.SpsPpsSegment.Array != null &&
!_extraData.SequenceEqual(rawH264IFrame.SpsPpsSegment))
{
if (_extraData.Length != rawH264IFrame.SpsPpsSegment.Count)
{
_extraData = new byte[rawH264IFrame.SpsPpsSegment.Count];
}
Buffer.BlockCopy(rawH264IFrame.SpsPpsSegment.Array, rawH264IFrame.SpsPpsSegment.Offset,
_extraData, 0, rawH264IFrame.SpsPpsSegment.Count);
fixed(byte *initDataPtr = &_extraData[0])
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
resultCode = FFmpegVideoPInvokeWin.SetVideoDecoderExtraData(_decoderHandle, (IntPtr)initDataPtr, _extraData.Length);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
resultCode = FFmpegVideoPInvokeLinux.SetVideoDecoderExtraData(_decoderHandle, (IntPtr)initDataPtr, _extraData.Length);
}
else
{
throw new PlatformNotSupportedException();
}
if (resultCode != 0)
{
throw new DecoderException(
$"TryDecode say: An error occurred while setting video extra data, {_videoCodecId} codec, code: {resultCode}");
}
}
}
}
FFmpegPixelFormat pixelFormat;
int width, height;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
resultCode = FFmpegVideoPInvokeWin.DecodeFrame(_decoderHandle, (IntPtr)rawBufferPtr,
rawVideoFrame.FrameSegment.Count, out width, out height,
out pixelFormat);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
resultCode = FFmpegVideoPInvokeLinux.DecodeFrame(_decoderHandle, (IntPtr)rawBufferPtr,
rawVideoFrame.FrameSegment.Count, out width, out height,
out pixelFormat);
}
else
{
throw new PlatformNotSupportedException();
}
if (resultCode != 0)
{
return(null);
}
if (_currentFrameParameters.Width != width || _currentFrameParameters.Height != height ||
_currentFrameParameters.PixelFormat != pixelFormat)
{
_currentFrameParameters = new DecodedVideoFrameParameters(width, height, pixelFormat);
DropAllVideoScalers();
}
return(new DecodedVideoFrame(TransformTo));
}
}
public DecodedVideoFrame(Action <IntPtr, int, TransformParameters> transformAction, DecodedVideoFrameParameters frameParameters)
{
this.transformAction = transformAction ?? throw new ArgumentNullException(nameof(transformAction));
FrameParameters = frameParameters ?? throw new ArgumentNullException(nameof(frameParameters));
}
/// <exception cref="DecoderException"></exception>
public static FFmpegDecodedVideoScaler Create(DecodedVideoFrameParameters decodedVideoFrameParameters,
TransformParameters transformParameters)
{
if (decodedVideoFrameParameters == null)
{
throw new ArgumentNullException(nameof(decodedVideoFrameParameters));
}
if (transformParameters == null)
{
throw new ArgumentNullException(nameof(transformParameters));
}
int sourceLeft = 0;
int sourceTop = 0;
int sourceWidth = decodedVideoFrameParameters.Width;
int sourceHeight = decodedVideoFrameParameters.Height;
int scaledWidth = decodedVideoFrameParameters.Width;
int scaledHeight = decodedVideoFrameParameters.Height;
if (!transformParameters.RegionOfInterest.IsEmpty)
{
sourceLeft =
(int)(decodedVideoFrameParameters.Width * transformParameters.RegionOfInterest.Left);
sourceTop =
(int)(decodedVideoFrameParameters.Height * transformParameters.RegionOfInterest.Top);
sourceWidth =
(int)(decodedVideoFrameParameters.Width * transformParameters.RegionOfInterest.Width);
sourceHeight =
(int)(decodedVideoFrameParameters.Height * transformParameters.RegionOfInterest.Height);
}
if (!transformParameters.TargetFrameSize.IsEmpty)
{
scaledWidth = transformParameters.TargetFrameSize.Width;
scaledHeight = transformParameters.TargetFrameSize.Height;
ScalingPolicy scalingPolicy = transformParameters.ScalePolicy;
float srcAspectRatio = (float)sourceWidth / sourceHeight;
float destAspectRatio = (float)scaledWidth / scaledHeight;
if (scalingPolicy == ScalingPolicy.Auto)
{
float relativeChange = Math.Abs(srcAspectRatio - destAspectRatio) / srcAspectRatio;
scalingPolicy = relativeChange > MaxAspectRatioError
? ScalingPolicy.RespectAspectRatio
: ScalingPolicy.Stretch;
}
if (scalingPolicy == ScalingPolicy.RespectAspectRatio)
{
if (destAspectRatio < srcAspectRatio)
{
scaledHeight = sourceHeight * scaledWidth / sourceWidth;
}
else
{
scaledWidth = sourceWidth * scaledHeight / sourceHeight;
}
}
}
PixelFormat scaledPixelFormat = transformParameters.TargetFormat;
FFmpegPixelFormat scaledFFmpegPixelFormat = GetFFmpegPixelFormat(scaledPixelFormat);
FFmpegScalingQuality scaleQuality = GetFFmpegScaleQuality(transformParameters.ScaleQuality);
int resultCode;
IntPtr handle;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
resultCode = FFmpegVideoPInvokeWin.CreateVideoScaler(sourceLeft, sourceTop, sourceWidth, sourceHeight,
decodedVideoFrameParameters.PixelFormat,
scaledWidth, scaledHeight, scaledFFmpegPixelFormat, scaleQuality, out handle);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
resultCode = FFmpegVideoPInvokeLinux.CreateVideoScaler(sourceLeft, sourceTop, sourceWidth, sourceHeight,
decodedVideoFrameParameters.PixelFormat,
scaledWidth, scaledHeight, scaledFFmpegPixelFormat, scaleQuality, out handle);
}
else
{
throw new PlatformNotSupportedException();
}
if (resultCode != 0)
{
throw new DecoderException(@"An error occurred while creating scaler, code: {resultCode}");
}
return(new FFmpegDecodedVideoScaler(handle, scaledWidth, scaledHeight, scaledPixelFormat));
}
public DecodedVideoFrame(Action <IntPtr, int, TransformParameters> transformAction, DecodedVideoFrameParameters parameters)
{
Parameters = parameters;
_transformAction = transformAction;
}
