public BGBuilder(AVHWDeviceType HWDevice, string url)
{
var mediaInfo = new MediaInfoDotNet.MediaFile(url);
var videoInfo = mediaInfo.Video[0];
var videoFrameRate = videoInfo.frameRate; // Hz (fps)
var videoFrameCount = videoInfo.frameCount;
int bgFrameCount = 0;
long bgFrameInterval = 5 * (long)videoInfo.frameRate; // in frames
long frameCount = 0L;
using (var vsd = new VideoStreamDecoder(url, HWDevice))
{
var srcSize = vsd.FrameSize;
var srcPixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice);
var dstSize = srcSize;
var dstPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24;
var width = dstSize.Width;
var height = dstSize.Height;
List <Bitmap> frames = new List <Bitmap>();
using (var vfc = new VideoFrameConverter(srcSize, srcPixelFormat, dstSize, dstPixelFormat))
{
// Every 30s, vote on most "unchanged" pixel colours (every 5 seconds = 6 images) and assemble the BG
// Store the voted-unchanged-colour images
while (vsd.TryDecodeNextFrame(out var frame))
{
if (frameCount % bgFrameInterval == 0)
{
var convertedFrame = vfc.Convert(frame);
Bitmap currImage = new Bitmap(width, height, convertedFrame.linesize[0], PixelFormat.Format24bppRgb, (IntPtr)convertedFrame.data[0]);
//currImage.Save($"bg{bgFrameCount:D6}.jpg", ImageFormat.Jpeg);
frames.Add(new Bitmap(currImage));
int numFrames = 20;
if (frames.Count == numFrames)
{
Rectangle rect = new Rectangle(0, 0, currImage.Width, currImage.Height);
Bitmap bgDst = new Bitmap(rect.Width, rect.Height, PixelFormat.Format24bppRgb);
BitmapData bgDstData = bgDst.LockBits(rect, ImageLockMode.WriteOnly, PixelFormat.Format24bppRgb);
BitmapData[] bmpDatas = new BitmapData[numFrames];
byte *[] p = new byte *[numFrames];
int colourDen = 12;
int colourDim = 256 / colourDen;
for (int i = 0; i < numFrames; i++)
{
bmpDatas[i] = frames[i].LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
p[i] = (byte *)bmpDatas[i].Scan0.ToPointer();
}
byte *d = (byte *)bgDstData.Scan0.ToPointer();
for (int yi = 0; yi < rect.Height; yi++)
{
for (int xi = 0; xi < rect.Width; xi++, d += 3)
{
SortedDictionary <int, List <Color> > votes = new SortedDictionary <int, List <Color> >();
for (int k = 0; k < numFrames; k++)
{
Color c = Color.FromArgb(*(p[k] + 0), *(p[k] + 1), *(p[k] + 2));
// 3D Color-space 256/3 ^3
int colorKey = (c.R / colourDen) +
(c.G / colourDen) * colourDim +
(c.B / colourDen) * colourDim * colourDim;
if (!votes.ContainsKey(colorKey))
{
votes[colorKey] = new List <Color>();
}
votes[colorKey].Add(c);
p[k] += 3;
}
int maxVotes = int.MinValue;
int keyOfMaxVotes = -1;
foreach (var key in votes.Keys)
{
if (votes[key].Count > maxVotes)
{
maxVotes = votes[key].Count;
keyOfMaxVotes = key;
}
}
int[] sum = new int[3];
for (int k = 0; k < 3; k++)
{
sum[k] = 0;
}
for (int k = 0; k < votes[keyOfMaxVotes].Count; k++)
{
sum[0] += votes[keyOfMaxVotes][k].R;
sum[1] += votes[keyOfMaxVotes][k].G;
sum[2] += votes[keyOfMaxVotes][k].B;
}
//if (votes[keyOfMaxVotes].Count != numFrames)
// Console.WriteLine("Interesting");
*(d + 0) = (byte)((float)sum[0] / (float)votes[keyOfMaxVotes].Count);
*(d + 1) = (byte)((float)sum[1] / (float)votes[keyOfMaxVotes].Count);
*(d + 2) = (byte)((float)sum[2] / (float)votes[keyOfMaxVotes].Count);
}
}
for (int i = 0; i < 6; i++)
{
frames[i].UnlockBits(bmpDatas[i]);
}
bgDst.UnlockBits(bgDstData);
bgDst.Save($"bgc{frameCount:D06}.jpg", ImageFormat.Jpeg);
frames.Clear();
}
bgFrameCount++;
}
frameCount++;
}
}
}
}
private unsafe void ProcessFrames(AVHWDeviceType HWDevice, string url, long startFrame = 0, long endFrame = long.MaxValue)
{
_videoFilepath = url;
_processingEnabled = true;
_perfTimer = new FPSTimer((int)_videoFrameRate);
long leadInFrames = 5;
if (!_processMultithreaded && leadInFrames < startFrame) // dont't do leadIn if we're too close to the start of the video
{
startFrame = startFrame - leadInFrames;
}
else
{
leadInFrames = 0L;
}
// Don't set endFrame in most cases since we want to be able to seek to end and not have this loop exit
//if (endFrame > _videoFrameCount)
// endFrame = _videoFrameCount - 1;
using (var vsd = new VideoStreamDecoder(url, HWDevice))
{
Console.WriteLine($"codec name: {vsd.CodecName}");
var info = vsd.GetContextInfo();
info.ToList().ForEach(x => Console.WriteLine($"{x.Key} = {x.Value}"));
var sourceSize = vsd.FrameSize;
var sourcePixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice);
var destinationSize = sourceSize;
var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24;
int framesSinceSeek = 0;
int framesSinceSeekThresh = 5;
using (var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat))
{
_frameNumber = startFrame;
if (startFrame != 0)
{
vsd.Seek(startFrame);
}
//byte[] currFrameData = new byte[destinationSize.Width * destinationSize.Height * 3];
//byte[] prevFrameData = new byte[destinationSize.Width * destinationSize.Height * 3];
var width = destinationSize.Width;
var height = destinationSize.Height;
Image <Bgr, byte> prevImage = new Image <Bgr, byte>(width, height); //Image Class from Emgu.CV
//FrameBlender[] backgroundBuilders = new FrameBlender[processSettings.backgroundFrameBlendInterval];
//Bitmap[] bgs = new Bitmap[processSettings.backgroundFrameBlendInterval];
//for (int i = 0; i < backgroundBuilders.Length; i++) {
// backgroundBuilders[i] = new FrameBlender(width, height, processSettings.backgroundFrameBlendCount);
// bgs[i] = new Bitmap(width, height, PixelFormat.Format32bppArgb);
//}
//FrameBlender frameSmoother = new FrameBlender(width, height, _processSettings.frameBlendCount);
Image <Bgra, float> background = null;
//var bgBuilder = new Emgu.CV.BackgroundSubtractorMOG2(500, 16, false);
var bgBuilder = new Emgu.CV.BackgroundSubtractorKNN(500, 4.0, false);
Image <Gray, byte> foregroundMask = new Image <Gray, byte>(width, height);
var currForeground = new Image <Bgr, byte>(width, height);
var prevForeground = new Image <Bgr, byte>(width, height);
//var movement = new Image<Gray, byte>(width, height);
//var movementHist = new Image<Gray, byte>(width, height);
bool decoderActive = true;
AVFrame frame;
while (decoderActive && (_frameNumber < endFrame) && _processingEnabled)
{
int seekFrameNum = _trackBarPos;
if (!_processMultithreaded)
{
if (Math.Abs(_frameNumber - seekFrameNum) > 250)
{
vsd.Seek(seekFrameNum);
_frameNumber = seekFrameNum;
framesSinceSeek = 0;
}
var trackBarSetMI = new MethodInvoker(() => trackBar1.Value = Math.Min((int)_frameNumber, trackBar1.Maximum - 1));
trackBar1.Invoke(trackBarSetMI);
}
try
{
decoderActive = vsd.TryDecodeNextFrame(out frame);
}
catch
{
continue;
}
if (framesSinceSeek < framesSinceSeekThresh)
{
++framesSinceSeek;
}
while (_processingSleep)
{
Thread.Sleep(500);
}
var convertedFrame = vfc.Convert(frame);
Image <Bgr, byte> currImage = new Image <Bgr, byte>(width, height, convertedFrame.linesize[0], (IntPtr)convertedFrame.data[0]);
// Shadow reduction: Shadows are lindear and less vertical, so stretch the image wider
// and then resize back to original aspect-ratio to discard some horizontal detail.
// Also, people are taller than bikes & balls
if (_settingsControl.EnableShadowReduction)
{
currImage = currImage.Resize(width, height / 8, Emgu.CV.CvEnum.Inter.Area).Resize(width, height, Emgu.CV.CvEnum.Inter.Area);
}
// Smooth using multiple frames
// Use 10% of previous frame for some speckle-noise & flicker reduction
if (_settingsControl.EnableDespeckle)
{
currImage = currImage.SmoothGaussian(3);
}
currImage = (0.3 * currImage.Mat + 0.7 * prevImage.Mat).ToImage <Bgr, byte>();
//currImage = frameSmoother.Update(currImage.ToBitmap()).ToImage<Bgr, byte>();
if (!_maskSet)
{
using (Bitmap bmp = ShowEditMaskForm(currImage.ToBitmap(), _mask))
{
_maskSet = true;
if (bmp == null)
{
continue;
}
_mask = GetMatFromSDImage(bmp).ToImage <Bgra, byte>()[2]; // mask is red-channel
// Clean-up and invert to form the correct mask
var whiteImg = new Image <Gray, byte>(width, height);
whiteImg.SetValue(255);
_mask = whiteImg.Copy(_mask).Not();
}
}
bgBuilder.Apply(currImage, foregroundMask);
if (_frameNumber == 0L)
{
background = currImage.Convert <Bgra, float>();
}
else
{
Image <Bgra, float> newBg = currImage.Convert <Bgra, float>();
newBg[3].SetValue(1, foregroundMask.Not());
background[3].SetValue(1);
background = (background.Mat * .97 + newBg.Mat * .03).ToImage <Bgra, float>();
//var picMI = new MethodInvoker(() => pictureBox1.Image = background.ToBitmap());
//pictureBox1.Invoke(picMI);
}
if (leadInFrames > 4L)
{
leadInFrames--;
_frameNumber++;
continue; // skip further processing until the lead-in is complete
}
else if (leadInFrames > 0L)
{
leadInFrames--;
}
Mat foregroundMat = background.Convert <Bgr, byte>().Not().Mat + currImage.Mat;
currForeground = foregroundMat.ToImage <Bgr, byte>();
if (_settingsControl.EnableDespeckle)
{
currForeground = currForeground.SmoothGaussian(3); // remove speckle (video low-light noise, small birds, insects, etc)
}
//currForeground = currImage.Copy(foregroundMask.Not());
Mat moveMat = currForeground.Mat - prevForeground.Mat;
_movement = (1.02 * (_movement.Mat * 0.85 + 0.15 * ((moveMat - _settingsControl.MovementNoiseFloor) * _settingsControl.MovementPixMul).ToImage <Bgr, byte>().Convert <Gray, float>().Mat)).ToImage <Gray, float>();
//var picMI = new MethodInvoker(() => pictureBox1.Image = _movement.ToBitmap());
//pictureBox1.Invoke(picMI);
if (_mask != null)
{
_movement = _movement.Copy(_mask);
}
currImage = new Image <Bgr, byte>(width, height, convertedFrame.linesize[0], (IntPtr)convertedFrame.data[0]);
prevImage.Bytes = currImage.Bytes;
prevForeground.Bytes = currForeground.Bytes;
if (leadInFrames == 0)
{
int currentFps = _perfTimer.Update();
double processingRate = (double)currentFps / (double)_videoFrameRate;
var time = TimeSpan.FromSeconds((double)_frameNumber / (double)_videoFrameRate);
// https://docs.microsoft.com/en-us/dotnet/standard/base-types/custom-timespan-format-strings
statusLabel.Text = $"{time:hh\\:mm\\:ss}";
statusProcessingRate.Text = $"Processing@{ processingRate: 0.0}x";
//var moveScore = _movement.GetSum().Intensity * _settingsControl.MovementScoreMul;
// *** Do perspective correction to movement score
var moveScore = 0.0;
for (int yi = 0; yi != _movement.Height; ++yi)
{
// Assume that a foreground person is 2x taller than a background person
double pc = 1.0 - ((double)yi / (double)_movement.Height); // 0..1, bottom..top
pc = 0.5 + 0.5 * pc; // 0.5..1.0, bottom..top
pc *= 2.0; // foreground person is 2x the height of background person
pc *= pc; // squared since movement~area
Rectangle rowRect = new Rectangle(0, yi, _movement.Width, 1);
Image <Gray, float> row = _movement.GetSubRect(rowRect);
moveScore += row.GetSum().Intensity *_settingsControl.MovementScoreMul *pc;
}
if (framesSinceSeek == framesSinceSeekThresh)
{
if (_frameNumber < _movementScores.Length)
{
_movementScores[_frameNumber] = (float)moveScore;
}
_movementScoreMax = Math.Max(_movementScoreMax, (float)moveScore);
UpdateChart();
// decay the maximum slowly to ensure early "noise" peaks don't destroy scaling forever
_movementScoreMax *= 0.9999f;
}
}
_processedFrameCount++;
_frameNumber++;
if (!_processMultithreaded && (_frameNumber < endFrame))
{
// Off-load some processing to another thread to allow faster updates on the main processing thread
Task.Run(() =>
{
try
{
UpdateProcessMainView(currImage.Mat);
}
catch { }
});
}
if (_frameNumber == endFrame)
{
var mi = new MethodInvoker(() => ProcessingCompleteAndExport());
this.Invoke(mi);
}
}
}
}
}
