public BGBuilder(AVHWDeviceType HWDevice, string url) { var mediaInfo = new MediaInfoDotNet.MediaFile(url); var videoInfo = mediaInfo.Video[0]; var videoFrameRate = videoInfo.frameRate; // Hz (fps) var videoFrameCount = videoInfo.frameCount; int bgFrameCount = 0; long bgFrameInterval = 5 * (long)videoInfo.frameRate; // in frames long frameCount = 0L; using (var vsd = new VideoStreamDecoder(url, HWDevice)) { var srcSize = vsd.FrameSize; var srcPixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); var dstSize = srcSize; var dstPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; var width = dstSize.Width; var height = dstSize.Height; List <Bitmap> frames = new List <Bitmap>(); using (var vfc = new VideoFrameConverter(srcSize, srcPixelFormat, dstSize, dstPixelFormat)) { // Every 30s, vote on most "unchanged" pixel colours (every 5 seconds = 6 images) and assemble the BG // Store the voted-unchanged-colour images while (vsd.TryDecodeNextFrame(out var frame)) { if (frameCount % bgFrameInterval == 0) { var convertedFrame = vfc.Convert(frame); Bitmap currImage = new Bitmap(width, height, convertedFrame.linesize[0], PixelFormat.Format24bppRgb, (IntPtr)convertedFrame.data[0]); //currImage.Save($"bg{bgFrameCount:D6}.jpg", ImageFormat.Jpeg); frames.Add(new Bitmap(currImage)); int numFrames = 20; if (frames.Count == numFrames) { Rectangle rect = new Rectangle(0, 0, currImage.Width, currImage.Height); Bitmap bgDst = new Bitmap(rect.Width, rect.Height, PixelFormat.Format24bppRgb); BitmapData bgDstData = bgDst.LockBits(rect, ImageLockMode.WriteOnly, PixelFormat.Format24bppRgb); BitmapData[] bmpDatas = new BitmapData[numFrames]; byte *[] p = new byte *[numFrames]; int colourDen = 12; int colourDim = 256 / colourDen; for (int i = 0; i < numFrames; i++) { bmpDatas[i] = frames[i].LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb); p[i] = (byte *)bmpDatas[i].Scan0.ToPointer(); } byte *d = (byte *)bgDstData.Scan0.ToPointer(); for (int yi = 0; yi < rect.Height; yi++) { for (int xi = 0; xi < rect.Width; xi++, d += 3) { SortedDictionary <int, List <Color> > votes = new SortedDictionary <int, List <Color> >(); for (int k = 0; k < numFrames; k++) { Color c = Color.FromArgb(*(p[k] + 0), *(p[k] + 1), *(p[k] + 2)); // 3D Color-space 256/3 ^3 int colorKey = (c.R / colourDen) + (c.G / colourDen) * colourDim + (c.B / colourDen) * colourDim * colourDim; if (!votes.ContainsKey(colorKey)) { votes[colorKey] = new List <Color>(); } votes[colorKey].Add(c); p[k] += 3; } int maxVotes = int.MinValue; int keyOfMaxVotes = -1; foreach (var key in votes.Keys) { if (votes[key].Count > maxVotes) { maxVotes = votes[key].Count; keyOfMaxVotes = key; } } int[] sum = new int[3]; for (int k = 0; k < 3; k++) { sum[k] = 0; } for (int k = 0; k < votes[keyOfMaxVotes].Count; k++) { sum[0] += votes[keyOfMaxVotes][k].R; sum[1] += votes[keyOfMaxVotes][k].G; sum[2] += votes[keyOfMaxVotes][k].B; } //if (votes[keyOfMaxVotes].Count != numFrames) // Console.WriteLine("Interesting"); *(d + 0) = (byte)((float)sum[0] / (float)votes[keyOfMaxVotes].Count); *(d + 1) = (byte)((float)sum[1] / (float)votes[keyOfMaxVotes].Count); *(d + 2) = (byte)((float)sum[2] / (float)votes[keyOfMaxVotes].Count); } } for (int i = 0; i < 6; i++) { frames[i].UnlockBits(bmpDatas[i]); } bgDst.UnlockBits(bgDstData); bgDst.Save($"bgc{frameCount:D06}.jpg", ImageFormat.Jpeg); frames.Clear(); } bgFrameCount++; } frameCount++; } } } }
private unsafe void ProcessFrames(AVHWDeviceType HWDevice, string url, long startFrame = 0, long endFrame = long.MaxValue) { _videoFilepath = url; _processingEnabled = true; _perfTimer = new FPSTimer((int)_videoFrameRate); long leadInFrames = 5; if (!_processMultithreaded && leadInFrames < startFrame) // dont't do leadIn if we're too close to the start of the video { startFrame = startFrame - leadInFrames; } else { leadInFrames = 0L; } // Don't set endFrame in most cases since we want to be able to seek to end and not have this loop exit //if (endFrame > _videoFrameCount) // endFrame = _videoFrameCount - 1; using (var vsd = new VideoStreamDecoder(url, HWDevice)) { Console.WriteLine($"codec name: {vsd.CodecName}"); var info = vsd.GetContextInfo(); info.ToList().ForEach(x => Console.WriteLine($"{x.Key} = {x.Value}")); var sourceSize = vsd.FrameSize; var sourcePixelFormat = HWDevice == AVHWDeviceType.AV_HWDEVICE_TYPE_NONE ? vsd.PixelFormat : GetHWPixelFormat(HWDevice); var destinationSize = sourceSize; var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_BGR24; int framesSinceSeek = 0; int framesSinceSeekThresh = 5; using (var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat)) { _frameNumber = startFrame; if (startFrame != 0) { vsd.Seek(startFrame); } //byte[] currFrameData = new byte[destinationSize.Width * destinationSize.Height * 3]; //byte[] prevFrameData = new byte[destinationSize.Width * destinationSize.Height * 3]; var width = destinationSize.Width; var height = destinationSize.Height; Image <Bgr, byte> prevImage = new Image <Bgr, byte>(width, height); //Image Class from Emgu.CV //FrameBlender[] backgroundBuilders = new FrameBlender[processSettings.backgroundFrameBlendInterval]; //Bitmap[] bgs = new Bitmap[processSettings.backgroundFrameBlendInterval]; //for (int i = 0; i < backgroundBuilders.Length; i++) { // backgroundBuilders[i] = new FrameBlender(width, height, processSettings.backgroundFrameBlendCount); // bgs[i] = new Bitmap(width, height, PixelFormat.Format32bppArgb); //} //FrameBlender frameSmoother = new FrameBlender(width, height, _processSettings.frameBlendCount); Image <Bgra, float> background = null; //var bgBuilder = new Emgu.CV.BackgroundSubtractorMOG2(500, 16, false); var bgBuilder = new Emgu.CV.BackgroundSubtractorKNN(500, 4.0, false); Image <Gray, byte> foregroundMask = new Image <Gray, byte>(width, height); var currForeground = new Image <Bgr, byte>(width, height); var prevForeground = new Image <Bgr, byte>(width, height); //var movement = new Image<Gray, byte>(width, height); //var movementHist = new Image<Gray, byte>(width, height); bool decoderActive = true; AVFrame frame; while (decoderActive && (_frameNumber < endFrame) && _processingEnabled) { int seekFrameNum = _trackBarPos; if (!_processMultithreaded) { if (Math.Abs(_frameNumber - seekFrameNum) > 250) { vsd.Seek(seekFrameNum); _frameNumber = seekFrameNum; framesSinceSeek = 0; } var trackBarSetMI = new MethodInvoker(() => trackBar1.Value = Math.Min((int)_frameNumber, trackBar1.Maximum - 1)); trackBar1.Invoke(trackBarSetMI); } try { decoderActive = vsd.TryDecodeNextFrame(out frame); } catch { continue; } if (framesSinceSeek < framesSinceSeekThresh) { ++framesSinceSeek; } while (_processingSleep) { Thread.Sleep(500); } var convertedFrame = vfc.Convert(frame); Image <Bgr, byte> currImage = new Image <Bgr, byte>(width, height, convertedFrame.linesize[0], (IntPtr)convertedFrame.data[0]); // Shadow reduction: Shadows are lindear and less vertical, so stretch the image wider // and then resize back to original aspect-ratio to discard some horizontal detail. // Also, people are taller than bikes & balls if (_settingsControl.EnableShadowReduction) { currImage = currImage.Resize(width, height / 8, Emgu.CV.CvEnum.Inter.Area).Resize(width, height, Emgu.CV.CvEnum.Inter.Area); } // Smooth using multiple frames // Use 10% of previous frame for some speckle-noise & flicker reduction if (_settingsControl.EnableDespeckle) { currImage = currImage.SmoothGaussian(3); } currImage = (0.3 * currImage.Mat + 0.7 * prevImage.Mat).ToImage <Bgr, byte>(); //currImage = frameSmoother.Update(currImage.ToBitmap()).ToImage<Bgr, byte>(); if (!_maskSet) { using (Bitmap bmp = ShowEditMaskForm(currImage.ToBitmap(), _mask)) { _maskSet = true; if (bmp == null) { continue; } _mask = GetMatFromSDImage(bmp).ToImage <Bgra, byte>()[2]; // mask is red-channel // Clean-up and invert to form the correct mask var whiteImg = new Image <Gray, byte>(width, height); whiteImg.SetValue(255); _mask = whiteImg.Copy(_mask).Not(); } } bgBuilder.Apply(currImage, foregroundMask); if (_frameNumber == 0L) { background = currImage.Convert <Bgra, float>(); } else { Image <Bgra, float> newBg = currImage.Convert <Bgra, float>(); newBg[3].SetValue(1, foregroundMask.Not()); background[3].SetValue(1); background = (background.Mat * .97 + newBg.Mat * .03).ToImage <Bgra, float>(); //var picMI = new MethodInvoker(() => pictureBox1.Image = background.ToBitmap()); //pictureBox1.Invoke(picMI); } if (leadInFrames > 4L) { leadInFrames--; _frameNumber++; continue; // skip further processing until the lead-in is complete } else if (leadInFrames > 0L) { leadInFrames--; } Mat foregroundMat = background.Convert <Bgr, byte>().Not().Mat + currImage.Mat; currForeground = foregroundMat.ToImage <Bgr, byte>(); if (_settingsControl.EnableDespeckle) { currForeground = currForeground.SmoothGaussian(3); // remove speckle (video low-light noise, small birds, insects, etc) } //currForeground = currImage.Copy(foregroundMask.Not()); Mat moveMat = currForeground.Mat - prevForeground.Mat; _movement = (1.02 * (_movement.Mat * 0.85 + 0.15 * ((moveMat - _settingsControl.MovementNoiseFloor) * _settingsControl.MovementPixMul).ToImage <Bgr, byte>().Convert <Gray, float>().Mat)).ToImage <Gray, float>(); //var picMI = new MethodInvoker(() => pictureBox1.Image = _movement.ToBitmap()); //pictureBox1.Invoke(picMI); if (_mask != null) { _movement = _movement.Copy(_mask); } currImage = new Image <Bgr, byte>(width, height, convertedFrame.linesize[0], (IntPtr)convertedFrame.data[0]); prevImage.Bytes = currImage.Bytes; prevForeground.Bytes = currForeground.Bytes; if (leadInFrames == 0) { int currentFps = _perfTimer.Update(); double processingRate = (double)currentFps / (double)_videoFrameRate; var time = TimeSpan.FromSeconds((double)_frameNumber / (double)_videoFrameRate); // https://docs.microsoft.com/en-us/dotnet/standard/base-types/custom-timespan-format-strings statusLabel.Text = $"{time:hh\\:mm\\:ss}"; statusProcessingRate.Text = $"Processing@{ processingRate: 0.0}x"; //var moveScore = _movement.GetSum().Intensity * _settingsControl.MovementScoreMul; // *** Do perspective correction to movement score var moveScore = 0.0; for (int yi = 0; yi != _movement.Height; ++yi) { // Assume that a foreground person is 2x taller than a background person double pc = 1.0 - ((double)yi / (double)_movement.Height); // 0..1, bottom..top pc = 0.5 + 0.5 * pc; // 0.5..1.0, bottom..top pc *= 2.0; // foreground person is 2x the height of background person pc *= pc; // squared since movement~area Rectangle rowRect = new Rectangle(0, yi, _movement.Width, 1); Image <Gray, float> row = _movement.GetSubRect(rowRect); moveScore += row.GetSum().Intensity *_settingsControl.MovementScoreMul *pc; } if (framesSinceSeek == framesSinceSeekThresh) { if (_frameNumber < _movementScores.Length) { _movementScores[_frameNumber] = (float)moveScore; } _movementScoreMax = Math.Max(_movementScoreMax, (float)moveScore); UpdateChart(); // decay the maximum slowly to ensure early "noise" peaks don't destroy scaling forever _movementScoreMax *= 0.9999f; } } _processedFrameCount++; _frameNumber++; if (!_processMultithreaded && (_frameNumber < endFrame)) { // Off-load some processing to another thread to allow faster updates on the main processing thread Task.Run(() => { try { UpdateProcessMainView(currImage.Mat); } catch { } }); } if (_frameNumber == endFrame) { var mi = new MethodInvoker(() => ProcessingCompleteAndExport()); this.Invoke(mi); } } } } }