private void VisualizeDenseFlow_Means(ref double vmag, ref double vang)
{
Image <Gray, float> flowX = new Image <Gray, float>(currentGray.Size);
Image <Gray, float> flowY = new Image <Gray, float>(currentGray.Size);
Image <Gray, float> mag = new Image <Gray, float>(currentGray.Size);
Image <Gray, float> ang = new Image <Gray, float>(currentGray.Size);
Image <Gray, byte> fullGray = new Image <Gray, byte>(currentImage.Size);
fullGray.SetValue(new Gray(255));
if (framesProcessed >= 2)
{
CvInvoke.CalcOpticalFlowFarneback(lastGray, currentGray, flowX, flowY, 0.5, 3, 50, 3, 5, 1.2, OpticalflowFarnebackFlag.Default);
CvInvoke.CartToPolar(flowX, flowY, mag, ang, true);
CvInvoke.Threshold(mag, mag, 4, float.MaxValue, ThresholdType.ToZero);
MCvScalar meanang = CvInvoke.Mean(ang.Mat, mag.Convert <Gray, byte>());
MCvScalar meanmag = CvInvoke.Mean(mag.Mat, mag.Convert <Gray, byte>());
vmag = meanmag.V0;
vang = meanang.V0;
if (vmag > 4)
{
if (minang > vang)
{
minang = vang;
}
if (maxang < vang)
{
maxang = vang;
}
}
CvInvoke.Normalize(mag, mag, 0, 255, NormType.MinMax);
CvInvoke.Normalize(ang, ang, 0, 255, NormType.MinMax);
Image <Hsv, byte> hsv = new Image <Hsv, byte>(new Image <Gray, byte>[] { ang.Convert <Gray, byte>(), fullGray, mag.Convert <Gray, byte>() });
double x = Math.Cos(vang * (Math.PI / 180.0));
double y = Math.Sin(vang * (Math.PI / 180.0));
CvInvoke.CvtColor(hsv, currentImage, ColorConversion.Hsv2Bgr);
currentImage.Erode(5);
currentImage.Dilate(5);
currentImage.Draw(new LineSegment2D(new Point(currentImage.Size.Width / 2, currentImage.Height / 2), new Point((int)(x * 100) + currentImage.Size.Width / 2, (int)(y * 100) + currentImage.Height / 2)), new Bgr(Color.Red), 2);
}
}
public void ProcessSrcToEnd(UMat inImg)
{
///Displays the input image, forward fourier transform(fFt) it and display magnitude of that fFt. It then calls the next function.
///Previous name was ProcessSrcTofFt.
VectorOfUMat ft; //forward (Fourier) transform
using (UMat grayImg = GetBgr2Gray(inImg)) { //make grayScale. It is a copy so as to not mess up the displaying.
ft = ForwardFt(grayImg);
}
// get magnitude and phase from real and imaginary parts
if (magT != null)
{
magT.Dispose();
}
magT = new UMat();
if (phT != null)
{
phT.Dispose();
}
phT = new UMat();
CvInvoke.CartToPolar(ft[0], ft[1], this.magT, this.phT);
ft[0].Dispose();
ft[1].Dispose();
ft.Dispose(); //TODO: check whether you have to dispose each UMat individually then the vector. Same for inverse fourier transform's output down there.
//display magnitude of forward fourier transform
fTView.Display_Image(GetNorm4Disp(this.magT, log: true));
if (firstTimeLoadingImage)
{
this.firstTimeLoadingImage = false;
// set rectangle selection to full image (I want user to see all boxes displaying image when start up)
this.ftSelectedRect = new Rectangle(new Point(0, 0), magT.Size);
this.oldImgSize = this.magT.Size;
}
else
{
//if the new image is smaller than the previous one,
if (magT.Size.Width < oldImgSize.Width || magT.Size.Height < oldImgSize.Height) //TODO: make this smaller-than-image checking a function cause it happens again at the end of processing, or somehow join the 2 to neaten it up.
//limit the selected rectangle area to within the new image to avoid trying to read outside the image.
{
LimitRectToWithinImage(this.magT, ref this.ftSelectedRect);
}
}
//continue the chain
ProcessSelRectToEnd(this.ftSelectedRect, this.magT, this.phT);
}
private void moduleKeyFrameExtraction(Mat inputMat)
{
Image <Bgr, Byte> imageMedianBlur = modulePreProcessing(inputMat);
//MedianBlur for Feature Extraction
/*Image<Bgr, Byte> imageFE = new Image<Bgr, Byte>(inputMat.Width, inputMat.Height);
* CvInvoke.MedianBlur(imageInput, imageFE, 7);
* pictureBox2.Image = imageFE.Bitmap;*/
//Sobel
Image <Gray, float> imageSobelInput = imageMedianBlur.Convert <Gray, float>();
Image <Gray, float> _imgSobelx = new Image <Gray, float>(imageSobelInput.Width, imageSobelInput.Height);
Image <Gray, float> _imgSobely = new Image <Gray, float>(imageSobelInput.Width, imageSobelInput.Height);
_imgSobelx = imageSobelInput.Sobel(1, 0, 3);
_imgSobely = imageSobelInput.Sobel(0, 1, 3);
Image <Gray, float> magnitude = new Image <Gray, float>(imageSobelInput.Width, imageSobelInput.Height);
Image <Gray, float> angle = new Image <Gray, float>(imageSobelInput.Width, imageSobelInput.Height);
CvInvoke.CartToPolar(_imgSobelx, _imgSobely, magnitude, angle, true);
//Image box2 image of gradient
//imageBox3.Image = magnitude;
//FrameNumber Display
//label2.Text = frameNumber.ToString();
double avg = magnitude.GetAverage().Intensity;
//Chart of gradient
//chart1.Series["Gradient"].Points.AddXY(frameNumber, avg);
if (!isFirst)
{
if (avg > 0)
{
isFirst = true;
first = frameNumber;
isLast = false;
}
}
if (!isLast)
{
if (avg == 0)
{
isLast = true;
last = frameNumber;
//label3.Text = "First: " + first + " Last: " + last;
int mid = (first + last) / 2;
moduleFeatureExtraction(first, last);
isFirst = false;
}
}
frameNumber++;
}
private static double CountDistanceThroughEdges(Mat leftEdges, Mat rightEdges, Rectangle roi,
double baseLine, double focalLengthPx)
{
if (roi == null)
{
return(-1);
}
Mat prev8bit = leftEdges;
Mat current8bit = rightEdges;
Image <Gray, byte> imageLeft = prev8bit.ToImage <Gray, byte>();
Image <Gray, byte> imageRight = current8bit.ToImage <Gray, byte>();
Image <Gray, float> flowX = new Image <Gray, float>(prev8bit.Width, prev8bit.Height);
Image <Gray, float> flowY = new Image <Gray, float>(prev8bit.Width, prev8bit.Height);
CvInvoke.CalcOpticalFlowFarneback(imageLeft, imageRight, flowX,
flowY, 0.5, 3, 25, 10, 5, 1.1, OpticalflowFarnebackFlag.FarnebackGaussian);
Mat magnitude = new Mat();
Mat angle = new Mat();
CvInvoke.CartToPolar(flowX.Mat, flowY.Mat, magnitude, angle);
int matWidth = magnitude.Width;
int matHeight = magnitude.Height;
float[] magData = new float[matWidth * matHeight];
magnitude.CopyTo(magData);
List <double> results = new List <double>();
for (int x = roi.X; x <= roi.X + roi.Width; x++)
{
for (int y = roi.Y; y <= roi.Y + roi.Height; y++)
{
float delta = GetElementFromArrayAsFromMatrix(magData, y, x, matWidth);
if (delta < epsilon)
{
continue;
}
double distance = (baseLine * focalLengthPx) / delta;
results.Add(distance);
}
}
if (results.Count == 0)
{
return(-1);
}
return(results.Average());
}
public static Mat GetEdges(Mat src)
{
const Single scharrKernelNormalizationDivisor = 16.0f;
const Single scharrKernelNormalizationFactor = 1.0f / scharrKernelNormalizationDivisor;
src.ConvertTo(src, DepthType.Cv32F, 1 / 255d);
var gradX = new Mat();
var gradY = new Mat();
CvInvoke.Scharr(src, gradX, DepthType.Cv32F, 1, 0, scharrKernelNormalizationFactor);
CvInvoke.Scharr(src, gradY, DepthType.Cv32F, 0, 1, scharrKernelNormalizationFactor);
var outImg = new Mat();
CvInvoke.CartToPolar(gradX, gradY, outImg, new Mat());
outImg.ConvertTo(outImg, DepthType.Cv8U, 255);
return(outImg);
}
public void InverseT(UMat reIn, UMat imIn, UMat magOut, UMat phOut)
{
///accepts real and imaginary parts, inverse (fourier) Transforms
///converts real and imaginary output parts to magnitude and
///phase, returns magnitude and phase parts.
///Refer to ForwardT() for more info on why I code like this.
///Reference: https://stackoverflow.com/questions/16812950/how-do-i-compute-dft-and-its-reverse-with-emgu
VectorOfUMat vec = this.vec;
vec.Push(reIn);
vec.Push(imIn);
UMat cImg = this.img32f2c;
CvInvoke.Merge(vec, cImg); //because Dft method accepts and outputs 2-channel image
CvInvoke.Dft(cImg, cImg, DxtType.Inverse, 0);
//new objects put into vec, vec[0] is reOut, vec[1] is imOut.
CvInvoke.Split(cImg, vec);
//convert output of inverse Transform to magnitude and polar
CvInvoke.CartToPolar(vec[0], vec[1], magOut, phOut);
vec.Clear();
}
public void ProcessForwardT(UMat inImg, UMat outMagT, UMat outPhT, bool zeroPad = false, bool switchQuadrants = true)
{
///Accepts a 1-channel image, updates outMagT and outPhT.
///magnitude and phase, cause I can't think of why you
///would wanna look at real and imaginary Transforms.
///Also can't think of how you can get complex-valued images.
///Quadrant rearranging doesn't support odd rows or cols.
///T stuff, reference: https://docs.opencv.org/master/d8/d01/tutorial_discrete_fourier_transform.html
//convert image to 32bit float because spacial frequency
//domain values are way bigger than spatial domain values.
UMat re = outMagT; //32-bit float real image, use memory from
//outMagT cause it's gonna be updated anyway
inImg.ConvertTo(re, DepthType.Cv32F);
if (zeroPad)
{
//zero pad for faster dft
ZeroPadImage(re);
}
//create imaginary image of zeros of same depthType
//and size as image representing real plane
UMat im = outPhT; //imaginary
im.Create(re.Rows, re.Cols, re.Depth, re.NumberOfChannels); //allocate memory so you can set it to zero array
//if memory hasn't already been allocated for it
im.SetTo(new MCvScalar(0));
/// Quick exerpt about VectorOfUMat:
/// if you create a VectorOfUMat vec, only if the first UMat variable
/// to store the object is the vector node, like
/// VectorOfUMat vec = new VectorOfUmat(new Umat(), new Umat());
/// vec.Push(someUMat.Clone());
/// then vec.Dispose/Clear actually disposes all the objects referenced
/// to by the UMats in vec. In this case, if you did:
/// VectorOfUMat vec = new VectorOfUMat(inImg.Clone(), inImg.Clone());
/// UMat one = vec[0];
/// one.Dispose();
/// one.Dispose actually does nothing.
/// Otherwise, if
/// UMat one = new UMat();
/// UMat two = new UMat();
/// VectorOfUMat vec = new VectorOfUmat(one);
/// vec.Push(two);
/// calling vec.Dispose() doesn't dispose the objects.
/// you have to call one.Dispose and two.Dispose.
/// Note: no matter whether the UMat's first variable stored
/// in is in a vector node or not, calling vec[index].Dispose
/// does NOTHING.
/// The situation is the same for vec.Clear, except Clear doesn't
/// dispose of vec itself, it just disposes the objects the UMats in
/// it reference to.
//Dft accepts 2-channel images, so we use Merge to merge
//our 2 1-channel images into a single 2-channel image.
//Merge accepts object arrays, so we create a VectorOfUMat
//of our 2 images to feed into Merge.
VectorOfUMat vec = this.vec;
UMat cImg = this.img32f2c;
vec.Push(re);
vec.Push(im); //vec[0] = re, vec[1] = im
;
CvInvoke.Merge(vec, cImg);
CvInvoke.Dft(cImg, cImg, DxtType.Forward, 0); //use back the same memory
//switch quadrants while images are still combined
if (switchQuadrants)
{
SwitchQuadrants(cImg);
}
//make the 2-channel array into 2 1-channel arrays
CvInvoke.Split(cImg, vec); //vec[0] is reT, vec[1] is imT, they are new objects.
CvInvoke.CartToPolar(vec[0], vec[1], outMagT, outPhT);
vec.Clear(); //dispose reT and imT.TODO: find a way to get rid of allocating memory for reT and imT.
}
public void ProcessAddPhaseToEnd(
double rDist,
double sensorPixelWidth,
double sensorPixelHeight,
double wavelength,
UMat magFoT,
UMat phFoT,
Rectangle ctrRect,
Point selRectMaxMagPoint)
{
///Based on a bunch of parameters (as you can see in the input), calculates the phase difference for each plane wave
///between the aperture(hologram) plane and image plane. It adds this phase difference to each plane wave, turning the
///fourier spectrum into one at the image plane, and then inverse fourier transforms to get the predicted magnitude and
///phase of light at the image plane. It then displays the predicted magnitude and phase images.
///This version modifies the original phF)). For Video mode.
///
/// This next part is just me rambling over the minimal parameters to choose and the ups and downs of each set.
///I went for just passing already calculated values instead of recalculating them.
///1. create matrix of size ( phCtrFoT OR ctrRect, which contains location info )
///2. value at each point is based on distance from ( rectSelMaxMagLocations[0] OR from ctrRect:
///Point FoT0HzPoint = new Point(magFoT.Cols / 2, magFoT.Rows / 2);
///Point rectSelMaxMagLocation = ctrRect.Location.Minus(FOT0HzPoint);
///3. copy over to phCtrFoT, which if you choose ctrRect and null, you have to create another header.
///
///I choose (phCtrFoT, roiMaxMagLoc)!
///
///However, due to needing to clone phFoT, and hence phCtrFoT as well to preserve the original when in static
///image mode, I chose (ctrRect, selRectMaxMagLocation) instead.
//Calculate phase difference matrix to add to sinewaves at particular (x,y)s.
//As much calculation is taken out of the loop as possible.
//done with image class because it is faster than Matrix<type> class.
//Only difference between image class and umat/mat (for this function) is at CvInvoke.Add,
//where umat/mat takes 100ms less time in 17s. Not worth the extra carefulness (yet).
double sensorHeight = phFoT.Rows * sensorPixelHeight;
double sensorWidth = phFoT.Cols * sensorPixelWidth;
double xFreqInterval = 1 / sensorWidth;
double yFreqInterval = 1 / sensorHeight;
double xFreqITimesWavelength = xFreqInterval * wavelength;
double yFreqITimesWavelength = yFreqInterval * wavelength;
double wavenumber = 2 * Math.PI / wavelength;
Image <Gray, float> phDiffMat = new Image <Gray, float>(ctrRect.Size);
int rows = phDiffMat.Rows;
int cols = phDiffMat.Cols;
for (int y = 0; y < rows; ++y)
{
for (int x = 0; x < cols; ++x)
{
double xTFreq = x - selRectMaxMagPoint.X; //fourier transform frequency
double yTFreq = y - selRectMaxMagPoint.Y;
double gammaX = xTFreq * xFreqITimesWavelength;
double gammaY = yTFreq * yFreqITimesWavelength;
double gammaXSqred = gammaX * gammaX;
double gammaYSqred = gammaY * gammaY;
double insideSqrt = 1 - gammaXSqred - gammaYSqred;
double sqrted = Math.Sqrt(insideSqrt);
double sqrtedTimesWavenumber = sqrted * wavenumber;
double ph = rDist * sqrtedTimesWavenumber;
float fPh = (float)ph;
phDiffMat.Data[y, x, 0] = fPh;
}
}
//if video mode is off, add phase difference to a copy of the first-order-only image,
//because we don't want to modify the input
UMat phFoT2Process, phCtrFoT2Process; //variable 2
if (video == false) //create a copy to preserve the original
{
phFoT2Process = phFoT.Clone(); //a copy
phCtrFoT2Process = new UMat(phFoT2Process, ctrRect); //roi of the copy
}
else
{
phFoT2Process = phFoT; //not-a-copy
phCtrFoT2Process = new UMat(phFoT, ctrRect); //roi of not-a-copy
}
CvInvoke.Add(phCtrFoT2Process, phDiffMat, phCtrFoT2Process, dtype: DepthType.Cv32F);
phDiffMat.Dispose();
VectorOfUMat iFt;
//convert magnitude and phase to real and imaginary images to input into inverse fourier transform function
using (UMat reFoT = new UMat(), imFoT = new UMat()) {
CvInvoke.PolarToCart(magFoT, phFoT2Process, reFoT, imFoT);
if (video == false)
{
//disposing phFoT2Process also takes care of phCtrFoT2Process, as the latter is an ROI of the first.
phFoT2Process.Dispose();
}
iFt = InverseFt(reFoT, imFoT); // inverse fourier transform
}
// get magnitude and phase from real and imaginary parts
UMat magFo = new UMat();
UMat phFo = new UMat();
CvInvoke.CartToPolar(iFt[0], iFt[1], magFo, phFo);
iFt.Dispose(); //disposes the real and imaginary images inside it too.
intensityView.Display_Image(GetNorm4Disp(magFo, log: true));
phaseView.Display_Image(GetNorm4Disp(phFo));
//if the new image is different size from the previous one,
if (magT.Size.Width < oldImgSize.Width || magT.Size.Height < oldImgSize.Height)
{
//Make the images fit their imageBoxes so the user sees the whole image loaded at first.
//FitImageToImageBox(imageBox1);
//FitImageToImageBox(imageBox2);
//FitImageToImageBox(imageBox3);
//FitImageToImageBox(imageBox4);
}
this.oldImgSize = this.magFoT.Size;
}
private void UpdateTimer_Tick(object sender, EventArgs e)
{
if (
false &&
framesProcessed == 2)
{
updateTimer.Stop();
return;
}
double rho = 0.0;
double theta_deg = 0.0;
perfSw.Restart();
try
{
framesProcessed++;
//store last gray image, pull current image, convert to gray
lastGray = currentGray;
currentImage = capture.QueryFrame().ToImage <Bgr, Byte>();
Image <Gray, byte> grayImage = new Image <Gray, byte>(currentImage.Size);
CvInvoke.CvtColor(currentImage, grayImage, ColorConversion.Bgr2Gray);
//apply gaussian blur to gray to smooth out noise
CvInvoke.GaussianBlur(grayImage, grayImage, new Size(15, 15), 1.8);
currentGray = grayImage;
//create variables to store optical flow in cart and polar coordinates
Image <Gray, float> flowX = new Image <Gray, float>(currentGray.Size);
Image <Gray, float> flowY = new Image <Gray, float>(currentGray.Size);
Image <Gray, float> mag = new Image <Gray, float>(currentGray.Size);
Image <Gray, float> ang = new Image <Gray, float>(currentGray.Size);
//image with all values set to 255
Image <Gray, byte> fullGray = new Image <Gray, byte>(currentImage.Size);
fullGray.SetValue(new Gray(255));
//wait until second frame to get flow
if (framesProcessed >= 2)
{
int threshold = 2;
//get flow images
CvInvoke.CalcOpticalFlowFarneback(lastGray, currentGray, flowX, flowY, 0.5, 3, 20, 3, 5, 1.2, OpticalflowFarnebackFlag.Default);
//convert x and y flow to magnitude and angle images
CvInvoke.CartToPolar(flowX, flowY, mag, ang, true);
//threshold the magnitude so we only look at vectors with motion
CvInvoke.Threshold(mag, mag, threshold, 1.0, ThresholdType.Binary);
//find the total number of pixels in the image that are over the threshold value
MCvScalar sumMask = CvInvoke.Sum(mag);
//apply the mask to the flow vectors
flowX = flowX.Copy(mag.Convert <Gray, byte>());
flowY = flowY.Copy(mag.Convert <Gray, byte>());
//sum of flow vector components
MCvScalar sumX = CvInvoke.Sum(flowX);
MCvScalar sumY = CvInvoke.Sum(flowY);
double avgX = 0.0;
double avgY = 0.0;
//avg of flow vector components
if (sumMask.V0 > 0.0)
{
avgX = sumX.V0 / sumMask.V0;
avgY = sumY.V0 / sumMask.V0;
}
//convert to polar radius (rho) and angle (theta)
rho = Math.Sqrt(avgX * avgX + avgY * avgY);
double theta = Math.Atan2(avgY, avgX);
//convert angle from radians to degrees
theta_deg = theta * 180 / Math.PI;
//clamp values to bytes for HSV image visualization
CvInvoke.Normalize(mag, mag, 0, 255, NormType.MinMax);
CvInvoke.Normalize(ang, ang, 0, 255, NormType.MinMax);
//create hue/saturation/value image to visualize magnitude and angle of flow
Image <Hsv, byte> hsv = new Image <Hsv, byte>(new Image <Gray, byte>[] { ang.Convert <Gray, byte>(), fullGray, mag.Convert <Gray, byte>() });
//adding average flow components history
avgXHistory.Add(avgX);
avgYHistory.Add(avgY);
double rho_avg = 0.0;
double x_avg = 0.0;
double y_avg = 0.0;
int smoothLength = 3;
//get average angle in the history
if (avgXHistory.Count() > 0)
{
x_avg = avgXHistory.Median(smoothLength);
y_avg = avgYHistory.Median(smoothLength);
rho_avg = Math.Sqrt(x_avg * x_avg + y_avg * y_avg);
}
//convert hsv to bgr image for bitmap conversion
CvInvoke.CvtColor(hsv, currentImage, ColorConversion.Hsv2Bgr);
//draw instaneous average flow direction line if magnitude exceeds threshold
if (rho > threshold)
{
currentImage.Draw(new LineSegment2D(new Point(currentImage.Size.Width / 2, currentImage.Height / 2), new Point((int)(avgX * 10) + currentImage.Size.Width / 2, (int)(avgY * 10) + currentImage.Height / 2)), new Bgr(Color.Red), 2);
}
//draw historical average flow direction line
if (rho_avg > threshold)
{
currentImage.Draw(new LineSegment2D(new Point(currentImage.Size.Width / 2, currentImage.Height / 2), new Point((int)(x_avg * 10) + currentImage.Size.Width / 2, (int)(y_avg * 10) + currentImage.Height / 2)), new Bgr(Color.Blue), 2);
}
//pull bitmap from image and draw to picture box
currentBitmap = currentImage.Bitmap;
videoPictureBox.Image = currentBitmap;
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message + " " + exp.StackTrace);
}
perfSw.Stop();
fps = ((double)framesProcessed / fpsSw.Elapsed.TotalSeconds);
statusLabel.Text = framesProcessed.ToString() + " totalms: " + fpsSw.ElapsedMilliseconds.ToString() + " fps: " + ((double)framesProcessed / fpsSw.Elapsed.TotalSeconds).ToString() + " perfms: " + perfSw.ElapsedMilliseconds + " vmag: " + rho + " vang: " + theta_deg;
}
private void StartMatching()
{
Image <Bgr, Byte> target = Image_Target.Clone().Resize(scale, Inter.Linear);
Image <Bgr, Byte> texture = Image_Texture.Clone();
Image <Gray, Byte> target_gray = target.Convert <Gray, Byte>();
Image <Gray, Byte> texture_gray = texture.Convert <Gray, Byte>();
Image <Gray, Byte> target_hist_matched = new Image <Gray, Byte>(target.Size);
Image <Gray, Byte> target_hist_matched_weighted = new Image <Gray, Byte>(target.Size);
Image <Gray, float> target_sobel_x = target_gray.Sobel(1, 0, 3);
Image <Gray, float> target_sobel_y = target_gray.Sobel(0, 1, 3);
Image <Gray, float> texture_sobel_x = texture_gray.Sobel(1, 0, 3);
Image <Gray, float> texture_sobel_y = texture_gray.Sobel(0, 1, 3);
Image <Gray, float> target_sobel_mag = new Image <Gray, float>(target_gray.Size);
Image <Gray, float> texture_sobel_mag = new Image <Gray, float>(texture_gray.Size);
Image_Result = new Image <Bgr, byte>(target.Width, target.Height, new Bgr(0, 0, 0));
imageBox_Result.Image = Image_Result;
this.Invoke(new MethodInvoker(() =>
{
progressBar_match.Value = 0;
progressBar_match.Maximum = (target.Width / size) * (target.Height / size);
}));
Matrix <byte> histLUT = new Matrix <byte>(1, 256);
Mat hist_target = new Mat();
Mat hist_texture = new Mat();
VectorOfMat vm_target = new VectorOfMat();
VectorOfMat vm_texture = new VectorOfMat();
vm_target.Push(target_gray);
vm_texture.Push(texture_gray);
CvInvoke.CalcHist(vm_target, new int[] { 0 }, null, hist_target, new int[] { 256 }, new float[] { 0, 255 }, false);
CvInvoke.CalcHist(vm_texture, new int[] { 0 }, null, hist_texture, new int[] { 256 }, new float[] { 0, 255 }, false);
float[] CDF_hist_target = new float[256];
float[] CDF_hist_texture = new float[256];
Marshal.Copy(hist_target.DataPointer, CDF_hist_target, 0, 256);
Marshal.Copy(hist_texture.DataPointer, CDF_hist_texture, 0, 256);
for (int i = 1; i < 256; i++)
{
CDF_hist_target[i] += CDF_hist_target[i - 1];
CDF_hist_texture[i] += CDF_hist_texture[i - 1];
}
for (int i = 0; i < 256; i++)
{
histLUT.Data[0, i] = 0;
for (int j = 0; j < 256; j++)
{
if (CDF_hist_texture[j] >= CDF_hist_target[i])
{
histLUT.Data[0, i] = (byte)j;
break;
}
}
}
CvInvoke.LUT(target_gray, histLUT, target_hist_matched);
target_hist_matched_weighted = target_hist_matched * weight_hist + target_gray * (1.0 - weight_hist);
CvInvoke.CartToPolar(target_sobel_x, target_sobel_y, target_sobel_mag, new Mat());
CvInvoke.CartToPolar(texture_sobel_x, texture_sobel_y, texture_sobel_mag, new Mat());
List <Matrix <float> > transformation_matrixs = new List <Matrix <float> >();
List <Matrix <float> > transformation_matrixs_invert = new List <Matrix <float> >();
List <RectangleF> rotatedRects = new List <RectangleF>();
for (int i = 1; i < rotations; i++)
{
double angle = i * (360.0 / (float)rotations);
RectangleF rotatedRect = new RotatedRect(new PointF(), texture.Size, (float)angle).MinAreaRect();
PointF center = new PointF(0.5f * texture.Width, 0.5f * texture.Height);
Matrix <float> transformation_matrix = new Matrix <float>(2, 3);
Matrix <float> transformation_matrix_invert = new Matrix <float>(2, 3);
CvInvoke.GetRotationMatrix2D(center, angle, 1.0, transformation_matrix);
transformation_matrix.Data[0, 2] += (rotatedRect.Width - texture.Width) / 2;
transformation_matrix.Data[1, 2] += (rotatedRect.Height - texture.Height) / 2;
CvInvoke.InvertAffineTransform(transformation_matrix, transformation_matrix_invert);
transformation_matrixs.Add(transformation_matrix);
transformation_matrixs_invert.Add(transformation_matrix_invert);
rotatedRects.Add(rotatedRect);
}
List <Image <Bgr, byte> > texture_rotations = new List <Image <Bgr, byte> >(rotations)
{
};
List <Image <Gray, byte> > texture_gray_rotations = new List <Image <Gray, byte> >(rotations)
{
};
List <Image <Gray, float> > texture_sobel_rotations = new List <Image <Gray, float> >(rotations)
{
};
List <Image <Gray, byte> > texture_mask_rotations = new List <Image <Gray, byte> >(rotations)
{
};
texture_rotations.Add(texture);
texture_gray_rotations.Add(texture_gray);
texture_sobel_rotations.Add(texture_sobel_mag);
texture_mask_rotations.Add(new Image <Gray, byte>(texture.Width, texture.Height, new Gray(255)));
for (int i = 1; i < rotations; i++)
{
texture_mask_rotations.Add(new Image <Gray, byte>(rotatedRects[i - 1].Size.ToSize()));
texture_rotations.Add(new Image <Bgr, byte>(rotatedRects[i - 1].Size.ToSize()));
texture_gray_rotations.Add(new Image <Gray, byte>(rotatedRects[i - 1].Size.ToSize()));
texture_sobel_rotations.Add(new Image <Gray, float>(rotatedRects[i - 1].Size.ToSize()));
}
for (int i = 1; i < rotations; i++)
{
CvInvoke.WarpAffine(texture, texture_rotations[i], transformation_matrixs[i - 1], rotatedRects[i - 1].Size.ToSize(), Inter.Nearest, Warp.Default, BorderType.Constant, new MCvScalar(0));
CvInvoke.WarpAffine(texture_gray, texture_gray_rotations[i], transformation_matrixs[i - 1], rotatedRects[i - 1].Size.ToSize(), Inter.Nearest, Warp.Default, BorderType.Constant, new MCvScalar(0));
CvInvoke.WarpAffine(texture_sobel_mag, texture_sobel_rotations[i], transformation_matrixs[i - 1], rotatedRects[i - 1].Size.ToSize(), Inter.Nearest, Warp.Default, BorderType.Constant, new MCvScalar(0));
CvInvoke.WarpAffine(texture_mask_rotations[0], texture_mask_rotations[i], transformation_matrixs[i - 1], rotatedRects[i - 1].Size.ToSize(), Inter.Nearest, Warp.Default, BorderType.Constant, new MCvScalar(0));
}
// Directory.SetCurrentDirectory(path);
String current_path = path + @"\matched_patches";
if (Directory.Exists(current_path))
{
DirectoryInfo di = new DirectoryInfo(current_path);
foreach (FileInfo file in di.GetFiles())
{
file.Delete();
}
foreach (DirectoryInfo dir in di.GetDirectories())
{
dir.Delete(true);
}
}
else
{
DirectoryInfo di = Directory.CreateDirectory(current_path);
}
for (int y = 0; y < target_hist_matched_weighted.Height / size; y++)
{
for (int x = 0; x < target_hist_matched_weighted.Width / size; x++)
{
Image <Bgr, byte> template;
Image <Gray, byte> template_gray;
Image <Gray, float> template_sobel;
template = target.Clone();
template_gray = target_hist_matched_weighted.Clone();
template_sobel = target_sobel_mag.Clone();
template.ROI = new Rectangle(x * size, y * size, size, size);
template_gray.ROI = new Rectangle(x * size, y * size, size, size);
template_sobel.ROI = new Rectangle(x * size, y * size, size, size);
template = template.Clone();
template_gray = template_gray.Copy();
template_sobel = template_sobel.Copy();
int minMatchIndex = -1;
double minMatchValue = double.MaxValue;
Point minMatchLoc = new Point();
Object _lock = new Object();
Parallel.For(0, rotations, i =>
{
Image <Gray, float> match_gray = new Image <Gray, float>(texture_gray.Size);
Image <Gray, float> match_sobel = new Image <Gray, float>(texture_sobel_mag.Size);
Image <Gray, float> match_sum = new Image <Gray, float>(texture.Size);
double minVal = 0, maxVal = 0;
Point minLoc = new Point(), maxLoc = new Point();
CvInvoke.MatchTemplate(texture_gray_rotations[i], template_gray, match_gray, TemplateMatchingType.Sqdiff);
CvInvoke.MatchTemplate(texture_sobel_rotations[i], template_sobel, match_sobel, TemplateMatchingType.Sqdiff);
match_sum = match_gray + match_sobel;
//CvInvoke.MinMaxLoc(match_sum, ref minVal, ref maxVal, ref minLoc, ref maxLoc);
//CudaInvoke.MinMaxLoc(match_sum, ref minVal, ref maxVal, ref minLoc, ref maxLoc, GetMinMaxLocMask(texture_mask_rotations[i], size));
CvInvoke.MinMaxLoc(match_sum, ref minVal, ref maxVal, ref minLoc, ref maxLoc, GetMinMaxLocMask(texture_mask_rotations[i], size));
lock (_lock)
{
if (minVal < minMatchValue && minVal > 0)
{
minMatchValue = minVal;
minMatchIndex = i;
minMatchLoc = minLoc;
}
}
match_gray.Dispose();
match_sobel.Dispose();
match_sum.Dispose();
});
Console.WriteLine($"minMatchValue = {minMatchValue}\r\nminMatchIndex = {minMatchIndex}\r\nminMatchLoc = {minMatchLoc}");
if (minMatchIndex < 0)
{
MessageBox.Show("Out of textures!!!!");
return;
}
texture_mask_rotations[minMatchIndex].Draw(new Rectangle(minMatchLoc.X, minMatchLoc.Y, size - 1, size - 1), new Gray(0), -1);
if (minMatchIndex > 0)
{
Image <Gray, byte> mask = new Image <Gray, byte>(texture_mask_rotations[0].Size);
CvInvoke.WarpAffine(texture_mask_rotations[minMatchIndex], mask, transformation_matrixs_invert[minMatchIndex - 1], mask.Size, Inter.Nearest, Warp.Default, BorderType.Constant, new MCvScalar(0));
texture_mask_rotations[0] = texture_mask_rotations[0] - (255 - mask);
}
for (int i = 1; i < rotations; i++)
{
Image <Gray, byte> mask_rot = new Image <Gray, byte>(texture_mask_rotations[i].Size);
CvInvoke.WarpAffine(texture_mask_rotations[0], mask_rot, transformation_matrixs[i - 1], mask_rot.Size, Inter.Nearest, Warp.Default, BorderType.Constant, new MCvScalar(0));
mask_rot.CopyTo(texture_mask_rotations[i]);
}
texture_rotations[minMatchIndex].ROI = new Rectangle(minMatchLoc, new Size(size, size));
CvInvoke.Imwrite($@"{current_path}\{x}_{y}.bmp", texture_rotations[minMatchIndex].Copy());
imageBox_match.Image = texture_rotations[minMatchIndex].Copy();
imageBox_template.Image = template.Copy();
Image_Result.ROI = new Rectangle(x * size, y * size, size, size);
texture_rotations[minMatchIndex].CopyTo(Image_Result);
Image_Result.ROI = Rectangle.Empty;
texture_rotations[minMatchIndex].ROI = Rectangle.Empty;
imageBox_Result.Image = Image_Result;
if (minMatchIndex > 0)
{
PointF[] maskPoints = { minMatchLoc, new PointF(minMatchLoc.X + size - 1, minMatchLoc.Y), new PointF(minMatchLoc.X, minMatchLoc.Y + size - 1), new PointF(minMatchLoc.X + size - 1, minMatchLoc.Y + size - 1) };
PointF[] maskPoints_rot = new PointF[4];
for (int i = 0; i < 4; i++)
{
maskPoints_rot[i].X = maskPoints[i].X * transformation_matrixs_invert[minMatchIndex - 1].Data[0, 0] + maskPoints[i].Y * transformation_matrixs_invert[minMatchIndex - 1].Data[0, 1] + 1.0f * transformation_matrixs_invert[minMatchIndex - 1].Data[0, 2];
maskPoints_rot[i].Y = maskPoints[i].X * transformation_matrixs_invert[minMatchIndex - 1].Data[1, 0] + maskPoints[i].Y * transformation_matrixs_invert[minMatchIndex - 1].Data[1, 1] + 1.0f * transformation_matrixs_invert[minMatchIndex - 1].Data[1, 2];
}
Point[] maskPoints_rot_round = { Point.Round(maskPoints_rot[0]), Point.Round(maskPoints_rot[1]), Point.Round(maskPoints_rot[3]), Point.Round(maskPoints_rot[2]) };
texture.FillConvexPoly(maskPoints_rot_round, new Bgr(0, 0, 0));
}
else
{
//texture.DrawPolyline(new Point[] { minMatchLoc, new Point(minMatchLoc.X + size, minMatchLoc.Y), new Point(minMatchLoc.X + size, minMatchLoc.Y + size), new Point(minMatchLoc.X, minMatchLoc.Y + size) }, true, new Bgr(0, 0, 0), 0);
texture.Draw(new Rectangle(minMatchLoc, new Size(size - 1, size - 1)), new Bgr(0, 0, 0), -1);
}
imageBox_Texture.Image = texture;
template.Dispose();
template_gray.Dispose();
template_sobel.Dispose();
GC.Collect();
this.Invoke(new MethodInvoker(() =>
{
progressBar_match.Value++;
}));
}
}
}
private void OpticalFlowEvent(ImageProcess sender, Mat mat)
{
if (cnt == 0)
{
//cnt ==0, clone mat for previous mat
prevMat = mat.Clone();
cnt++;
return;
}
Image <Emgu.CV.Structure.Gray, byte> prev_img = prevMat.ToImage <Emgu.CV.Structure.Gray, byte>();
Image <Emgu.CV.Structure.Gray, byte> curr_img = mat.ToImage <Emgu.CV.Structure.Gray, byte>();
Mat flow = new Mat(prev_img.Height, prev_img.Width, DepthType.Cv32F, 2);
CvInvoke.CalcOpticalFlowFarneback(prev_img, curr_img, flow, 0.5, 3, 15, 3, 6, 1.3, 0);
Mat[] flow_parts = new Mat[2];
flow_parts = flow.Split();
Mat magnitude = new Mat(), angle = new Mat(), magn_norm = new Mat();
CvInvoke.CartToPolar(flow_parts[0], flow_parts[1], magnitude, angle, true);
CvInvoke.Normalize(magnitude, magn_norm, 0.0, 1.0, NormType.MinMax);
/*
* //start drawing
* float factor = (float)((1.0 / 360.0) * (180.0 / 255.0));
* Mat colorAngle = angle * factor;
* //angle *= ((1f / 360f) * (180f / 255f));
* //build hsv image
* Mat[] _hsv= new Mat[3];
* Mat hsv = new Mat();
* Mat hsv8 = new Mat();
* Mat bgr = new Mat();
* _hsv[0] = colorAngle;
* _hsv[1] = Mat.Ones(colorAngle.Height,angle.Width,DepthType.Cv32F,1);
* _hsv[2] = magn_norm;
* VectorOfMat vm = new VectorOfMat(_hsv);
* CvInvoke.Merge(vm, hsv);
* hsv.ConvertTo(hsv8,DepthType.Cv8U, 255.0);
* CvInvoke.CvtColor(hsv8, bgr,ColorConversion.Hsv2Bgr);
* bgr.Save("opticalFlow.bmp");
*/
List <double> offset = CalculateDirection(angle, magn_norm);
Console.WriteLine(offset[0] + " , " + offset[1]);
//test direction
/*if( Math.Abs(offset[0]) >Math.Abs(offset[1]))
* {
*
* if (offset[0] > 0 )
* {
* Console.WriteLine("Right");
* }
* else
* {
* Console.WriteLine("Left");
* }
*
* }
* else
* {
*
* if (offset[1] > 0)
* {
* Console.WriteLine("Down");
* }
* else
* {
* Console.WriteLine("Up");
* }
*
* }*/
prevMat = mat.Clone();
}
//Followed this source code mostly
//https://github.com/opencv-java/fourier-transform/blob/master/src/it/polito/teaching/cv/FourierController.java
public static void DFTForward(DFTArgs args)
{
var imgSrc = CvInvoke.Imread(args.src, ImreadModes.Grayscale | ImreadModes.AnyDepth);
//get optimal dimensions (power of 2 i think..)
int xdftsz = CvInvoke.GetOptimalDFTSize(imgSrc.Rows);
int ydftsz = CvInvoke.GetOptimalDFTSize(imgSrc.Cols);
//pad input image to optimal dimensions
CvInvoke.CopyMakeBorder(imgSrc, imgSrc,
0, xdftsz - imgSrc.Rows, 0, ydftsz - imgSrc.Cols,
BorderType.Constant, new MCvScalar(0)
);
imgSrc.PrintInfo("1");
//use 32F format for calcs
imgSrc.ConvertTo(imgSrc, DepthType.Cv32F);
imgSrc.PrintInfo("2");
//create a 2 channel mat using the input as the fist channel
var planes = new VectorOfMat();
planes.Push(imgSrc);
planes.Push(new Mat(imgSrc.Size, DepthType.Cv32F, 1));
Mat complex = new Mat();
CvInvoke.Merge(planes, complex);
complex.PrintInfo("3");
//do the fourrier transform
CvInvoke.Dft(complex, complex, DxtType.Forward, 0);
complex.PrintInfo("4");
//split channels into real / imaginary
var compos = new VectorOfMat(2);
CvInvoke.Split(complex, compos);
//convert real / imaginary to magnitude / phase - which is easier to deal with when looking for artifacts
Mat mag = new Mat();
Mat phs = new Mat();
CvInvoke.CartToPolar(compos[0], compos[1], mag, phs);
mag.PrintInfo("5m");
//convert to log scale since magnitude tends to have a huge range
Helpers.AddS(mag, 1.0, mag);
CvInvoke.Log(mag, mag);
mag.PrintInfo("6m");
phs.PrintInfo("6p");
//regular DFT puts low frequencies in the corners - this flips them to the center
RearrangeQuadrants(mag);
RearrangeQuadrants(phs);
double magMax, magMin;
CvInvoke.MinMaxIdx(mag, out magMin, out magMax, null, null);
//Console.WriteLine("-mi "+magMin+","+magMax+"]");
double phsMax, phsMin;
CvInvoke.MinMaxIdx(phs, out phsMin, out phsMax, null, null);
//convert to a 'normal' format and scale the data
Mat magOut = new Mat();
Mat phsOut = new Mat();
CvInvoke.Normalize(mag, magOut, 0, 65535, NormType.MinMax, DepthType.Cv16U);
CvInvoke.Normalize(phs, phsOut, 0, 65535, NormType.MinMax, DepthType.Cv16U);
string name = Path.GetFileNameWithoutExtension(args.dst);
magOut.PrintInfo("7m");
phsOut.PrintInfo("7p");
Console.WriteLine("-mi " + magMin + " -mx " + magMax + " -pi " + phsMin + " -px " + phsMax);
magOut.Save(name + "-mag.png");
phsOut.Save(name + "-phs.png");
}
/// <summary>
/// NCC to find template
/// </summary>
private void NCCExecute()
{
try
{
Trace.TraceInformation("NCC matching start");
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
Mat dst = destination.Clone();
/// convert to gray image
Mat grayImage = new Mat();
CvInvoke.CvtColor(dst, grayImage, ColorConversion.Rgb2Gray);
/// use the sobel filter on the source image which returns the gradients in the X (Gx) and Y (Gy) direction.
Mat gx = new Mat();
Mat gy = new Mat();
CvInvoke.Sobel(grayImage, gx, DepthType.Cv64F, 1, 0);
CvInvoke.Sobel(grayImage, gy, DepthType.Cv64F, 0, 1);
/// compute the magnitude and direction
Mat magnitude = new Mat();
Mat direction = new Mat();
CvInvoke.CartToPolar(gx, gy, magnitude, direction);
/// template matching
var _gx = gx.ToImage <Gray, double>();
var _gy = gy.ToImage <Gray, double>();
var _magnitude = magnitude.ToImage <Gray, double>();
long totalLength = contoursLength;
double nMinScore = minScore / totalLength; // normalized min score
double nGreediness = (1 - greediness * minScore) / (1 - greediness) / totalLength;
double partialScore = 0;
double resultScore = 0;
int resultX = 0;
int resultY = 0;
for (int i = 0, h = grayImage.Height; i < h; i++)
{
for (int j = 0, w = grayImage.Width; j < w; j++)
{
double sum = 0;
long num = 0;
for (int m = 0, rank = contoursRelative.Size; m < rank; m++)
{
for (int n = 0, length = contoursRelative[m].Size; n < length; n++)
{
num += 1;
int curX = j + contoursRelative[m][n].X;
int curY = i + contoursRelative[m][n].Y;
if (curX < 0 || curY < 0 || curX > grayImage.Width - 1 || curY > grayImage.Height - 1)
{
continue;
}
double sdx = _gx.Data[curY, curX, 0];
double sdy = _gy.Data[curY, curX, 0];
double tdx = contoursInfo[m][n].DerivativeX;
double tdy = contoursInfo[m][n].DerivativeY;
if ((sdy != 0 || sdx != 0) && (tdx != 0 || tdy != 0))
{
double nMagnitude = _magnitude.Data[curY, curX, 0];
if (nMagnitude != 0)
{
sum += (sdx * tdx + sdy * tdy) * contoursInfo[m][n].MagnitudeN / nMagnitude;
}
}
partialScore = sum / num;
#if FAST
if (partialScore < minScore)
#else
if (partialScore < Math.Min((minScore - 1) + (nGreediness * num), nMinScore * num))
#endif
{ break; }
}
}
if (partialScore > resultScore)
{
resultScore = partialScore;
resultX = j;
resultY = i;
Trace.TraceInformation($"Current Score : {resultScore}");
}
}
}
Trace.TraceInformation($"Score : {resultScore}");
/// overlay display origin image, edge(green) and origin point(red)
/// NOTE: origin point is the first edge point
var point = new System.Drawing.Point(resultX, resultY);
Trace.TraceInformation($"Point : {point}");
CvInvoke.DrawContours(dst, contoursRelative, -1, new Bgr(System.Drawing.Color.Green).MCvScalar, 5, offset: point);
CvInvoke.Circle(dst, point, 2, new Bgr(System.Drawing.Color.Red).MCvScalar, -1);
grayImage.Dispose();
Destination.Dispose();
Destination = dst;
Trace.TraceInformation($"NCC matching end. time: {(double)stopwatch.ElapsedTicks * 1000 / Stopwatch.Frequency} ms");
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
Trace.TraceError(ex.StackTrace);
}
}
/// <summary>
/// Create a template
/// </summary>
private void TemplateExecute()
{
try
{
contoursRelative = new VectorOfVectorOfPoint();
contoursInfo = new List <PointInfo[]>();
contoursLength = 0;
/// clone origin image
Mat src = template.Clone();
/// convert to gray image
Mat grayImage = new Mat();
CvInvoke.CvtColor(src, grayImage, ColorConversion.Rgb2Gray);
/// using the canny algorithm to get edges
Mat output = new Mat();
Mat hierarchy = new Mat();
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
CvInvoke.Canny(grayImage, output, 100, 800);
CvInvoke.FindContours(output, contours, hierarchy, RetrType.External, ChainApproxMethod.ChainApproxNone);
/// use the sobel filter on the template image which returns the gradients in the X (Gx) and Y (Gy) direction.
Mat gx = new Mat();
Mat gy = new Mat();
CvInvoke.Sobel(grayImage, gx, DepthType.Cv64F, 1, 0);
CvInvoke.Sobel(grayImage, gy, DepthType.Cv64F, 0, 1);
/// compute the magnitude and direction
Mat magnitude = new Mat();
Mat direction = new Mat();
CvInvoke.CartToPolar(gx, gy, magnitude, direction);
/// save edge info
var _gx = gx.ToImage <Gray, double>();
var _gy = gy.ToImage <Gray, double>();
var _magnitude = magnitude.ToImage <Gray, double>();
double magnitudeTemp = 0;
PointInfo pointInfo = new PointInfo();
int originx = contours[0][0].X;
int originy = contours[0][0].Y;
for (int i = 0, m = contours.Size; i < m; i++)
{
int n = contours[i].Size;
contoursLength += n;
contoursInfo.Add(new PointInfo[n]);
System.Drawing.Point[] points = new System.Drawing.Point[n];
for (int j = 0; j < n; j++)
{
int x = contours[i][j].X;
int y = contours[i][j].Y;
points[j].X = x - originx;
points[j].Y = y - originy;
pointInfo.DerivativeX = _gx.Data[y, x, 0];
pointInfo.DerivativeY = _gy.Data[y, x, 0];
magnitudeTemp = _magnitude.Data[y, x, 0];
pointInfo.Magnitude = magnitudeTemp;
if (magnitudeTemp != 0)
{
pointInfo.MagnitudeN = 1 / magnitudeTemp;
}
contoursInfo[i][j] = pointInfo;
}
contoursRelative.Push(new VectorOfPoint(points));
}
/// overlay display origin image, edge(green) and origin point(red)
/// NOTE: origin point is the first edge point
CvInvoke.DrawContours(src, contours, -1, new Bgr(System.Drawing.Color.Green).MCvScalar, 5);
var point = new System.Drawing.Point(contours[0][0].X, contours[0][0].Y);
CvInvoke.Circle(src, point, 2, new Bgr(System.Drawing.Color.Red).MCvScalar, -1);
gx.Dispose();
_gx.Dispose();
gy.Dispose();
_gy.Dispose();
magnitude.Dispose();
_magnitude.Dispose();
direction.Dispose();
Template.Dispose();
Template = src;
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
Trace.TraceError(ex.StackTrace);
}
}
private void HOG()
{
Image <Gray, byte> slika = My_ImageRecieve.Convert <Gray, byte>();
Image <Gray, float> photoX = slika.Sobel(1, 0, 1);
Image <Gray, float> photoY = slika.Sobel(0, 1, 1);
Image <Gray, float> magnitude = new Image <Gray, float>(slika.Size); //magnitude spremenljivka za sliko
Image <Gray, float> direction = new Image <Gray, float>(slika.Size); //spremenljivka za kote
var binsMat = new float[5000, 10000][];
binType[,] beansMat = new binType[5000, 10000];
List <List <float[]> > biniMat2d = new List <List <float[]> >();
List <float[]> biniMat1d = new List <float[]>();
CvInvoke.CartToPolar(photoX, photoY, magnitude, direction, true);
//pretvorba iz 0-360 v 0-180
for (int i = 0; i < direction.Height; i++)
{
for (int j = 0; j < direction.Width; j++)
{
int temp = (int)direction.Data[i, j, 0];
temp = temp - 360;
if (temp < 0)
{
temp = temp * -1;
}
direction.Data[i, j, 0] = (byte)temp;
}
}
//konec pretvorbe
pictureBox5.Image = direction.ToBitmap();
int[,] celica = new int[8, 8];
for (int x = 0; x < direction.Height - 8; x = x + 8)
{
if (direction.Height - x >= 1)
{
currentbinMatY = binMatY;
Console.WriteLine(binMatY);
binMatY = 0;
for (int y = 0; y < direction.Width - 8; y = y + 8)
{
if (direction.Width - y >= 1)
{
for (int i = 0; i < 8; i++)
{
for (int j = 0; j < 8; j++)
{
bins = new float[9];
celica[i, j] = (int)direction.Data[x + i, y + j, 0];
if (celica[i, j] == 0 || celica[i, j] == 180)//0 ali 180 je enako //POLNI BIN
{
bins[0] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 0 && celica[i, j] < 20)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 0;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 10)
{
bins[0] = getMag * part2;
bins[1] = getMag * part;
}
else if (celica[i, j] > 10)
{
bins[0] = getMag * part;
bins[1] = getMag * part2;
}
else
{
bins[0] = getMag * part;
bins[1] = getMag * part2;
}
}
else if (celica[i, j] == 20)//POLNI BIN
{
bins[1] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 20 && celica[i, j] < 40)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 20;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 30)
{
bins[1] = getMag * part2;
bins[2] = getMag * part;
}
else if (celica[i, j] > 30)
{
bins[1] = getMag * part;
bins[2] = getMag * part2;
}
else
{
bins[1] = getMag * part;
bins[2] = getMag * part2;
}
}
else if (celica[i, j] == 40)//POLNI BIN
{
bins[2] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 40 && celica[i, j] < 60)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 40;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 50)
{
bins[2] = getMag * part2;
bins[3] = getMag * part;
}
else if (celica[i, j] > 50)
{
bins[2] = getMag * part;
bins[3] = getMag * part2;
}
else
{
bins[2] = getMag * part;
bins[3] = getMag * part2;
}
}
else if (celica[i, j] == 60)//POLNI BIN
{
bins[3] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 60 && celica[i, j] < 80)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 60;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 70)
{
bins[3] = getMag * part2;
bins[4] = getMag * part;
}
else if (celica[i, j] > 70)
{
bins[3] = getMag * part;
bins[4] = getMag * part2;
}
else
{
bins[3] = getMag * part;
bins[4] = getMag * part2;
}
}
else if (celica[i, j] == 80)//POLNI BIN
{
bins[4] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 80 && celica[i, j] < 100)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 80;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 90)
{
bins[4] = getMag * part2;
bins[5] = getMag * part;
}
else if (celica[i, j] > 90)
{
bins[4] = getMag * part;
bins[5] = getMag * part2;
}
else
{
bins[4] = getMag * part;
bins[5] = getMag * part2;
}
}
else if (celica[i, j] == 100)//POLNI BIN
{
bins[5] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 100 && celica[i, j] < 120)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 100;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
Console.WriteLine(magnitude.Data[x + i, y + j, 0]);
if (celica[i, j] < 110)
{
bins[5] = getMag * part2;
bins[6] = getMag * part;
}
else if (celica[i, j] > 110)
{
bins[5] = getMag * part;
bins[6] = getMag * part2;
}
else
{
bins[5] = getMag * part;
bins[6] = getMag * part2;
}
}
else if (celica[i, j] == 120)//POLNI BIN
{
bins[6] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 120 && celica[i, j] < 140)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 120;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 130)
{
bins[6] = getMag * part2;
bins[7] = getMag * part;
}
else if (celica[i, j] > 130)
{
bins[6] = getMag * part;
bins[7] = getMag * part2;
}
else
{
bins[6] = getMag * part;
bins[7] = getMag * part2;
}
}
else if (celica[i, j] == 140)//POLNI BIN
{
bins[7] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 140 && celica[i, j] < 160)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 140;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 150)
{
bins[7] = getMag * part2;
bins[8] = getMag * part;
}
else if (celica[i, j] > 150)
{
bins[7] = getMag * part;
bins[8] = getMag * part2;
}
else
{
bins[7] = getMag * part;
bins[8] = getMag * part2;
}
}
else if (celica[i, j] == 160)//POLNI BIN
{
bins[8] = (float)magnitude.Data[x + i, y + j, 0];
}
else if (celica[i, j] > 160 && celica[i, j] < 180)//RAZDELITEV MED SOSEDE
{
int tmpDir = celica[i, j];
float subLbin = tmpDir - 160;
float part = subLbin / 20;
float part2 = 1 - part;
float getMag = (float)magnitude.Data[x + i, y + j, 0];
if (celica[i, j] < 170)
{
bins[8] = getMag * part2;
bins[0] = getMag * part;
}
else if (celica[i, j] > 170)
{
bins[8] = getMag * part;
bins[0] = getMag * part2;
}
else
{
bins[8] = getMag * part;
bins[0] = getMag * part2;
}
}
} //j celica
} //i celica
biniMat1d.Add(bins);//list
beansMat[binMatX, binMatY].beans = bins;//probamo z strukturo
binsMat[binMatX, binMatY] = bins; //2d array of arrays
//Console.Write(beansMat[binMatX, binMatY].beans);
binMatY++; //++ y coordinate of 2d array of arrays
}//znotraj ifa za lego [i,j]WIDTH
}//direction width
biniMat2d.Add(biniMat1d); //list
binMatX++; //++ x coordinate of 2d array of arrays
}//znotraj ifa za lego [i,j]HEIGHT
}//direction height
Console.WriteLine("________");
float[] temp1 = new float[9];
for (int i = 0; i < binMatX - 1; i++) //-1 ker bomo z x + x+1 vedno pogledali z predzadnjim tudi zadnjega v x in y smeri
{
for (int j = 0; j < binMatY - 1; j++) //-||-
{
Console.WriteLine("[" + i + "," + j + "] = ");
//Console.WriteLine(beansMat[i, j].beans);
temp1 = beansMat[i, j].beans;
for (int k = 0; k < 9; k++)
{
Console.Write(temp1[k]);
}
Console.WriteLine(".");
} //end of Y
} //end of X
Console.WriteLine("______________");
//prepare cells for storage of bins for each cell and 1 blockBins that will concatenate 4 cells into 1 blockbin(4bins->1bin)
float[] cell1 = new float[9];
float[] cell2 = new float[9];
float[] cell3 = new float[9];
float[] cell4 = new float[9];
float[] blockBins = new float[37];
int blockBinC = 0;//counter
float[] concatBlockBinsNormalized = new float[binMatX * binMatY * 36];
int concatBlockBinsNormalizedC = 0;//counter
//prepare a SortedDictionary TODO
Dictionary <float, int> dictionary = new Dictionary <float, int>();
for (int i = 0; i < binMatX - 1; i++) //-1 ker bomo z x + x+1 vedno pogledali z predzadnjim tudi zadnjega v x in y smeri
{
for (int j = 0; j < binMatY - 1; j++) //-||-
{
//cell1 = binsMat[i, j];
cell1 = beansMat[i, j].beans;
for (int l = 0; l < 9; l++)
{
Console.Write(cell1[l]);
}//console wrrite test
Console.WriteLine(".");
for (int l = 0; l < 9; l++)
{
blockBins[blockBinC] = cell1[l];
blockBinC++;
}//go through floats in each cell and copy them to blockBins
//cell2 = binsMat[i, j+1];
cell2 = beansMat[i, j + 1].beans;
for (int l = 0; l < 9; l++)
{
blockBins[blockBinC] = cell2[l];
blockBinC++;
}//go through floats in each cell and copy them to blockBins
//cell3 = binsMat[i+1, j];
cell3 = beansMat[i + 1, j].beans;
for (int l = 0; l < 9; l++)
{
blockBins[blockBinC] = cell3[l];
blockBinC++;
}//go through floats in each cell and copy them to blockBins
//cell4 = binsMat[i+1, j+1];
cell4 = beansMat[i + 1, j + 1].beans;
for (int l = 0; l < 9; l++)
{
blockBins[blockBinC] = cell4[l];
blockBinC++;
}//go through floats in each cell and copy them to blockBins
//normalizacija blockBins:
float[] blockBinsNormalized = new float[37];
float suma = 0;
for (int k = 0; k < 37; k++)//racunanje skale
{
suma = suma + (blockBins[k] * blockBins[k]);
}
float scale = (float)Math.Sqrt(suma); //skala
for (int k = 0; k < 37; k++) //normalizacija blockBins*scale -> blockBinsNormalized
{
blockBinsNormalized[k] = blockBins[k] * scale;
}
for (int k = 0; k < 37; k++)//zlivanje v koncni vector vseh vrednosti
{
concatBlockBinsNormalized[concatBlockBinsNormalizedC] = blockBinsNormalized[k];
concatBlockBinsNormalizedC++;
}
blockBinC = 0;
} //end of Y
} //end of X
//izdelava slovarja
for (int i = 0; i < concatBlockBinsNormalizedC; i++)//zlivanje v koncni vector vseh vrednosti
{
// See whether it contains this string.
if (!dictionary.ContainsKey(concatBlockBinsNormalized[i]))
{
dictionary.Add(concatBlockBinsNormalized[i], 1);
}
if (dictionary.ContainsKey(concatBlockBinsNormalized[i]))
{
dictionary[concatBlockBinsNormalized[i]]++;
}
}
foreach (KeyValuePair <float, int> pair in dictionary)
{
Console.WriteLine("{0}, {1}", pair.Key, pair.Value);
}
}
/// <summary>
/// get the motion influence map from the processed frames.
/// </summary>
/// <param name="vid">Path of the video.</param>
/// <param name="xBlockSize">Horizontal size of the block of the block.</param>
/// <param name="yBlockSize">Vertical size of the block of the block.</param>
/// <param name="noOfRowInBlock">'Number of rows of the mega grid.</param>
/// <param name="noOfColInBlock">Number of columns of the mega gird.</param>
/// <param name="total_frames">Total frames that we will process.</param>
/// <param name="clustering">Boolean to determinate wherher to cluster later or not</param>
/// <param name="frame_nr">Number of the starting frame.</param>
/// <returns>Motion influence map.</returns>
public List <double[][][]> get_motion_influence_map(String vid, out int xBlockSize, out int yBlockSize, out int noOfRowInBlock, out int noOfColInBlock, out int total_frames, bool clustering, int frame_nr = 0)
{
List <double[][][]> ret = new List <double[][][]>();
xBlockSize = 0;
yBlockSize = 0;
noOfRowInBlock = 0;
noOfColInBlock = 0;
total_frames = 0;
try
{
mag = new Mat();
ang = new Mat();
frame = new Mat();
prev_frame = new Mat();
cap = new VideoCapture(vid);
if (!clustering)
{
total_frames = 3;
}
else
{
total_frames = Convert.ToInt32(cap.GetCaptureProperty(CapProp.FrameCount));
}
cap.SetCaptureProperty(CapProp.PosFrames, frame_nr);
frame = cap.QueryFrame();
prev_frame = frame;
}
catch (NullReferenceException except)
{
Console.WriteLine(except.Message);
}
int mm = 0;
Console.WriteLine("Total Frames : {0}", total_frames);
while (mm < total_frames - 1)
{
Console.WriteLine("Frame : " + frame_nr);
prev_frame = frame;
frame_nr += 1;
cap.SetCaptureProperty(CapProp.PosFrames, frame_nr);
frame = cap.QueryFrame();
Image <Gray, Byte> prev_grey_img = new Image <Gray, byte>(frame.Width, frame.Height);
Image <Gray, Byte> curr_grey_img = new Image <Gray, byte>(frame.Width, frame.Height);
Image <Gray, float> flow_x = new Image <Gray, float>(frame.Width, frame.Height);
Image <Gray, float> flow_y = new Image <Gray, float>(frame.Width, frame.Height);
curr_grey_img = frame.ToImage <Gray, byte>();
prev_grey_img = prev_frame.ToImage <Gray, Byte>();
CvInvoke.CalcOpticalFlowFarneback(prev_grey_img, curr_grey_img, flow_x, flow_y, 0.5, 3, 15, 3, 6, 1.3, 0);
CvInvoke.CartToPolar(flow_x, flow_y, mag, ang);
OptFlowOfBlocks.calcOptFlowOfBlocks(mag, ang, curr_grey_img, out opFlowOfBlocks, out centreOfBlocks,
out rows, out cols, out noOfRowInBlock, out noOfColInBlock, out xBlockSize, out yBlockSize, out blockSize);
motionInMapGenerator(opFlowOfBlocks, blockSize, centreOfBlocks, xBlockSize, yBlockSize, out motionInfVal);
ret.Add(motionInfVal);
mm++;
}
return(ret);
}
