// function that depicts results of optical flow operations
// requires reference to image being processed, the results of Farneback algorithm stored in flow_x and flow_y
// step gives the distance between pixels that are depicted, shift_that_counts is threshold for vector length that is used for calculations
private Image <Bgr, Byte> Draw_Farneback_flow_map(Image <Bgr, Byte> img_curr, Image <Gray, float> flow_x, Image <Gray, float> flow_y, OpticalFlowVariable optiVars)
{
// NOTE: flow Images (flow_x and flow_y) are organized like this:
// at index (is position of pixel before optical flow operation) of Image array
// the shift of this specific pixel after the flow operation is stored
// if no shift has occured value stored at index is zero
// (i.e., pixel[index] = 0
GC.Collect(0, GCCollectionMode.Forced);
Image <Bgr, Byte> blackFrame = new Image <Bgr, Byte>(new Bitmap(1280 / frameReduction, 720 / frameReduction));
System.Drawing.Point from_dot_xy = new System.Drawing.Point(); // point variable to draw lines between dots before and after flow (=vectors)
System.Drawing.Point to_dot_xy = new System.Drawing.Point(); // point variable, which will be endpoint of line between dots before and after flow
MCvScalar col; // variable to store color values of lines representing flow vectors
col.V0 = 100;
col.V1 = 255;
col.V2 = 0;
col.V3 = 100;
// for drawing central line based on window size
System.Drawing.Point[] window_centre = new System.Drawing.Point[2];
window_centre[0].X = img_curr.Width / 2;// * Convert.ToInt32(txt_resize_factor.Text)/ 2;
window_centre[0].Y = 0;
window_centre[1].X = img_curr.Width / 2; //* Convert.ToInt32(txt_resize_factor.Text) / 2;
window_centre[1].Y = orig_height;
// Point variables that constitute starting point for drawing summed and mean vectors onto image
System.Drawing.Point vector_right = new System.Drawing.Point();
System.Drawing.Point vector_left = new System.Drawing.Point();
// variables used for summing vectors to left and to the right of the window's centre
System.Drawing.Point vector_right_end_window = new System.Drawing.Point();
System.Drawing.Point vector_left_end_window = new System.Drawing.Point();
// determine centre of output window (needed for summed vectors)
int mid_point_horz = 1280 * frameReduction / 2; // width
int mid_point_vert = 720 * frameReduction / 2; // height
// landmark coordinates that are origin of direction vectors
// near centre of image window; to depict motion of left and right half of "body" (or more precisely, window)
vector_right.X = (mid_point_horz + 10) * optiVars.stepRate;
vector_right.Y = mid_point_vert * optiVars.stepRate;
vector_left.X = (mid_point_horz - 10) * optiVars.stepRate;
vector_left.Y = mid_point_vert * optiVars.stepRate;
// counting landmarks in flow field that exceed a certain value (shift_that_counts); left and right is based on centre of window (half of width)
double count_X_right = 0;
double count_Y_right = 0;
double count_X_left = 0;
double count_Y_left = 0;
// loops over image matrix; position of dots before and after optical flow operations are compared and vector is drawn between the old and the new position
for (int i = 0; i < flow_x.Rows; i += optiVars.stepRate) // NOTE: steps are given by step variable in arguments of method
{
for (int j = 0; j < flow_x.Cols; j += optiVars.stepRate) // BEGIN FOR
// pixel shift measured by optical flow is transferred to point variables
// storing starting point of motion (from_dot..) and its end points (to_dot...)
{
to_dot_xy.X = (int)flow_x.Data[i, j, 0]; // access single pixel of flow matrix where x-coords of pixel after flow are stored; only gives the shift
to_dot_xy.Y = (int)flow_y.Data[i, j, 0]; // access single pixel of flow matrix where y-coords of pixel after flow are stored; only gives the shift
from_dot_xy.X = j; // index of loop is position on image (here: x-coord); X is cols
from_dot_xy.Y = i; // index of of loop is position on image (here: y-coord); Y is rows
// LEFT SIDE OF WINDOW BASED CENTRE
if (j < window_centre[0].X)
{
// count the x and y indices and sum them when they exceed the value given by shift_that_counts (here:0)
if (Math.Abs(to_dot_xy.X) > optiVars.shiftThatCounts)
{
count_X_left++;
}
if (Math.Abs(to_dot_xy.Y) > optiVars.shiftThatCounts)
{
count_Y_left++;
}
// sum up vectors
vector_left_end_window.Y += to_dot_xy.Y;
vector_left_end_window.X += to_dot_xy.X;
}
else //(j > window_centre[0].X)// WINDOW BASED CENTRE
{
// like above; count the x and y indices and sum them
if (Math.Abs(to_dot_xy.X) > optiVars.shiftThatCounts)
{
count_X_right++;
}
if (Math.Abs(to_dot_xy.Y) > optiVars.shiftThatCounts)
{
count_Y_right++;
}
// sum vectors
vector_right_end_window.Y += to_dot_xy.Y;
vector_right_end_window.X += to_dot_xy.X;
}
to_dot_xy.X = from_dot_xy.X + to_dot_xy.X; // new x-coord position of pixel (taking into account distance from the origin)
to_dot_xy.Y = from_dot_xy.Y + to_dot_xy.Y; // new y-coord postion of pixel
// draw line between coords on image and pixel shift stored in flow field after applying optical-flow
if (GetDistance(from_dot_xy.X, from_dot_xy.Y, to_dot_xy.X, to_dot_xy.Y) > optiVars.shiftThatCounts)
{
CvInvoke.Line(blackFrame, from_dot_xy, to_dot_xy, col, 1);
}
//CvInvoke.Imshow("Flow field vectors", img_curr); // show image with flow depicted as lines
} // END of both for loops
}
Mat blackDst = new Mat();
Mat BlackMat = blackFrame.Mat;
using (GpuMat gMatSrc = new GpuMat())
using (GpuMat gMatDst = new GpuMat()) {
gMatSrc.Upload(BlackMat);
Emgu.CV.Cuda.CudaInvoke.Resize(gMatSrc, gMatDst, new Size(0, 0), frameReduction, frameReduction, Inter.Area);
gMatDst.Download(blackDst);
}
GC.Collect();
return(blackDst.ToImage <Bgr, Byte>());
}
// calculates the optical flow according to the Farneback algorithm
public Bitmap Dense_Optical_Flow(Bitmap bmp, OpticalFlowVariable optiVariables, Camera cam)
{
frameReduction = optiVariables.frameReduction < 1 ? 1 : optiVariables.frameReduction;
// frame becomes previous frame (i.e., prev_frame stores information about current frame)
prev_frame = matframe;
Image <Bgr, Byte> imageCV = new Image <Bgr, byte>(bmp); //Image Class from Emgu.CV
matframe = imageCV.Mat; //This is your Image converted to Mat
if (prev_frame == null)
{
return(bmp);
}
// frame_nr increment by number of steps given in textfield on user interface
frame_nr += 1;
// intialize this Image Matrix before resizing (see below), so it remains at original size
img_average_vectors = new Image <Bgr, byte>(matframe.Width, matframe.Height);
orig_height = matframe.Height;
Size n_size = new Size(matframe.Width / frameReduction,
matframe.Height / frameReduction);
// Resize frame and previous frame (smaller to reduce processing load)
//Source
Mat matFramDst = new Mat();
using (GpuMat gMatSrc = new GpuMat())
using (GpuMat gMatDst = new GpuMat()) {
gMatSrc.Upload(matframe);
Emgu.CV.Cuda.CudaInvoke.Resize(gMatSrc, gMatDst, new Size(0, 0), (double)1 / frameReduction, (double)1 / frameReduction);
gMatDst.Download(matFramDst);
}
matframe = matFramDst;
if (prev_frame.Height != matframe.Height)
{
return(bmp);
}
// images that are compared during the flow operations (see below)
// these need to be greyscale images
Image <Gray, Byte> prev_grey_img, curr_grey_img;
prev_grey_img = new Image <Gray, byte>(prev_frame.Width, prev_frame.Height);
curr_grey_img = new Image <Gray, byte>(matframe.Width, matframe.Height);
// Image arrays to store information of flow vectors (one image array for each direction, which is x and y)
Image <Gray, float> flow_x;
Image <Gray, float> flow_y;
flow_x = new Image <Gray, float>(matframe.Width, matframe.Height);
flow_y = new Image <Gray, float>(matframe.Width, matframe.Height);
// assign information stored in frame and previous frame in greyscale images (works without convert function)
CvInvoke.CvtColor(matframe, curr_grey_img, ColorConversion.Bgr2Gray);
CvInvoke.CvtColor(prev_frame, prev_grey_img, ColorConversion.Bgr2Gray);
// Apply Farneback dense optical flow
// parameters are the two greyscale images (these are compared)
// and two image arrays storing the flow information
// the results of the procedure are stored
// the rest of the parameters are:
// pryScale: specifies image scale to build pyramids: 0.5 means that each next layer is twice smaller than the former
// levels: number of pyramid levels: 1 means no extra layers
// winSize: the average window size; larger values = more robust to noise but more blur
// iterations: number of iterations at each pyramid level
// polyN: size of pixel neighbourhood: higher = more precision but more blur
// polySigma
// flags
CvInvoke.CalcOpticalFlowFarneback(prev_grey_img, curr_grey_img, flow_x, flow_y, 0.5, 3, 10, 3, 6, 1.3, 0);
// call function that shows results of Farneback algorithm
Image <Bgr, Byte> farnebackImg = Draw_Farneback_flow_map(matframe.ToImage <Bgr, Byte>(), flow_x, flow_y, optiVariables);// given in global variables section
// Release memory
prev_grey_img.Dispose();
curr_grey_img.Dispose();
flow_x.Dispose();
flow_y.Dispose();
//return farnebackImg.ToBitmap();
Image <Bgra, Byte> alphaImgShape = new Image <Bgra, byte>(imageCV.Size.Width, imageCV.Size.Height, new Bgra(0, 0, 0, .5));
CvInvoke.AddWeighted(alphaImgShape, .5, BlackTransparent(farnebackImg), .5, 0, alphaImgShape);
Mat alphaimg = new Mat();
CvInvoke.CvtColor(imageCV, alphaimg, ColorConversion.Bgr2Bgra);
if (CudaInvoke.HasCuda)
{
using (GpuMat gMatSrc = new GpuMat())
using (GpuMat gMatSrc2 = new GpuMat())
using (GpuMat gMatDst = new GpuMat()) {
gMatSrc.Upload(alphaimg);
gMatSrc2.Upload(alphaImgShape);
CudaInvoke.AlphaComp(gMatSrc, gMatSrc2, gMatDst, AlphaCompTypes.Plus);
gMatDst.Download(alphaimg);
}
return(alphaimg.Bitmap);
}
else
{
return(Overlay(imageCV, alphaImgShape).ToBitmap());
}
}
