void CascadeDetect(Mat image, int imgWidth, int imgHeight, ref BoxOutline outline)
{
MatOfRect hands = new MatOfRect();
Mat gray = new Mat(imgHeight, imgWidth, CvType.CV_8UC3);
Imgproc.cvtColor(image, gray, Imgproc.COLOR_BGR2GRAY);
Imgproc.equalizeHist(gray, gray);
cascadeDetector.detectMultiScale(
gray,
hands,
1.1,
2,
0 | Objdetect.CASCADE_DO_CANNY_PRUNING | Objdetect.CASCADE_SCALE_IMAGE | Objdetect.CASCADE_FIND_BIGGEST_OBJECT,
new Size(10, 10),
new Size());
OpenCVForUnity.CoreModule.Rect[] handsArray = hands.toArray();
if (handsArray.Length != 0)
{
outline = new BoxOutline
{
XMin = (float)handsArray[0].x,
XMax = (float)handsArray[0].x + handsArray[0].width,
YMin = (float)handsArray[0].y,
YMax = (float)handsArray[0].y + handsArray[0].height
};
Debug.Log("cascade: palm detected!");
}
else
{
outline = null;
}
}
private List <BoxOutline> GetBoxes(float[,,] boxes, float[,] scores, float[,] classes, double minScore)
{
var x = boxes.GetLength(0);
var y = boxes.GetLength(1);
var z = boxes.GetLength(2);
float ymin = 0, xmin = 0, ymax = 0, xmax = 0;
var results = new List <BoxOutline>();
for (int i = 0; i < x; i++)
{
for (int j = 0; j < y; j++)
{
if (scores[i, j] < minScore)
{
continue;
}
for (int k = 0; k < z; k++)
{
var box = boxes[i, j, k];
switch (k)
{
case 0:
ymin = box;
break;
case 1:
xmin = box;
break;
case 2:
ymax = box;
break;
case 3:
xmax = box;
break;
}
}
int value = Convert.ToInt32(classes[i, j]);
var label = this.labels[value];
var boxOutline = new BoxOutline
{
YMin = ymin,
XMin = xmin,
YMax = ymax,
XMax = xmax,
Label = label,
Score = scores[i, j],
};
results.Add(boxOutline);
}
}
return(results);
}
void PostAction(Texture2D tex, Mat mat, BoxOutline outl)
{
if (outl != null)
{
Imgproc.rectangle(mat, new Point(outl.XMin, outl.YMin), new Point(outl.XMax, outl.YMax), new Scalar(255, 0, 0));
}
Utils.matToTexture2D(mat, tex);
}
private void DrawBoxOutline(BoxOutline outline)
{
var xMin = outline.XMin * this.backgroundSize.x + this.backgroundOrigin.x;
var xMax = outline.XMax * this.backgroundSize.x + this.backgroundOrigin.x;
var yMin = outline.YMin * this.backgroundSize.y + this.backgroundOrigin.y;
var yMax = outline.YMax * this.backgroundSize.y + this.backgroundOrigin.y;
DrawRectangle(new Rect(xMin, yMin, xMax - xMin, yMax - yMin), 4, Color.red);
DrawLabel(new Rect(xMin + 10, yMin + 10, 200, 20), $"{outline.Label}: {(int)(outline.Score * 100)}%");
}
/// <summary>
/// Helper function to draw the box on your screen if you are in debug mode
/// </summary>
/// <param name="outline">The box that you want to draw</param>
void DrawBoxOutline(BoxOutline outline)
{
Vector2 diff = outline.cornerMax - outline.cornerMin;
//Vector2 midPoint = 0.5f * (outline.cornerMin + outline.cornerMax);
Rect boundingBox = new Rect(outline.cornerMin.x, outline.cornerMin.y, diff.x + 10, diff.y + 10);
Rect labelBox = new Rect(outline.cornerMin.x + 10, outline.cornerMin.y + 10, diff.x, diff.y);
DrawRectangle(boundingBox, 4);
GUI.Box(labelBox, labels[outline.ClassID], labelStyle);
}
private void TFDetect(Mat image, int imgWidth, int imgHeight, ref BoxOutline outline)
{
if (image == null)
{
Debug.Log("unable to find colors");
return;
}
var blob = Dnn.blobFromImage(image, 1, new Size(300, 300), new Scalar(0, 0, 0), true, false);
tfDetector.setInput(blob);
Mat prob = tfDetector.forward();
Mat newMat = prob.reshape(1, (int)prob.total() / prob.size(3));
float maxScore = 0;
int scoreInd = 0;
for (int i = 0; i < newMat.rows(); i++)
{
var score = (float)newMat.get(i, 2)[0];
if (score > maxScore)
{
maxScore = score;
scoreInd = i;
}
}
//Debug.Log(maxScore);
if (maxScore > 0.7)
{
float left = (float)(newMat.get(scoreInd, 3)[0] * imgWidth);
float top = (float)(newMat.get(scoreInd, 4)[0] * imgHeight);
float right = (float)(newMat.get(scoreInd, 5)[0] * imgWidth);
float bottom = (float)(newMat.get(scoreInd, 6)[0] * imgHeight);
left = (int)Mathf.Max(0, Mathf.Min(left, imgWidth - 1));
top = (int)Mathf.Max(0, Mathf.Min(top, imgHeight - 1));
right = (int)Mathf.Max(0, Mathf.Min(right, imgWidth - 1));
bottom = (int)Mathf.Max(0, Mathf.Min(bottom, imgHeight - 1));
outline = new BoxOutline
{
XMin = right,
XMax = left,
YMin = bottom,
YMax = top
};
}
else
{
outline = null;
}
prob.Dispose();
newMat.Dispose();
}
IEnumerator CreateBoxes(ResultJson result)
{
List <BoxOutline> boxes = new List <BoxOutline>();
if (result.BoidImages == null)
{
yield return(boxes);
}
UpdateWindowShift();
// Box format: x_min,y_min,x_max,y_max
// Assume boidImages.count == boidClasses.count
for (int i = 0; i < result.BoidImages.Count; i++)
{
BoxOutline box = new BoxOutline();
List <float> boxCoord = result.BoidImages[i];
if (DrawBoxes)
{
box.cornerMin = new Vector2(boxCoord[0] * scaleFactor + ShiftX, boxCoord[1] * scaleFactor + ShiftY);
box.cornerMax = new Vector2(boxCoord[2] * scaleFactor + ShiftX, boxCoord[3] * scaleFactor + ShiftY);
}
// Draw the ray by casting viewport point to ray
Vector2 boxCenter = new Vector2((boxCoord[0] + boxCoord[2]) / 2.0f, (boxCoord[1] + boxCoord[3]) / 2.0f);
boxCenter /= ImageSize;
Vector3 temp = new Vector3(boxCenter.x, boxCenter.y, 0.0f);
box.objectRay = boidEyes.ViewportPointToRay(temp);
//box.cornerMin = new Vector2(boxCoord[0], boxCoord[1]);
//box.cornerMax = new Vector3(boxCoord[2], boxCoord[3]);
if (ClassLabelDict.ContainsKey(result.BoidClasses[i]))
{
box.ClassID = ClassLabelDict[result.BoidClasses[i]];
boxes.Add(box);
Debug.Log(gameObject.GetInstanceID() + ": " + "Found class " + labels[box.ClassID]);
}
}
actualBoid.ParseCV(boxes);
// DebugBoxes(boxes);
yield return(boxes);
}
private void RecalculateBoundingBoxes()
{
foreach (var reference in references)
{
if (TemplateManager.LOADED_TEMPLATES.ContainsKey(reference.name))
{
var obj = TemplateManager.LOADED_TEMPLATES[reference.name];
var refBox = obj.GetBoundingBox();
refBox.Min += reference.position;
refBox.Max += reference.position;
if (referenceOutlines.ContainsKey(reference))
{
referenceOutlines[reference].boundingBox = refBox;
}
else
{
referenceOutlines[reference] = new BoxOutline(refBox);
}
}
}
}
void ContourDetect(Mat image, int imgWidth, int imgHeight, ref BoxOutline outline)
{
// filter skin color
var output_mask = GetSkinMask(image, imgWidth, imgHeight);
// find the convex hull of finger
int cx = -1, cy = -1;
FindDefects(output_mask, ref cx, ref cy, 1, 4);
if (cx == -1 && cy == -1)
{
outline = null;
return;
}
outline = new BoxOutline
{
XMin = (float)cx - 15,
XMax = (float)cx + 15,
YMin = (float)cy - 15,
YMax = (float)cy + 15
};
}
