// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net.empty() || classes == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
blob = Dnn.blobFromImage(bgrMat, 1, new Size(224, 224), new Scalar(104, 117, 123), false, true);
net.setInput(blob);
Mat prob = net.forward();
Core.MinMaxLocResult minmax = Core.minMaxLoc(prob.reshape(1, 1));
// Debug.Log ("Best match " + (int)minmax.maxLoc.x);
// Debug.Log ("Best match class " + classes [(int)minmax.maxLoc.x]);
// Debug.Log ("Probability: " + minmax.maxVal * 100 + "%");
prob.Dispose();
Imgproc.putText(rgbaMat, "Best match class " + classes [(int)minmax.maxLoc.x], new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net.empty() || classes == null)
{
Imgproc.putText(rgbaMat, "model file or class names list file is not loaded.", new Point(5, rgbaMat.rows() - 50), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "The model and class names list can be downloaded here:", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
blob = Dnn.blobFromImage(rgbaMat, 1, new Size(224, 224), new Scalar(104, 117, 123), false);
net.setInput(blob);
Mat prob = net.forward();
Core.MinMaxLocResult minmax = Core.minMaxLoc(prob.reshape(1, 1));
// Debug.Log ("Best match " + (int)minmax.maxLoc.x);
// Debug.Log ("Best match class " + classes [(int)minmax.maxLoc.x]);
// Debug.Log ("Probability: " + minmax.maxVal * 100 + "%");
prob.Dispose();
Imgproc.putText(rgbaMat, "Best match class " + classes [(int)minmax.maxLoc.x], new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
private MatOfPoint OrderCornerPoints(MatOfPoint corners)
{
if (corners.size().area() <= 0 || corners.rows() < 4)
{
return(corners);
}
// rearrange the points in the order of upper left, upper right, lower right, lower left.
using (Mat x = new Mat(corners.size(), CvType.CV_32SC1))
using (Mat y = new Mat(corners.size(), CvType.CV_32SC1))
using (Mat d = new Mat(corners.size(), CvType.CV_32SC1))
using (Mat dst = new Mat(corners.size(), CvType.CV_32SC2))
{
Core.extractChannel(corners, x, 0);
Core.extractChannel(corners, y, 1);
// the sum of the upper left points is the smallest and the sum of the lower right points is the largest.
Core.add(x, y, d);
Core.MinMaxLocResult result = Core.minMaxLoc(d);
dst.put(0, 0, corners.get((int)result.minLoc.y, 0));
dst.put(2, 0, corners.get((int)result.maxLoc.y, 0));
// the difference in the upper right point is the smallest, and the difference in the lower left is the largest.
Core.subtract(y, x, d);
result = Core.minMaxLoc(d);
dst.put(1, 0, corners.get((int)result.minLoc.y, 0));
dst.put(3, 0, corners.get((int)result.maxLoc.y, 0));
dst.copyTo(corners);
}
return(corners);
}
public Point[] calc_peaks(Mat im,
Point[] points,
OpenCVForUnity.Size ssize)
{
int n = points.Length;
// Debug.Log ("n == int(patches.size()) " + patches.Count);
using (Mat pt = (new MatOfPoint2f(points)).reshape(1, 2 * n))
using (Mat S = calc_simil(pt))
using (Mat Si = inv_simil(S))
{
float[] pt_float = new float[pt.total()];
Utils.copyFromMat <float>(pt, pt_float);
int pt_cols = pt.cols();
float[] S_float = new float[S.total()];
Utils.copyFromMat <float>(S, S_float);
int S_cols = S.cols();
float[] A_float = new float[2 * 3];
Point[] pts = apply_simil(Si, points);
for (int i = 0; i < n; i++)
{
OpenCVForUnity.Size wsize = new OpenCVForUnity.Size(ssize.width + patches [i].patch_size().width, ssize.height + patches [i].patch_size().height);
using (Mat A = new Mat(2, 3, CvType.CV_32F))
{
Utils.copyFromMat <float>(A, A_float);
int A_cols = A.cols();
A_float [0] = S_float [0];
A_float [1] = S_float [1];
A_float [1 * A_cols] = S_float [1 * S_cols];
A_float [(1 * A_cols) + 1] = S_float [(1 * S_cols) + 1];
A_float [2] = (float)(pt_float [(2 * pt_cols) * i] -
(A_float [0] * (wsize.width - 1) / 2 + A_float [1] * (wsize.height - 1) / 2));
A_float [(1 * A_cols) + 2] = (float)(pt_float [((2 * pt_cols) * i) + 1] -
(A_float [1 * A_cols] * (wsize.width - 1) / 2 + A_float [(1 * A_cols) + 1] * (wsize.height - 1) / 2));
Utils.copyToMat(A_float, A);
using (Mat I = new Mat())
{
Imgproc.warpAffine(im, I, A, wsize, Imgproc.INTER_LINEAR + Imgproc.WARP_INVERSE_MAP);
using (Mat R = patches [i].calc_response(I, false))
{
Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(R);
pts [i].x = pts [i].x + minMaxLocResult.maxLoc.x - 0.5 * ssize.width;
pts [i].y = pts [i].y + minMaxLocResult.maxLoc.y - 0.5 * ssize.height;
}
}
}
}
return(apply_simil(S, pts));
}
}
/// Normalizes [I] such that min(I) -> 0 and max(I) -> 255
static void DesaturateMat(ref Mat I)
{
Core.MinMaxLocResult res = Core.minMaxLoc(I);
double alpha = 255 / (res.maxVal - res.minVal);
double beta = -res.minVal * alpha;
Mat O = (I - new Scalar(res.minVal)) * alpha;
Debug.LogFormat("Mat Max and Min: {0}, {1} \n Alpha & Beta: {2}, {3} \n I(200, 200) -> O(200, 200): {4} vs. {5} vs. {6}",
res.maxVal, res.minVal, alpha, beta, I.get(200, 200)[0], (alpha * I.get(200, 200)[0]) + beta, O.get(200, 200)[0]);
I = O;
}
void Start()
{
grayMat = Imgcodecs.imread(Application.dataPath + "/Textures/feature.jpg", 0);
outputHist = new Mat();
images = new List <Mat>();
Core.split(grayMat, images);
//定义变量计算直方图
Mat mask = new Mat(); //不空即可
MatOfInt channels = new MatOfInt(new int[] { 1 });
MatOfInt histSize = new MatOfInt(new int[] { 256 });;
MatOfFloat ranges = new MatOfFloat(new float[] { 0, 255 });
bool accum = false; //是否累积
Imgproc.calcHist(images, channels, mask, outputHist, histSize, ranges, accum);
Debug.Log(outputHist);
//画出直方图
int scale = 1;
Mat histPic = new Mat(256, 256 * scale, CvType.CV_8U, new Scalar(255)); //单通道255，白色
//找到最大值和最小值
Core.MinMaxLocResult res = Core.minMaxLoc(grayMat); //注意不要搞错Mat，是对grayMat计算
double minValue = res.minVal;
double maxValue = res.maxVal;
Debug.Log("min:" + minValue + ", max:" + maxValue);
//纵坐标缩放比例
double rate = (256 / maxValue) * 1d;
for (int i = 0; i < 256; i++)
{
//得到每个i和箱子的值
//float value = outputHist.at<float>(i);
byte[] byteArray = new byte[grayMat.width() * grayMat.height()];
Utils.copyFromMat <byte>(grayMat, byteArray);
float value = (float)byteArray[i];
//Debug.Log(value);
//画直线
Imgproc.line(histPic, new Point(i * scale, 256), new Point(i * scale, 256 - value * rate), new Scalar(0)); //单通道0，黑色
}
Texture2D t2d = new Texture2D(histPic.width(), histPic.height());
Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero);
m_showImage.sprite = sp;
m_showImage.preserveAspect = true;
Utils.matToTexture2D(histPic, t2d);
}
public Mat multiValueThresholdRGB(Mat img, List <int> rValueList, List <int> gValueList, List <int> bValueList, int seperation)
{
Mat imgGray = new Mat(img.rows(), img.cols(), CvType.CV_8UC1);
Imgproc.cvtColor(img, imgGray, Imgproc.COLOR_BGR2GRAY);
Core.MinMaxLocResult minMaxResult = Core.minMaxLoc(imgGray);
int threshValue = (int)minMaxResult.minVal;
int threshValueInterval = (int)((minMaxResult.maxVal - minMaxResult.minVal) / seperation);
Mat multiThreshMat = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
Mat multiThreshStageMat = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
Mat colorStageMat = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
Mat threshMask = new Mat(img.rows(), img.cols(), CvType.CV_8UC1);
colorStageMat = createMonochromaticRGB(rValueList[(seperation - 2)],
gValueList[(seperation - 2)],
bValueList[(seperation - 2)],
img.cols(), img.rows());
for (int i = 0; i < seperation; i++)
{
colorStageMat = createMonochromaticRGB(rValueList[(seperation - i - 1)],
gValueList[(seperation - i - 1)],
bValueList[(seperation - i - 1)],
img.cols(), img.rows());
if (i < seperation - 1)
{
if (i == 0)
{
colorStageMat = create_form(colorStageMat);
}
threshValue += threshValueInterval;
threshMask = rangeThreshold(imgGray, threshValue - threshValueInterval, threshValue, 255);
Core.bitwise_and(colorStageMat, colorStageMat, multiThreshStageMat, threshMask);
Core.bitwise_or(multiThreshMat, multiThreshStageMat, multiThreshMat);
}
else if (i == seperation - 1)
{
Imgproc.threshold(imgGray, threshMask, threshValue, 255, Imgproc.THRESH_BINARY);
Core.bitwise_and(colorStageMat, colorStageMat, multiThreshStageMat, threshMask);
Core.bitwise_or(multiThreshMat, multiThreshStageMat, multiThreshMat);
}
}
return(multiThreshMat);
}
public Mat multiValueThresholdHSV(Mat img, List <int> hValueList, List <int> sValueList, List <int> vValueList, int seperation)
{
int imgWidth = img.cols();
int imgHeight = img.rows();
Mat imgGray = new Mat(imgHeight, imgWidth, CvType.CV_8UC1);
Imgproc.cvtColor(img, imgGray, Imgproc.COLOR_BGR2GRAY);
Core.MinMaxLocResult minMaxResult = Core.minMaxLoc(imgGray);
int minGrayValue = (int)minMaxResult.minVal;
int maxGrayValue = (int)minMaxResult.maxVal;
int threshValue = minGrayValue;
int threshValueInterval = (int)((maxGrayValue - minGrayValue) / seperation);
Mat multiThreshMat = new Mat(imgHeight, imgWidth, CvType.CV_8UC3);
Mat multiThreshStageMat = new Mat(imgHeight, imgWidth, CvType.CV_8UC3);
Mat colorStageMat = new Mat(imgHeight, imgWidth, CvType.CV_8UC3);
Mat threshMask = new Mat(imgHeight, imgWidth, CvType.CV_8UC1);
colorStageMat = createMonochromaticHSV2RGB(hValueList[(seperation - 2)],
sValueList[(seperation - 2)],
vValueList[(seperation - 2)],
imgWidth, imgHeight);
for (int i = 0; i < seperation; i++)
{
colorStageMat = createMonochromaticRGB(hValueList[(seperation - i - 1)],
sValueList[(seperation - i - 1)],
vValueList[(seperation - i - 1)],
imgWidth, imgHeight);
if (i < seperation - 1)
{
threshValue += threshValueInterval;
threshMask = rangeThreshold(imgGray, threshValue - threshValueInterval, threshValue, 255);
Core.bitwise_and(colorStageMat, colorStageMat, multiThreshStageMat, threshMask);
Core.bitwise_or(multiThreshMat, multiThreshStageMat, multiThreshMat);
}
else if (i == seperation - 1)
{
Imgproc.threshold(imgGray, threshMask, threshValue, 255, Imgproc.THRESH_BINARY);
Core.bitwise_and(colorStageMat, colorStageMat, multiThreshStageMat, threshMask);
Core.bitwise_or(multiThreshMat, multiThreshStageMat, multiThreshMat);
}
}
return(multiThreshMat);
}
public Point[] calc_peaks(Mat im,
Point[] points,
OpenCVForUnity.Size ssize)
{
int n = points.Length;
// Debug.Log ("n == int(patches.size()) " + patches.Count);
using (Mat pt = (new MatOfPoint2f(points)).reshape(1, 2 * n))
using (Mat S = calc_simil(pt))
using (Mat Si = inv_simil(S)) {
Point[] pts = apply_simil(Si, points);
for (int i = 0; i < n; i++)
{
OpenCVForUnity.Size wsize = new OpenCVForUnity.Size(ssize.width + patches [i].patch_size().width, ssize.height + patches [i].patch_size().height);
using (Mat A = new Mat(2, 3, CvType.CV_32F)) {
A.put(0, 0, S.get(0, 0) [0]);
A.put(0, 1, S.get(0, 1) [0]);
A.put(1, 0, S.get(1, 0) [0]);
A.put(1, 1, S.get(1, 1) [0]);
A.put(0, 2, pt.get(2 * i, 0) [0] -
(A.get(0, 0) [0] * (wsize.width - 1) / 2 + A.get(0, 1) [0] * (wsize.height - 1) / 2));
A.put(1, 2, pt.get(2 * i + 1, 0) [0] -
(A.get(1, 0) [0] * (wsize.width - 1) / 2 + A.get(1, 1) [0] * (wsize.height - 1) / 2));
using (Mat I = new Mat()) {
Imgproc.warpAffine(im, I, A, wsize, Imgproc.INTER_LINEAR + Imgproc.WARP_INVERSE_MAP);
using (Mat R = patches [i].calc_response(I, false)) {
Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(R);
pts [i].x = pts [i].x + minMaxLocResult.maxLoc.x - 0.5 * ssize.width;
pts [i].y = pts [i].y + minMaxLocResult.maxLoc.y - 0.5 * ssize.height;
}
}
}
}
return(apply_simil(S, pts));
}
}
private Mat get_template(CascadeClassifier clssfr, Rect area, int size)
{
Mat template = new Mat();
Mat mROI = mGray.Submat(area);
MatOfRect eyes = new MatOfRect();
Point iris = new Point();
Rect eye_template = new Rect();
clssfr.DetectMultiScale(mROI, eyes, 1.15, 2, Objdetect.CascadeFindBiggestObject | Objdetect.CascadeScaleImage,
new Size(30, 30), new Size());
Rect[] eyesArray = eyes.ToArray();
for (int i = 0; i < eyesArray.Length;)
{
Rect e = eyesArray[i];
e.X = area.X + e.X;
e.Y = area.Y + e.Y;
Rect eye_only_rectangle = new Rect((int)e.Tl().X,
(int)(e.Tl().Y + e.Height * 0.4), (int)e.Width,
(int)(e.Height * 0.6));
mROI = mGray.Submat(eye_only_rectangle);
Mat vyrez = mRgba.Submat(eye_only_rectangle);
Core.MinMaxLocResult mmG = Core.MinMaxLoc(mROI);
Imgproc.Circle(vyrez, mmG.MinLoc, 2, new Scalar(255, 255, 255, 255), 2);
iris.X = mmG.MinLoc.X + eye_only_rectangle.X;
iris.Y = mmG.MinLoc.Y + eye_only_rectangle.Y;
eye_template = new Rect((int)iris.X - size / 2, (int)iris.Y - size / 2, size, size);
Imgproc.Rectangle(mRgba, eye_template.Tl(), eye_template.Br(), new Scalar(255, 0, 0, 255), 2);
template = (mGray.Submat(eye_template)).Clone();
return(template);
}
return(template);
}
protected void decodeText(Mat scores, out string text)
{
text = "";
const string alphabet = "0123456789abcdefghijklmnopqrstuvwxyz";
char[] alphabetCharArr = alphabet.ToCharArray();
Mat scoresMat = scores.reshape(1, scores.size(0));
List <char> elements = new List <char>();
for (int rowIndex = 0; rowIndex < scoresMat.rows(); ++rowIndex)
{
Core.MinMaxLocResult result = Core.minMaxLoc(scoresMat.row(rowIndex));
Point p = result.maxLoc;
if ((int)p.x > 0 && (int)p.x <= alphabet.Length)
{
elements.Add(alphabetCharArr[(int)p.x - 1]);
}
else
{
elements.Add('-');
}
}
if (elements.Count > 0 && elements[0] != '-')
{
text += elements[0];
}
for (int elementIndex = 1; elementIndex < elements.Count; ++elementIndex)
{
if (elementIndex > 0 && elements[elementIndex] != '-' && elements[elementIndex - 1] != elements[elementIndex])
{
text += elements[elementIndex];
}
}
}
/// <summary>
/// Get result form all output
/// </summary>
/// <param name="output"></param>
/// <param name="image"></param>
/// <param name="threshold"></param>
/// <param name="nmsThreshold">threshold for nms</param>
/// <param name="nms">Enable Non-maximum suppression or not</param>
private static void GetResult(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold, bool nms = true)
{
//for nms
List <int> classIds = new List <int>();
List <float> confidences = new List <float>();
List <float> probabilities = new List <float>();
List <Rect2d> boxes = new List <Rect2d>();
var w = image.width();
var h = image.height();
/*
* YOLO3 COCO trainval output
* 0 1 : center 2 3 : w/h
* 4 : confidence 5 ~ 84 : class probability
*/
const int prefix = 5; //skip 0~4
foreach (Mat prob in output)
{
for (int i = 0; i < prob.rows(); i++)
{
var confidence = (float)prob.get(i, 4)[0];
if (confidence > threshold)
{
//get classes probability
Core.MinMaxLocResult minAndMax = Core.minMaxLoc(prob.row(i).colRange(prefix, prob.cols()));
int classes = (int)minAndMax.maxLoc.x;
var probability = (float)prob.get(i, classes + prefix)[0];
if (probability > threshold) //more accuracy, you can cancel it
{
//get center and width/height
float centerX = (float)prob.get(i, 0)[0] * w;
float centerY = (float)prob.get(i, 1)[0] * h;
float width = (float)prob.get(i, 2)[0] * w;
float height = (float)prob.get(i, 3)[0] * h;
if (!nms)
{
// draw result (if don't use NMSBoxes)
Draw(image, classes, confidence, probability, centerX, centerY, width, height);
continue;
}
//put data to list for NMSBoxes
classIds.Add(classes);
confidences.Add(confidence);
probabilities.Add(probability);
boxes.Add(new Rect2d(centerX, centerY, width, height));
}
}
}
}
if (!nms)
{
return;
}
//using non-maximum suppression to reduce overlapping low confidence box
MatOfRect2d bboxes = new MatOfRect2d();
MatOfFloat scores = new MatOfFloat();
MatOfInt indices = new MatOfInt();
bboxes.fromList(boxes);
scores.fromList(probabilities);
Dnn.NMSBoxes(bboxes, scores, threshold, nmsThreshold, indices);
int[] indicesA = indices.toArray();
foreach (var i in indicesA)
{
var box = boxes[i];
Draw(image, classIds[i], confidences[i], probabilities[i], box.x, box.y, box.width, box.height);
}
}
void Run()
{
Utils.setDebugMode(true);
Mat img = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(img, img, Imgproc.COLOR_RGBA2BGR);
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
if (net != null)
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat input = Dnn.blobFromImage(img, 1.0, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.setInput(input, "image");
Mat output = net.forward("Openpose/concat_stage7");
output = output.reshape(1, 57);
List <Point> points = new List <Point>();
for (int i = 0; i < BODY_PARTS.Count; i++)
{
Mat heatMap = output.row(i).reshape(1, 46);
Core.MinMaxLocResult result = Core.minMaxLoc(heatMap);
heatMap.Dispose();
double x = (frameWidth * result.maxLoc.x) / 46;
double y = (frameHeight * result.maxLoc.y) / 46;
if (result.maxVal > 0.3)
{
points.Add(new Point(x, y));
}
else
{
points.Add(null);
}
}
for (int i = 0; i < POSE_PAIRS.GetLength(0); i++)
{
string partFrom = POSE_PAIRS[i, 0];
string partTo = POSE_PAIRS[i, 1];
int idFrom = BODY_PARTS[partFrom];
int idTo = BODY_PARTS[partTo];
if (points[idFrom] != null && points[idTo] != null)
{
Debug.Log("x=" + points[idFrom].x + " y=" + points[idFrom].y);
Imgproc.line(img, points[idFrom], points[idTo], new Scalar(0, 255, 0), 3);
Imgproc.ellipse(img, points[idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
Imgproc.ellipse(img, points[idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
float avgFrameRate = Time.frameCount / Time.time;
Imgproc.putText(img, "FR=" + avgFrameRate, new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
}
private void match_eye(Rect area, Mat mTemplate, int type)
{
Point matchLoc;
Mat mROI = mGray.Submat(area);
int result_cols = mROI.Cols() - mTemplate.Cols() + 1;
int result_rows = mROI.Rows() - mTemplate.Rows() + 1;
// Check for bad template size
if (mTemplate.Cols() == 0 || mTemplate.Rows() == 0)
{
return;
}
Mat mResult = new Mat(result_cols, result_rows, CvType.Cv8u);
switch (type)
{
case TM_SQDIFF:
Imgproc.MatchTemplate(mROI, mTemplate, mResult, Imgproc.TmSqdiff);
break;
case TM_SQDIFF_NORMED:
Imgproc.MatchTemplate(mROI, mTemplate, mResult, Imgproc.TmSqdiffNormed);
break;
case TM_CCOEFF:
Imgproc.MatchTemplate(mROI, mTemplate, mResult, Imgproc.TmCcoeff);
break;
case TM_CCOEFF_NORMED:
Imgproc.MatchTemplate(mROI, mTemplate, mResult, Imgproc.TmCcoeffNormed);
break;
case TM_CCORR:
Imgproc.MatchTemplate(mROI, mTemplate, mResult, Imgproc.TmCcorr);
break;
case TM_CCORR_NORMED:
Imgproc.MatchTemplate(mROI, mTemplate, mResult, Imgproc.TmCcorrNormed);
break;
}
Core.MinMaxLocResult mmres = Core.MinMaxLoc(mResult);
// there is difference in matching methods - best match is max/min value
if (type == TM_SQDIFF || type == TM_SQDIFF_NORMED)
{
matchLoc = mmres.MinLoc;
}
else
{
matchLoc = mmres.MaxLoc;
}
Point matchLoc_tx = new Point(matchLoc.X + area.X, matchLoc.Y + area.Y);
Point matchLoc_ty = new Point(matchLoc.X + mTemplate.Cols() + area.X,
matchLoc.Y + mTemplate.Rows() + area.Y);
Imgproc.Rectangle(mRgba, matchLoc_tx, matchLoc_ty, new Scalar(255, 255, 0, 255));
Rect rec = new Rect(matchLoc_tx, matchLoc_ty);
}
void Run(string jpg_path)
{
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(jpg_path);
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (!string.IsNullOrEmpty(graph_filepath))
{
net = Dnn.readNetFromTensorflow(graph_filepath);
}
if (net == null)
{
Imgproc.putText(img, "Model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat input = Dnn.blobFromImage(img, 1.0, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.setInput(input, "image");
Mat output = net.forward("Openpose/concat_stage7");
output = output.reshape(1, 57);
List <Point> points = new List <Point>();
for (int i = 0; i < BODY_PARTS.Count; i++)
{
Mat heatMap = output.row(i).reshape(1, 46);
Core.MinMaxLocResult result = Core.minMaxLoc(heatMap);
heatMap.Dispose();
double x = (frameWidth * result.maxLoc.x) / 46;
double y = (frameHeight * result.maxLoc.y) / 46;
if (result.maxVal > 0.3)
{
points.Add(new Point(x, y));
}
else
{
points.Add(null);
}
}
for (int i = 0; i < POSE_PAIRS.GetLength(0); i++)
{
string partFrom = POSE_PAIRS[i, 0];
string partTo = POSE_PAIRS[i, 1];
int idFrom = BODY_PARTS[partFrom];
int idTo = BODY_PARTS[partTo];
if (points[idFrom] != null && points[idTo] != null)
{
Imgproc.line(img, points[idFrom], points[idTo], new Scalar(0, 255, 0), 3);
Imgproc.ellipse(img, points[idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
Imgproc.ellipse(img, points[idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
}
}
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/COCO_val2014_000000000589.jpg is not loaded.The image file can be downloaded here: \"https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/examples/media/COCO_val2014_000000000589.jpg\" folder. ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(caffemodel_filepath) || string.IsNullOrEmpty(prototxt_filepath))
{
Debug.LogError("model file is not loaded. The model and prototxt file can be downloaded here: \"http://posefs1.perception.cs.cmu.edu/OpenPose/models/pose/mpi/pose_iter_160000.caffemodel\",\"https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/openpose_pose_mpi_faster_4_stages.prototxt\". Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(prototxt_filepath, caffemodel_filepath);
//Intel's Deep Learning Inference Engine backend is supported on Windows 64bit platform only. Please refer to ReadMe.pdf for the setup procedure.
//net.setPreferableBackend (Dnn.DNN_BACKEND_INFERENCE_ENGINE);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat input = Dnn.blobFromImage(img, 1.0 / 255, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.setInput(input);
// TickMeter tm = new TickMeter ();
// tm.start ();
Mat output = net.forward();
// tm.stop ();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
output = output.reshape(1, 16);
float[] data = new float[46 * 46];
List <Point> points = new List <Point> ();
for (int i = 0; i < BODY_PARTS.Count; i++)
{
output.get(i, 0, data);
Mat heatMap = new Mat(1, data.Length, CvType.CV_32FC1);
heatMap.put(0, 0, data);
//Originally, we try to find all the local maximums. To simplify a sample
//we just find a global one. However only a single pose at the same time
//could be detected this way.
Core.MinMaxLocResult result = Core.minMaxLoc(heatMap);
heatMap.Dispose();
double x = (frameWidth * (result.maxLoc.x % 46)) / 46;
double y = (frameHeight * (result.maxLoc.x / 46)) / 46;
if (result.maxVal > 0.1)
{
points.Add(new Point(x, y));
}
else
{
points.Add(null);
}
}
for (int i = 0; i < POSE_PAIRS.GetLength(0); i++)
{
string partFrom = POSE_PAIRS [i, 0];
string partTo = POSE_PAIRS [i, 1];
int idFrom = BODY_PARTS [partFrom];
int idTo = BODY_PARTS [partTo];
if (points [idFrom] != null && points [idTo] != null)
{
Imgproc.line(img, points [idFrom], points [idTo], new Scalar(0, 255, 0), 3);
Imgproc.ellipse(img, points [idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
Imgproc.ellipse(img, points [idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
}
}
MatOfDouble timings = new MatOfDouble();
long t = net.getPerfProfile(timings);
Debug.Log("t: " + t);
Debug.Log("timings.dump(): " + timings.dump());
double freq = Core.getTickFrequency() / 1000;
Debug.Log("freq: " + freq);
Imgproc.putText(img, (t / freq) + "ms", new Point(10, img.height() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar(0, 0, 255), 2);
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image_filepath);
if (img.empty())
{
Debug.LogError(image_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(caffemodel_filepath) || string.IsNullOrEmpty(prototxt_filepath))
{
Debug.LogError(caffemodel_filepath + " or " + prototxt_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
net = Dnn.readNet(prototxt_filepath, caffemodel_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat input = Dnn.blobFromImage(img, inScale, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.setInput(input);
//TickMeter tm = new TickMeter ();
//tm.start ();
Mat output = net.forward();
//tm.stop ();
//Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
//Debug.Log("output.size(0) " + output.size(0));
//Debug.Log("output.size(1) " + output.size(1));
//Debug.Log("output.size(2) " + output.size(2));
//Debug.Log("output.size(3) " + output.size(3));
float[] data = new float[output.size(2) * output.size(3)];
output = output.reshape(1, output.size(1));
List <Point> points = new List <Point>();
for (int i = 0; i < BODY_PARTS.Count; i++)
{
output.get(i, 0, data);
Mat heatMap = new Mat(1, data.Length, CvType.CV_32FC1);
heatMap.put(0, 0, data);
//Originally, we try to find all the local maximums. To simplify a sample
//we just find a global one. However only a single pose at the same time
//could be detected this way.
Core.MinMaxLocResult result = Core.minMaxLoc(heatMap);
heatMap.Dispose();
double x = (frameWidth * (result.maxLoc.x % 46)) / 46;
double y = (frameHeight * (result.maxLoc.x / 46)) / 46;
if (result.maxVal > 0.1)
{
points.Add(new Point(x, y));
}
else
{
points.Add(null);
}
}
for (int i = 0; i < POSE_PAIRS.GetLength(0); i++)
{
string partFrom = POSE_PAIRS[i, 0];
string partTo = POSE_PAIRS[i, 1];
int idFrom = BODY_PARTS[partFrom];
int idTo = BODY_PARTS[partTo];
if (points[idFrom] != null && points[idTo] != null)
{
Imgproc.line(img, points[idFrom], points[idTo], new Scalar(0, 255, 0), 3);
Imgproc.ellipse(img, points[idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
Imgproc.ellipse(img, points[idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
}
}
MatOfDouble timings = new MatOfDouble();
long t = net.getPerfProfile(timings);
Debug.Log("t: " + t);
Debug.Log("timings.dump(): " + timings.dump());
double freq = Core.getTickFrequency() / 1000;
Debug.Log("freq: " + freq);
Imgproc.putText(img, (t / freq) + "ms", new Point(10, img.height() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar(0, 0, 255), 2);
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
