// Use this for initialization
void Start()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(Utils.getFilePath("dnn/004545.jpg"));
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/004545.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
#endif
Size inVideoSize = new Size(img.width(), img.height());
Size cropSize;
if (inVideoSize.width / (float)inVideoSize.height > WHRatio)
{
cropSize = new Size(inVideoSize.height * WHRatio, inVideoSize.height);
}
else
{
cropSize = new Size(inVideoSize.width, inVideoSize.width / WHRatio);
}
OpenCVForUnity.Rect crop = new OpenCVForUnity.Rect(new Point((inVideoSize.width - cropSize.width) / 2, (inVideoSize.height - cropSize.height) / 2), cropSize);
Net net = null;
string model_filepath = Utils.getFilePath("dnn/MobileNetSSD_deploy.caffemodel");
string prototxt_filepath = Utils.getFilePath("dnn/MobileNetSSD_deploy.prototxt");
#if !UNITY_WSA_10_0
if (string.IsNullOrEmpty(model_filepath) || string.IsNullOrEmpty(prototxt_filepath))
{
Debug.LogError("model file is not loaded.The model and prototxt file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD\".Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(prototxt_filepath, model_filepath);
}
#endif
if (net == null)
{
img = new Mat(img, crop);
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 50), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "The model and prototxt file can be downloaded here:", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "https://github.com/chuanqi305/MobileNet-SSD.", new Point(5, img.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Mat blob = Dnn.blobFromImage(img, inScaleFactor, new Size(inWidth, inHeight), new Scalar(meanVal), false);
net.setInput(blob);
TickMeter tm = new TickMeter();
tm.start();
Mat prob = net.forward();
prob = prob.reshape(1, (int)prob.total() / 7);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
img = new Mat(img, crop);
float[] data = new float[7];
float confidenceThreshold = 0.2f;
for (int i = 0; i < prob.rows(); i++)
{
prob.get(i, 0, data);
float confidence = data [2];
if (confidence > confidenceThreshold)
{
int class_id = (int)(data [1]);
float xLeftBottom = data [3] * img.cols();
float yLeftBottom = data [4] * img.rows();
float xRightTop = data [5] * img.cols();
float yRightTop = data [6] * img.rows();
Debug.Log("class_id: " + class_id);
Debug.Log("Confidence: " + confidence);
Debug.Log(" " + xLeftBottom
+ " " + yLeftBottom
+ " " + xRightTop
+ " " + yRightTop);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop),
new Scalar(0, 255, 0));
string label = classNames [class_id] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom - labelSize.height),
new Point(xLeftBottom + labelSize.width, yLeftBottom + baseLine [0]),
new Scalar(255, 255, 255), Core.FILLED);
Imgproc.putText(img, label, new Point(xLeftBottom, yLeftBottom),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
prob.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(COCO_val2014_000000000589_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/COCO_val2014_000000000589.jpg is not loaded.The image file can be downloaded here: \"https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/examples/media/COCO_val2014_000000000589.jpg\" folder. ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(pose_iter_160000_caffemodel_filepath) || string.IsNullOrEmpty(openpose_pose_mpi_faster_4_stages_prototxt_filepath))
{
Debug.LogError("model file is not loaded. The model and prototxt file can be downloaded here: \"http://posefs1.perception.cs.cmu.edu/OpenPose/models/pose/mpi/pose_iter_160000.caffemodel\",\"https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/openpose_pose_mpi_faster_4_stages.prototxt\". Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(openpose_pose_mpi_faster_4_stages_prototxt_filepath, pose_iter_160000_caffemodel_filepath);
//Intel's Deep Learning Inference Engine backend is supported on Windows 64bit platform only. Please refer to ReadMe.pdf for the setup procedure.
//net.setPreferableBackend (Dnn.DNN_BACKEND_INFERENCE_ENGINE);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat input = Dnn.blobFromImage(img, 1.0 / 255, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.setInput(input);
// TickMeter tm = new TickMeter ();
// tm.start ();
Mat output = net.forward();
// tm.stop ();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
output = output.reshape(1, 16);
float[] data = new float[46 * 46];
List <Point> points = new List <Point> ();
for (int i = 0; i < BODY_PARTS.Count; i++)
{
output.get(i, 0, data);
Mat heatMap = new Mat(1, data.Length, CvType.CV_32FC1);
heatMap.put(0, 0, data);
//Originally, we try to find all the local maximums. To simplify a sample
//we just find a global one. However only a single pose at the same time
//could be detected this way.
Core.MinMaxLocResult result = Core.minMaxLoc(heatMap);
heatMap.Dispose();
double x = (frameWidth * (result.maxLoc.x % 46)) / 46;
double y = (frameHeight * (result.maxLoc.x / 46)) / 46;
if (result.maxVal > 0.1)
{
points.Add(new Point(x, y));
}
else
{
points.Add(null);
}
}
for (int i = 0; i < POSE_PAIRS.GetLength(0); i++)
{
string partFrom = POSE_PAIRS [i, 0];
string partTo = POSE_PAIRS [i, 1];
int idFrom = BODY_PARTS [partFrom];
int idTo = BODY_PARTS [partTo];
if (points [idFrom] != null && points [idTo] != null)
{
Imgproc.line(img, points [idFrom], points [idTo], new Scalar(0, 255, 0), 3);
Imgproc.ellipse(img, points [idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
Imgproc.ellipse(img, points [idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
}
}
MatOfDouble timings = new MatOfDouble();
long t = net.getPerfProfile(timings);
Debug.Log("t: " + t);
Debug.Log("timings.dump(): " + timings.dump());
double freq = Core.getTickFrequency() / 1000;
Debug.Log("freq: " + freq);
Imgproc.putText(img, (t / freq) + "ms", new Point(10, img.height() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar(0, 0, 255), 2);
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(dnn004545_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/004545.jpg is not loaded.The image file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD/blob/master/images/004545.jpg\".Please copy to \"Assets/StreamingAssets/dnn/\" folder. ");
img = new Mat(375, 500, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(MobileNetSSD_deploy_caffemodel_filepath) || string.IsNullOrEmpty(MobileNetSSD_deploy_prototxt_filepath))
{
Debug.LogError("model file is not loaded.The model and prototxt file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD\".Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(MobileNetSSD_deploy_prototxt_filepath, MobileNetSSD_deploy_caffemodel_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Mat blob = Dnn.blobFromImage(img, inScaleFactor, new Size(inWidth, inHeight), new Scalar(meanVal, meanVal, meanVal), false, false);
net.setInput(blob);
TickMeter tm = new TickMeter();
tm.start();
Mat prob = net.forward();
prob = prob.reshape(1, (int)prob.total() / 7);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
float[] data = new float[7];
float confidenceThreshold = 0.2f;
for (int i = 0; i < prob.rows(); i++)
{
prob.get(i, 0, data);
float confidence = data [2];
if (confidence > confidenceThreshold)
{
int class_id = (int)(data [1]);
float left = data [3] * img.cols();
float top = data [4] * img.rows();
float right = data [5] * img.cols();
float bottom = data [6] * img.rows();
Debug.Log("class_id: " + class_id);
Debug.Log("Confidence: " + confidence);
Debug.Log(" " + left
+ " " + top
+ " " + right
+ " " + bottom);
Imgproc.rectangle(img, new Point(left, top), new Point(right, bottom),
new Scalar(0, 255, 0), 2);
string label = classNames [class_id] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
top = Mathf.Max(top, (float)labelSize.height);
Imgproc.rectangle(img, new Point(left, top),
new Point(left + labelSize.width, top + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255), Core.FILLED);
Imgproc.putText(img, label, new Point(left, top + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
prob.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image_filepath, Imgcodecs.IMREAD_COLOR);
Mat colorized = new Mat(img.rows(), img.cols(), img.type());
if (img.empty())
{
Debug.LogError(image_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
Net net = null;
if (string.IsNullOrEmpty(caffemodel_filepath) || string.IsNullOrEmpty(prototxt_filepath))
{
Debug.LogError(caffemodel_filepath + " or " + prototxt_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
net = Dnn.readNetFromCaffe(prototxt_filepath, caffemodel_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
// setup additional layers:
int[] sz = new int[] { 2, 313, 1, 1 };
Mat pts_in_hull = new Mat(sz, CvType.CV_32F);
pts_in_hull.put(new int[] { 0, 0, 0, 0 }, hull_pts);
Layer class8_ab = net.getLayer(new DictValue("class8_ab"));
List <Mat> blobs = class8_ab.get_blobs();
blobs.Add(pts_in_hull);
class8_ab.set_blobs(blobs);
Layer conv8_313_rh = net.getLayer(new DictValue("conv8_313_rh"));
blobs = conv8_313_rh.get_blobs();
blobs.Add(new Mat(1, 313, CvType.CV_32F, new Scalar(2.606)));
conv8_313_rh.set_blobs(blobs);
// extract L channel and subtract mean
Mat img_32F = new Mat();
Mat lab = new Mat();
Mat L = new Mat();
Mat input = new Mat();
img.convertTo(img_32F, CvType.CV_32F, 1.0 / 255);
Imgproc.cvtColor(img_32F, lab, Imgproc.COLOR_BGR2Lab);
Core.extractChannel(lab, L, 0);
Imgproc.resize(L, input, new Size(inWidth, inHeight));
Core.subtract(input, new Scalar(50.0), input);
// run the L channel through the network
Mat inputBlob = Dnn.blobFromImage(input);
net.setInput(inputBlob);
Mat result = net.forward();
// retrieve the calculated a,b channels from the network output
Mat result_a = new Mat(result, new Range[] { new Range(0, 1), new Range(0, 1), new Range(0, result.size(2)), new Range(0, result.size(3)) });
Mat result_b = new Mat(result, new Range[] { new Range(0, 1), new Range(1, 2), new Range(0, result.size(2)), new Range(0, result.size(3)) });
result_a = result_a.reshape(1, result.size(2));
result_b = result_b.reshape(1, result.size(2));
Mat a = new Mat(img.size(), CvType.CV_32F);
Mat b = new Mat(img.size(), CvType.CV_32F);
Imgproc.resize(result_a, a, img.size());
Imgproc.resize(result_b, b, img.size());
// merge, and convert back to BGR
List <Mat> chn = new List <Mat>();
chn.Add(L); chn.Add(a); chn.Add(b);
Core.merge(chn, lab);
Imgproc.cvtColor(lab, img_32F, Imgproc.COLOR_Lab2BGR);
img_32F.convertTo(colorized, CvType.CV_8U, 255.0);
MatOfDouble timings = new MatOfDouble();
long t = net.getPerfProfile(timings);
double freq = Core.getTickFrequency() / 1000;
Debug.Log("Inference time " + (t / freq) + "ms");
Imgproc.putText(colorized, (t / freq) + "ms", new Point(10, img.height() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2);
Imgproc.putText(img, "gray", new Point(10, 20), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2);
Imgproc.putText(colorized, "colorized", new Point(10, 20), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2);
}
Imgproc.cvtColor(colorized, colorized, Imgproc.COLOR_BGR2RGB);
Mat display = new Mat(img.rows() * 2, img.cols(), img.type());
using (Mat gray = new Mat(img.rows(), img.cols(), CvType.CV_8UC1))
using (Mat displayUpperHalf = new Mat(display, new Range(0, img.rows())))
using (Mat displayLowerHalf = new Mat(display, new Range(img.rows(), display.rows())))
{
Imgproc.cvtColor(img, gray, Imgproc.COLOR_BGR2GRAY);
Imgproc.cvtColor(gray, img, Imgproc.COLOR_GRAY2RGB);
img.copyTo(displayUpperHalf);
colorized.copyTo(displayLowerHalf);
}
Texture2D texture = new Texture2D(display.cols(), display.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(display, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(display.width(), display.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = display.width();
float imageHeight = display.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Utils.setDebugMode(false);
}