// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : bgrMat.cols(),
inpHeight > 0 ? inpHeight : bgrMat.rows());
Mat blob = Dnn.blobFromImage(bgrMat, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(bgrMat, bgrMat, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
//Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
postprocess(rgbaMat, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
}
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
protected virtual void Process(Mat img)
{
Mat inputBlob = PreProcess(img);
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outputBlobs = Predict(inputBlob);
tm.stop();
PostProcess(img, outputBlobs);
for (int i = 0; i < outputBlobs.Count; i++)
{
outputBlobs[i].Dispose();
}
inputBlob.Dispose();
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
if (!string.IsNullOrEmpty(classes))
{
classNames = readClassNames(classes_filepath);
if (classNames == null)
{
Debug.LogError(classes_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
}
else if (classesList.Count > 0)
{
classNames = classesList;
}
Mat img = Imgcodecs.imread(input_filepath);
if (img.empty())
{
Debug.LogError(input_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(config_filepath) || string.IsNullOrEmpty(model_filepath))
{
Debug.LogError(config_filepath + " or " + model_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
//! [Initialize network]
net = Dnn.readNet(model_filepath, config_filepath);
//! [Initialize network]
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
outBlobNames = getOutputsNames(net);
//for (int i = 0; i < outBlobNames.Count; i++)
//{
// Debug.Log("names [" + i + "] " + outBlobNames[i]);
//}
outBlobTypes = getOutputsTypes(net);
//for (int i = 0; i < outBlobTypes.Count; i++)
//{
// Debug.Log("types [" + i + "] " + outBlobTypes[i]);
//}
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : img.cols(),
inpHeight > 0 ? inpHeight : img.rows());
Mat blob = Dnn.blobFromImage(img, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(img, img, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
postprocess(img, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
net.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(dnn004545_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/004545.jpg is not loaded.The image file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD/blob/master/images/004545.jpg\".Please copy to \"Assets/StreamingAssets/dnn/\" folder. ");
img = new Mat(375, 500, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(MobileNetSSD_deploy_caffemodel_filepath) || string.IsNullOrEmpty(MobileNetSSD_deploy_prototxt_filepath))
{
Debug.LogError("model file is not loaded.The model and prototxt file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD\".Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(MobileNetSSD_deploy_prototxt_filepath, MobileNetSSD_deploy_caffemodel_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Mat blob = Dnn.blobFromImage(img, inScaleFactor, new Size(inWidth, inHeight), new Scalar(meanVal, meanVal, meanVal), false, false);
net.setInput(blob);
TickMeter tm = new TickMeter();
tm.start();
Mat prob = net.forward();
prob = prob.reshape(1, (int)prob.total() / 7);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
float[] data = new float[7];
float confidenceThreshold = 0.2f;
for (int i = 0; i < prob.rows(); i++)
{
prob.get(i, 0, data);
float confidence = data [2];
if (confidence > confidenceThreshold)
{
int class_id = (int)(data [1]);
float left = data [3] * img.cols();
float top = data [4] * img.rows();
float right = data [5] * img.cols();
float bottom = data [6] * img.rows();
Debug.Log("class_id: " + class_id);
Debug.Log("Confidence: " + confidence);
Debug.Log(" " + left
+ " " + top
+ " " + right
+ " " + bottom);
Imgproc.rectangle(img, new Point(left, top), new Point(right, bottom),
new Scalar(0, 255, 0), 2);
string label = classNames [class_id] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
top = Mathf.Max(top, (float)labelSize.height);
Imgproc.rectangle(img, new Point(left, top),
new Point(left + labelSize.width, top + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255), Core.FILLED);
Imgproc.putText(img, label, new Point(left, top + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
prob.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
List <string> classNames = readClassNames(coco_names_filepath);
#if !UNITY_WSA_10_0
if (classNames == null)
{
Debug.LogError("class names list file is not loaded.The model and class names list can be downloaded here: \"https://github.com/pjreddie/darknet/tree/master/data/coco.names\".Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
#endif
Mat img = Imgcodecs.imread(person_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/person.jpg is not loaded.The image file can be downloaded here: \"https://github.com/pjreddie/darknet/blob/master/data/person.jpg\".Please copy to \"Assets/StreamingAssets/dnn/\" folder. ");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(tiny_yolo_cfg_filepath) || string.IsNullOrEmpty(tiny_yolo_weights_filepath))
{
Debug.LogError("model file is not loaded. the cfg-file and weights-file can be downloaded here: https://github.com/pjreddie/darknet/blob/master/cfg/tiny-yolo.cfg and https://pjreddie.com/media/files/tiny-yolo.weights. Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
//! [Initialize network]
net = Dnn.readNetFromDarknet(tiny_yolo_cfg_filepath, tiny_yolo_weights_filepath);
//! [Initialize network]
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
//! [Resizing without keeping aspect ratio]
Mat resized = new Mat();
Imgproc.resize(img, resized, new Size(network_width, network_height));
//! [Resizing without keeping aspect ratio]
//! [Prepare blob]
Mat inputBlob = Dnn.blobFromImage(resized, 1 / 255.0, new Size(), new Scalar(0), true, true); //Convert Mat to batch of images
//! [Prepare blob]
//! [Set input blob]
net.setInput(inputBlob, "data"); //set the network input
//! [Set input blob]
TickMeter tm = new TickMeter();
tm.start();
//! [Make forward pass]
Mat detectionMat = net.forward("detection_out"); //compute output
//! [Make forward pass]
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
Debug.Log("detectionMat.ToString(): " + detectionMat.ToString());
float[] position = new float[5];
float[] confidences = new float[80];
float confidenceThreshold = 0.24f;
for (int i = 0; i < detectionMat.rows(); i++)
{
detectionMat.get(i, 0, position);
detectionMat.get(i, 5, confidences);
int maxIdx = confidences.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I;
float confidence = confidences [maxIdx];
if (confidence > confidenceThreshold)
{
float x = position [0];
float y = position [1];
float width = position [2];
float height = position [3];
int xLeftBottom = (int)((x - width / 2) * img.cols());
int yLeftBottom = (int)((y - height / 2) * img.rows());
int xRightTop = (int)((x + width / 2) * img.cols());
int yRightTop = (int)((y + height / 2) * img.rows());
Debug.Log("confidence: " + confidence);
Debug.Log(" " + xLeftBottom
+ " " + yLeftBottom
+ " " + xRightTop
+ " " + yRightTop);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop),
new Scalar(0, 255, 0), 2);
if (maxIdx < classNames.Count)
{
string label = classNames [maxIdx] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom),
new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255), Core.FILLED);
Imgproc.putText(img, label, new Point(xLeftBottom, yLeftBottom + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
}
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Start()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(Utils.getFilePath("dnn/004545.jpg"));
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/004545.jpg is not loaded.The image file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD/blob/master/images/004545.jpg\".Please copy to \"Assets/StreamingAssets/dnn/\" folder. ");
img = new Mat(375, 500, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
Size inVideoSize = new Size(img.width(), img.height());
Size cropSize;
if (inVideoSize.width / (float)inVideoSize.height > WHRatio)
{
cropSize = new Size(inVideoSize.height * WHRatio, inVideoSize.height);
}
else
{
cropSize = new Size(inVideoSize.width, inVideoSize.width / WHRatio);
}
OpenCVForUnity.Rect crop = new OpenCVForUnity.Rect(new Point((inVideoSize.width - cropSize.width) / 2, (inVideoSize.height - cropSize.height) / 2), cropSize);
Net net = null;
string model_filepath = Utils.getFilePath("dnn/MobileNetSSD_deploy.caffemodel");
string prototxt_filepath = Utils.getFilePath("dnn/MobileNetSSD_deploy.prototxt");
if (string.IsNullOrEmpty(model_filepath) || string.IsNullOrEmpty(prototxt_filepath))
{
Debug.LogError("model file is not loaded.The model and prototxt file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD\".Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(prototxt_filepath, model_filepath);
}
if (net == null)
{
img = new Mat(img, crop);
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Mat blob = Dnn.blobFromImage(img, inScaleFactor, new Size(inWidth, inHeight), new Scalar(meanVal), false, true);
net.setInput(blob);
TickMeter tm = new TickMeter();
tm.start();
Mat prob = net.forward();
prob = prob.reshape(1, (int)prob.total() / 7);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
img = new Mat(img, crop);
float[] data = new float[7];
float confidenceThreshold = 0.2f;
for (int i = 0; i < prob.rows(); i++)
{
prob.get(i, 0, data);
float confidence = data [2];
if (confidence > confidenceThreshold)
{
int class_id = (int)(data [1]);
float xLeftBottom = data [3] * img.cols();
float yLeftBottom = data [4] * img.rows();
float xRightTop = data [5] * img.cols();
float yRightTop = data [6] * img.rows();
Debug.Log("class_id: " + class_id);
Debug.Log("Confidence: " + confidence);
Debug.Log(" " + xLeftBottom
+ " " + yLeftBottom
+ " " + xRightTop
+ " " + yRightTop);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop),
new Scalar(0, 255, 0), 2);
string label = classNames [class_id] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom),
new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255), Core.FILLED);
Imgproc.putText(img, label, new Point(xLeftBottom, yLeftBottom + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
prob.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
classNames = readClassNames(classes_filepath);
if (classNames == null)
{
Debug.LogError(classes_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
classColors = new List <Scalar>();
for (int i = 0; i < classNames.Count; i++)
{
classColors.Add(new Scalar(UnityEngine.Random.Range(0, 255), UnityEngine.Random.Range(0, 255), UnityEngine.Random.Range(0, 255)));
}
Mat img = Imgcodecs.imread(image_filepath);
if (img.empty())
{
Debug.LogError(image_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(height, width, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(model_filepath) || string.IsNullOrEmpty(config_filepath))
{
Debug.LogError(model_filepath + " or " + config_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
net = Dnn.readNetFromTensorflow(model_filepath, config_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameW = img.cols();
float frameH = img.rows();
// Create a 4D blob from a frame.
Mat blob = Dnn.blobFromImage(img, 1.0, new Size(width, height), new Scalar(0, 0, 0), true, false);
//Run a model
net.setInput(blob);
List <Mat> outputBlobs = new List <Mat>();
List <string> outputName = new List <string>();
outputName.Add("detection_out_final");
outputName.Add("detection_masks");
TickMeter tm = new TickMeter();
tm.start();
net.forward(outputBlobs, outputName);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
Mat boxes = outputBlobs[0];
Mat masks = outputBlobs[1];
//int numClasses = masks.size(1);
int numDetections = boxes.size(2);
int mask_sizeH = masks.size(2);
int mask_sizeW = masks.size(3);
float[] box_data = new float[boxes.size(3)];
float[] mask_data = new float[masks.size(2) * masks.size(3)];
for (int i = 0; i < numDetections; i++)
{
boxes.get(new int[] { 0, 0, i, 0 }, box_data);
float score = box_data[2];
if (score > thr)
{
int classId = (int)box_data[1];
float left = (int)frameW * box_data[3];
float top = (int)frameH * box_data[4];
float right = (int)frameW * box_data[5];
float bottom = (int)frameH * box_data[6];
left = (int)Mathf.Max(0, Mathf.Min(left, frameW - 1));
top = (int)Mathf.Max(0, Mathf.Min(top, frameH - 1));
right = (int)Mathf.Max(0, Mathf.Min(right, frameW - 1));
bottom = (int)Mathf.Max(0, Mathf.Min(bottom, frameH - 1));
masks.get(new int[] { i, classId, 0, 0 }, mask_data);
Mat classMask = new Mat(mask_sizeH, mask_sizeW, CvType.CV_32F);
classMask.put(0, 0, mask_data);
Imgproc.resize(classMask, classMask, new Size(right - left + 1, bottom - top + 1));
Core.compare(classMask, new Scalar(0.5), classMask, Core.CMP_GT);
Mat roi = new Mat(img, new OpenCVForUnity.CoreModule.Rect(new Point(left, top), new Point(right + 1, bottom + 1)));
Mat coloredRoi = new Mat(roi.size(), CvType.CV_8UC3);
Imgproc.rectangle(coloredRoi, new Point(0, 0), new Point(coloredRoi.width(), coloredRoi.height()), classColors[classId], -1);
Core.addWeighted(coloredRoi, 0.7, roi, 0.3, 0, coloredRoi);
coloredRoi.copyTo(roi, classMask);
coloredRoi.Dispose();
classMask.Dispose();
drawPred(classId, score, left, top, right, bottom, img);
Debug.Log("classId:" + classId + " cnof:" + score + " l:" + left + " t:" + top + " r:" + right + " b:" + bottom);
}
}
boxes.Dispose();
masks.Dispose();
blob.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
net.Dispose();
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image_filepath, Imgcodecs.IMREAD_COLOR);
if (img.empty())
{
Debug.LogError(image_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net detector = null;
Net recognizer = null;
if (string.IsNullOrEmpty(detectionmodel_filepath) || string.IsNullOrEmpty(recognitionmodel_filepath))
{
Debug.LogError(detectionmodel_filepath + " or " + recognitionmodel_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
detector = Dnn.readNet(detectionmodel_filepath);
recognizer = Dnn.readNet(recognitionmodel_filepath);
}
if (detector == null || recognizer == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
TickMeter tickMeter = new TickMeter();
List <Mat> outs = new List <Mat>();
List <string> outNames = new List <string>();
outNames.Add("feature_fusion/Conv_7/Sigmoid");
outNames.Add("feature_fusion/concat_3");
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : img.cols(), inpHeight > 0 ? inpHeight : img.rows());
Mat blob = Dnn.blobFromImage(img, 1.0, inpSize, new Scalar(123.68, 116.78, 103.94), true, false); // blobFromImage(frame, blob, 1.0, Size(inpWidth, inpHeight), Scalar(123.68, 116.78, 103.94), true, false);
// Run detection model.
detector.setInput(blob);
tickMeter.start();
detector.forward(outs, outNames);
tickMeter.stop();
Mat scores = outs[0];
Mat geometry = outs[1];
// Decode predicted bounding boxes.
List <RotatedRect> boxes = new List <RotatedRect>();
List <float> confidences = new List <float>();
decodeBoundingBoxes(scores, geometry, confThreshold, boxes, confidences);
// Apply non-maximum suppression procedure.
MatOfRotatedRect boxesMat = new MatOfRotatedRect(boxes.ToArray());
MatOfFloat confidencesMat = new MatOfFloat(confidences.ToArray());
MatOfInt indicesMat = new MatOfInt();
Dnn.NMSBoxesRotated(boxesMat, confidencesMat, confThreshold, nmsThreshold, indicesMat);
List <int> indices = indicesMat.toList();
Point ratio = new Point(img.cols() / inpWidth, img.rows() / inpHeight);
// Render text.
for (int i = 0; i < indices.Count; ++i)
{
RotatedRect box = boxes[indices[i]];
Point[] vertices = new Point[4];
box.points(vertices);
for (int j = 0; j < 4; ++j)
{
vertices[j].x *= ratio.x;
vertices[j].y *= ratio.y;
}
for (int j = 0; j < 4; ++j)
{
Imgproc.line(img, vertices[j], vertices[(j + 1) % 4], new Scalar(0, 255, 0), 1);
}
if (recognizer != null)
{
Mat cropped = new Mat();
fourPointsTransform(img, vertices, cropped);
//Debug.Log(cropped);
Imgproc.cvtColor(cropped, cropped, Imgproc.COLOR_BGR2GRAY);
Mat blobCrop = Dnn.blobFromImage(cropped, 1.0 / 127.5, new Size(), Scalar.all(127.5));
recognizer.setInput(blobCrop);
//Debug.Log(blobCrop);
tickMeter.start();
Mat result = recognizer.forward();
tickMeter.stop();
string wordRecognized;
decodeText(result, out wordRecognized);
Imgproc.putText(img, wordRecognized, vertices[1], Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0), 1, Imgproc.LINE_AA, false);
Debug.Log(wordRecognized);
cropped.Dispose();
blobCrop.Dispose();
result.Dispose();
}
}
Debug.Log("Inference time, ms: " + tickMeter.getTimeMilli());
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
detector.Dispose();
recognizer.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run(Mat img)
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
if (!string.IsNullOrEmpty(classes))
{
classNames = readClassNames(classes_filepath);
#if !UNITY_WSA_10_0
if (classNames == null)
{
Debug.LogError(classes_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
#endif
}
else if (classesList.Count > 0)
{
classNames = classesList;
}
Net net = null;
if (string.IsNullOrEmpty(config_filepath) || string.IsNullOrEmpty(model_filepath))
{
Debug.LogError(config_filepath + " or " + model_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
//! [Initialize network]
net = Dnn.readNet(model_filepath, config_filepath);
//! [Initialize network]
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
outBlobNames = getOutputsNames(net);
// for (int i = 0; i < outBlobNames.Count; i++) {
// Debug.Log ("names [" + i + "] " + outBlobNames [i]);
// }
outBlobTypes = getOutputsTypes(net);
// for (int i = 0; i < outBlobTypes.Count; i++) {
// Debug.Log ("types [" + i + "] " + outBlobTypes [i]);
// }
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : img.cols(),
inpHeight > 0 ? inpHeight : img.rows());
Mat blob = Dnn.blobFromImage(img, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(img, img, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
postprocess(img, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
net.Dispose();
}
Utils.setDebugMode(false);
}
/// <summary>
/// Process
/// </summary>
/// <returns></returns>
private async void Process()
{
float DOWNSCALE_RATIO = 1.0f;
while (true)
{
// Check TaskCancel
if (tokenSource.Token.IsCancellationRequested)
{
break;
}
rgbaMat = webCamTextureToMatHelper.GetMat();
// Debug.Log ("rgbaMat.ToString() " + rgbaMat.ToString ());
Mat downScaleRgbaMat = null;
DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(rgbaMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = rgbaMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.cvtColor(downScaleRgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
await Task.Run(() =>
{
// detect faces on the downscale image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : bgrMat.cols(),
inpHeight > 0 ? inpHeight : bgrMat.rows());
Mat blob = Dnn.blobFromImage(bgrMat, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(bgrMat, bgrMat, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
postprocess(bgrMat, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
if (enableDownScale)
{
for (int i = 0; i < _boxesList.Count; ++i)
{
var rect = _boxesList[i];
_boxesList[i] = new OpenCVForUnity.CoreModule.Rect(
(int)(rect.x * DOWNSCALE_RATIO),
(int)(rect.y * DOWNSCALE_RATIO),
(int)(rect.width * DOWNSCALE_RATIO),
(int)(rect.height * DOWNSCALE_RATIO));
}
}
}
//Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
MatOfRect boxes = new MatOfRect();
boxes.fromList(_boxesList);
MatOfFloat confidences = new MatOfFloat();
confidences.fromList(_confidencesList);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
// Debug.Log ("indices.dump () "+indices.dump ());
// Debug.Log ("indices.ToString () "+indices.ToString());
for (int i = 0; i < indices.total(); ++i)
{
int idx = (int)indices.get(i, 0)[0];
OpenCVForUnity.CoreModule.Rect box = _boxesList[idx];
drawPred(_classIdsList[idx], _confidencesList[idx], box.x, box.y,
box.x + box.width, box.y + box.height, rgbaMat);
}
indices.Dispose();
boxes.Dispose();
confidences.Dispose();
}
});
Utils.fastMatToTexture2D(rgbaMat, texture);
Thread.Sleep(10);
}
}
public void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
classNames = readClassNames(classes_filepath);
if (classNames == null)
{
Debug.LogError(classes_filepath +
" is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
//Mat img = new Mat(imageTex.height, imageTex.width, CvType.);
//Utils.texture2DToMat(imageTex, img);
Mat img = Imgcodecs.imread(input_filepath);
if (img.empty())
{
Debug.LogError(input_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(model_filepath) || string.IsNullOrEmpty(config_filepath))
{
Debug.LogError(model_filepath + " or " + config_filepath +
" is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
net = Dnn.readNetFromTensorflow(model_filepath, config_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30),
Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10),
Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
outBlobNames = getOutputsNames(net);
outBlobTypes = getOutputsTypes(net);
Mat blob = Dnn.blobFromImage(img, 0.007843, new Size(300, 300), new Scalar(127.5, 127.5, 127.5));
net.setInput(blob);
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
postprocess(img, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
net.Dispose();
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
}
void ObjectDetection()
{
// If true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image);
if (img.empty())
{
Debug.LogError("Image " + image + " is not loaded.");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
Net net = null;
if (string.IsNullOrEmpty(cfg) || string.IsNullOrEmpty(weight))
{
Debug.LogError(cfg + " or " + weight + " is not loaded.");
}
else
{
//load model and config
net = Dnn.readNet(weight, cfg);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
//setting blob, size can be:320/416/608
//opencv blob setting can check here https://github.com/opencv/opencv/tree/master/samples/dnn#object-detection
Mat blob = Dnn.blobFromImage(img, 1.0 / 255, new Size(416, 416), new Scalar(0), false, false);
//input data
net.setInput(blob);
//get output layer name
List <string> outNames = net.getUnconnectedOutLayersNames();
//create mats for output layer
List <Mat> outs = outNames.Select(_ => new Mat()).ToList();
#region forward model
TickMeter tm = new TickMeter();
tm.start();
net.forward(outs, outNames);
tm.stop();
Debug.Log("Runtime: " + tm.getTimeMilli() + " ms");
#endregion
//get result from all output
GetResult(outs, img, threshold, nmsThreshold);
}
// Show Image
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Update is called once per frame
void Update()
{
for (int i = 0; i < validTimer.Length; i++)
{
validTimer[i] -= Time.deltaTime;
if (validTimer[i] <= 0.0f)
{
timerEnded(i);
}
}
switch (ClassID)
{
case 20: //LCD
GUIObject[3].transform.position = Vector3.MoveTowards(GUIObject[3].transform.position, sphere[3].transform.position, Time.deltaTime * speed);
break;
case 12: //Dog
GUIObject[3].transform.position = Vector3.MoveTowards(GUIObject[3].transform.position, sphere[3].transform.position, Time.deltaTime * speed);
break;
case 9: // Chair
GUIObject[2].transform.position = Vector3.MoveTowards(GUIObject[2].transform.position, sphere[2].transform.position, Time.deltaTime * speed);
break;
case 5: // Bottle
GUIObject[1].transform.position = Vector3.MoveTowards(GUIObject[1].transform.position, sphere[1].transform.position, Time.deltaTime * speed);
break;
case 15: // Civilian
GUIObject[0].transform.position = Vector3.MoveTowards(GUIObject[0].transform.position, sphere[0].transform.position, Time.deltaTime * speed);
break;
default:
print("Unknown");
break;
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : bgrMat.cols(),
inpHeight > 0 ? inpHeight : bgrMat.rows());
Mat blob = Dnn.blobFromImage(bgrMat, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1) // Faster-RCNN or R-FCN
{
Imgproc.resize(bgrMat, bgrMat, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat> ();
net.forward(outs, outBlobNames);
tm.stop();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
postprocess(rgbaMat, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs [i].Dispose();
}
blob.Dispose();
}
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}