private IEnumerator CalibrateCameraUsingImages()
{
string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory);
string[] imageFiles = GetImageFilesInDirectory(dirPath);
if (imageFiles.Length < 1)
{
yield break;
}
isCalibrating = true;
markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = false;
Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar);
foreach (var path in imageFiles)
{
Uri fullPath = new Uri(path);
string relativePath = rootPath.MakeRelativeUri(fullPath).ToString();
using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) {
if (gray.width() != bgrMat.width() || gray.height() != bgrMat.height())
{
continue;
}
Mat frameMat = gray.clone();
double e = CaptureFrame(frameMat);
if (e > 0)
{
repErr = e;
}
DrawFrame(gray, bgrMat);
Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);
Utils.matToTexture2D(rgbaMat, texture);
}
yield return(new WaitForSeconds(0.5f));
}
isCalibrating = false;
markerTypeDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = true;
}
Mat str2mat(String s)
{
// Decode data
//Debug.Log(s);
byte[] byteArr = System.Convert.FromBase64String(s);
List <byte> byteList = new List <Byte>(byteArr);
// Decode data
//string decoded_string = base64_decode(s);
//List<byte> byteList = new List<Byte>(Encoding.ASCII.GetBytes(decoded_string));
Mat data = Converters.vector_char_to_Mat(byteList);
//Debug.Log("Converted to vector mat");
//System.Collections.Generic.List<char> data(decoded_string.begin(), decoded_string.end());
//List<uchar> data(decoded_string.begin(), decoded_string.end());
//Debug.Log("Converted to image");
Mat img = Imgcodecs.imdecode(data, Imgcodecs.IMREAD_UNCHANGED);
return(img);
}
private void Update()
{
if (!ct.begin_flag)
{
if (!ct.rec)
{
ct.Receive();
}
}
if (ct.receiveMessage != null)
{
Debug.Log("Server:" + ct.receiveMessage);
ct.receiveMessage = null;
}
/*if (ct.receive_joint != null)
* {
* Debug.Log("Server:" + ct.receive_joint);
* ct.receive_joint = null;
* }*/
if (ct.recieve_flag != 0)
{
//Debug.Log("Server:" + ct.frame_size.ToString());
if (ct.frame_img.Length != ct.dataLength_frame)
{
Debug.Log("Receive : " + ct.frame_img.Length.ToString() + ", len : " + ct.dataLength.ToString());
}
Mat mat_img = new Mat(1, ct.frame_img.Length, CvType.CV_8U);
mat_img.put(0, 0, ct.frame_img);
Mat frame_img_mat = Imgcodecs.imdecode(mat_img, 1);
Imgproc.cvtColor(frame_img_mat, frame_img_mat, Imgproc.COLOR_BGR2RGB);
//Debug.Log(frame_img_mat.size());
byte[] image = new byte[252 * 448 * 3];
frame_img_mat.get(0, 0, image);
var texture = rawImage.texture as Texture2D;
texture.LoadRawTextureData(image); //TODO: Should be able to do this: texture.LoadRawTextureData(pointerToImage, 1280 * 720 * 4);
texture.Apply();
ct.recieve_flag = 0;
ct.rec = false;
}
}
void Start()
{
src1 = Imgcodecs.imread(Application.dataPath + "/Textures/p1.jpg", 1);
src2 = Imgcodecs.imread(Application.dataPath + "/Textures/p2.jpg", 1);
Imgproc.cvtColor(src1, src1, Imgproc.COLOR_BGR2RGB);
Imgproc.cvtColor(src2, src2, Imgproc.COLOR_BGR2RGB);
images.Add(src1);
images.Add(src2);
//全景图
Mat pano = new Mat();
//缺少stitcher.hpp
Texture2D t2d = new Texture2D(dstMat.width(), dstMat.height());
Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero);
m_showImage.sprite = sp;
m_showImage.preserveAspect = true;
Utils.matToTexture2D(dstMat, t2d);
}
void Start()
{
srcMat = Imgcodecs.imread(Application.dataPath + "/Textures/sample.jpg");
Imgproc.cvtColor(srcMat, srcMat, Imgproc.COLOR_BGR2RGB);
//基础色度图
for (int i = 0; i < 13; i++)
{
Mat dstMat = new Mat();
//Imgproc.applyColorMap(srcMat, dstMat, Imgproc.COLORMAP_JET);
Imgproc.applyColorMap(srcMat, dstMat, i);
Texture2D t2d = new Texture2D(dstMat.width(), dstMat.height());
Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero);
m_imageList[i].sprite = sp;
m_imageList[i].preserveAspect = true;
Utils.matToTexture2D(dstMat, t2d);
}
}
public void takePhoto()
{
snapshotCount = 40;
Debug.Log("TAKE PHOTO");
Texture2D tex = new Texture2D(Screen.width, Screen.height, TextureFormat.RGBA32, false, true);
drawFlag = true;
InvokeRepeating("pauseForPhoto", 0.001f, 0.001f);
Imgproc.resize(rgbMat, textureInstance, new Size(Screen.width, Screen.height));
Debug.Log("texture is" + textureInstance.width() + ", " + textureInstance.height());
Debug.Log("tex is" + tex.width + ", " + tex.height);
Utils.fastMatToTexture2D(textureInstance, tex);
//write to singleton
ImageManager.instance.photo = tex;
//write image
Imgcodecs.imwrite("Assets/snapshot.jpeg", textureInstance);
}
/// <summary>
/// Raises the save button event.
/// </summary>
public void OnSaveButton()
{
if (patternRawImage.texture != null)
{
Texture2D patternTexture = (Texture2D)patternRawImage.texture;
Mat patternMat = new Mat(patternRect.size(), CvType.CV_8UC3);
Utils.texture2DToMat(patternTexture, patternMat);
Imgproc.cvtColor(patternMat, patternMat, Imgproc.COLOR_RGB2BGR);
string savePath = Application.persistentDataPath;
Debug.Log("savePath " + savePath);
Imgcodecs.imwrite(savePath + "/patternImg.jpg", patternMat);
#if UNITY_5_3 || UNITY_5_3_OR_NEWER
SceneManager.LoadScene("WebCamTextureMarkerLessARExample");
#else
Application.LoadLevel("WebCamTextureMarkerLessARExample");
#endif
}
}
/// <summary>
/// Raises the webcam texture initialized event.
/// </summary>
private void OnCVInited()
{
if (colors == null || colors.Length != webCamTexture.width * webCamTexture.height)
{
colors = new Color32[webCamTexture.width * webCamTexture.height];
}
if (texture == null || texture.width != webCamTexture.width || texture.height != webCamTexture.height)
{
texture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
}
rgbaMat = new Mat(webCamTexture.height, webCamTexture.width, CvType.CV_8UC4);
// 出力先の設定するならココ. 参照: OpenCVForUnityExample.
outputScreenQuad.setupScreenQuadAndCamera(webCamTexture.height, webCamTexture.width, CvType.CV_8UC3);
Mat matOrg = Imgcodecs.imread(Utils.getFilePath("hand1.png"));
mat = new Mat(matOrg.size(), CvType.CV_8UC3);
Imgproc.cvtColor(matOrg, mat, Imgproc.COLOR_BGRA2RGB);
}
void Start()
{
string path = Application.dataPath + "/Textures/sample.jpg";
srcImage = Imgcodecs.imread(path, 1);
t2d = new Texture2D(srcImage.width(), srcImage.height());
Utils.matToTexture2D(srcImage, t2d);
undeal.texture = t2d;
//判断文件加载是否正确
//Debug.Log(File.Exists(path));
//split/merge 对颜色通道进行处理
//split 通道的拆分。用于将一幅多通道的图像的各个通道分离。
Core.split(srcImage, channels);
Debug.Log(channels.Count); //jpg/png都只拆出3个通道(YUV?)
//提取红色通道的数据
imageRed = new Mat();
imageRed = channels[0];
//提取绿色通道的数据
imageGreen = new Mat();
imageGreen = channels[1];
//提取蓝色通道的数据
imageBlue = new Mat();
imageBlue = channels[2];
channels[0] = channels[2];
channels[2] = imageRed;
//对拆分的通道数据合并
//merge 与split 相反。可以将多个单通道图像合成一幅多通道图像。
mergeImage = new Mat();
Core.merge(channels, mergeImage);
t2d = new Texture2D(mergeImage.width(), mergeImage.height());
output.texture = t2d;
Utils.matToTexture2D(mergeImage, t2d);
}
void Start()
{
srcMat = Imgcodecs.imread(Application.dataPath + "/Textures/feature.jpg", 1);
grayMat = new Mat();
Imgproc.cvtColor(srcMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
//会把五边形识别成圆。模糊处理，提高精确度。
Imgproc.GaussianBlur(grayMat, grayMat, new Size(7, 7), 2, 2);
Mat circles = new Mat();
//霍夫圆
Imgproc.HoughCircles(grayMat, circles, Imgproc.CV_HOUGH_GRADIENT, 2, 10, 160, 50, 10, 40);
//Debug.Log(circles);
//圆心坐标
Point pt = new Point();
for (int i = 0; i < circles.cols(); i++)
{
double[] data = circles.get(0, i);
pt.x = data[0];
pt.y = data[1];
double rho = data[2];
//绘制圆心
Imgproc.circle(srcMat, pt, 3, new Scalar(255, 255, 0), -1, 8, 0);
//绘制圆轮廓
Imgproc.circle(srcMat, pt, (int)rho, new Scalar(255, 0, 0, 255), 5);
}
//在Mat上写字
Imgproc.putText(srcMat, "W:" + srcMat.width() + " H:" + srcMat.height(), new Point(5, srcMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Texture2D t2d = new Texture2D(srcMat.width(), srcMat.height());
Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero);
m_showImage.sprite = sp;
m_showImage.preserveAspect = true;
Utils.matToTexture2D(srcMat, t2d);
}
void Start()
{
srcMat = Imgcodecs.imread(Application.dataPath + "/Textures/palace.jpg");
Imgproc.cvtColor(srcMat, srcMat, Imgproc.COLOR_BGR2RGB);
//dstMat = srcMat.clone();
//dstMat = removeBackground(srcMat);
//dstMat = MyThresholdHsv(srcMat);
//dstMat = myGrabCut(srcMat, new Point(50d, 0d), new Point(300d, 250d));
//dstMat = MyFindLargestRectangle(srcMat);
//dstMat = MyWatershed(srcMat);
//dstMat = MyCanny(srcMat, 100);
dstMat = MyFloodFill(srcMat);
Texture2D t2d = new Texture2D(dstMat.width(), dstMat.height());
Utils.matToTexture2D(dstMat, t2d);
Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero);
m_dstImage.sprite = sp;
m_dstImage.preserveAspect = true;
}
// Update is called once per frame
void Update()
{
if (frame3D.snapped)
{
Debug.Log("writing color to " + filePath + ".png");
Mat colorFlipped = new Mat();
Core.flip(frame3D.waterMarkedColor, colorFlipped, 0);
Mat rgb = new Mat();
Imgproc.cvtColor(colorFlipped, rgb, Imgproc.COLOR_RGB2BGR);
Imgcodecs.imwrite(filePath + ".png", rgb);
Mat depthFlipped = new Mat();
Core.flip(frame3D.waterMarkedDepth, depthFlipped, 0);
Debug.Log("writing depth to " + filePath + "_depth.png");
Imgcodecs.imwrite(filePath + "_depth.png", depthFlipped);
// Load the directory where the images are saved
Debug.Log("explorer.exe" + " /n, /e, " + outpath.Replace('/', '\\'));
System.Diagnostics.Process.Start("explorer.exe", "/n, /e, " + outpath.Replace('/', '\\'));
ShowWindow(GetActiveWindow(), 2);
}
}
private void SaveMarkerImg()
{
// save the markerImg.
string saveDirectoryPath = Path.Combine(Application.persistentDataPath, "ArUcoCreateMarkerExample");
string savePath = "";
#if UNITY_WEBGL && !UNITY_EDITOR
string format = "jpg";
MatOfInt compressionParams = new MatOfInt(Imgcodecs.CV_IMWRITE_JPEG_QUALITY, 100);
#else
string format = "png";
MatOfInt compressionParams = new MatOfInt(Imgcodecs.CV_IMWRITE_PNG_COMPRESSION, 0);
#endif
switch (markerType)
{
default:
case MarkerType.CanonicalMarker:
savePath = Path.Combine(saveDirectoryPath, "CanonicalMarker-d" + (int)dictionaryId + "-i" + (int)markerId + "-sp" + markerSize + "-bb" + borderBits + "." + format);
break;
case MarkerType.GridBoard:
savePath = Path.Combine(saveDirectoryPath, "GridBoard-mx" + gridBoradMarkersX + "-my" + gridBoradMarkersY + "-d" + (int)dictionaryId + "-os" + markerSize + "-bb" + borderBits + "." + format);
break;
case MarkerType.ChArUcoBoard:
savePath = Path.Combine(saveDirectoryPath, "ChArUcoBoard-mx" + chArUcoBoradMarkersX + "-my" + chArUcoBoradMarkersY + "-d" + (int)dictionaryId + "-os" + markerSize + "-bb" + borderBits + "." + format);
break;
}
if (!Directory.Exists(saveDirectoryPath))
{
Directory.CreateDirectory(saveDirectoryPath);
}
Imgcodecs.imwrite(savePath, markerImg, compressionParams);
savePathInputField.text = savePath;
Debug.Log("savePath: " + savePath);
}
public Mat Extract(string path)
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
var embedder = Dnn.readNetFromTorch(model_filepath);
Mat img = Imgcodecs.imread(Utils.getFilePath("faces/" + path));
if (img.empty())
{
Debug.LogError("image is not loaded");
return(img);
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
var roi = GetBB(img);
Mat cropped_face = img.submat((int)roi.y, (int)roi.y + (int)roi.height,
(int)roi.x, (int)roi.width + (int)roi.x);
var faceBlob = Dnn.blobFromImage(cropped_face, scalefactor, new Size(inpWidth, inpHeight), new Scalar(0, 0, 0), true, false);
embedder.setInput(faceBlob);
var netOut = embedder.forward();
if (gameObject.GetComponent <Renderer>() != null && displayBB)
{
GenericUtils.AdjustImageScale(cropped_face, this.gameObject);
Texture2D texture = new Texture2D(cropped_face.cols(), cropped_face.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(cropped_face, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
}
//_embedder.Dispose();
//cropped_face.Dispose();
img.Dispose();
return(netOut);
}
//Used to save pictures of faces to a jpg in the parent folder of the project, used in training the Eigen Face Recognizer.
public bool outRectToFile(Mat frame, ref int fileCounter)
{
Mat localCpy = new Mat();
int counter = 0;
String myFile;
foreach (OpenCVForUnity.Rect face in faceSquares.toArray())
{
myFile = "face" + fileCounter + ".jpg";
fileCounter++;
counter++;
//localCpy = new Mat (new Size (face.width, face.height))
localCpy = new Mat(frame, face);
Imgcodecs.imwrite(myFile, localCpy);
}
if (counter == 0)
{
return(false);
}
else
{
return(true);
}
}
public void Save(string Filename)
{
Imgcodecs.Imwrite(Filename, Mat);
}
void Run(string jpg_path)
{
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(jpg_path);
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (!string.IsNullOrEmpty(graph_filepath))
{
net = Dnn.readNetFromTensorflow(graph_filepath);
}
if (net == null)
{
Imgproc.putText(img, "Model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat input = Dnn.blobFromImage(img, 1.0, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.setInput(input, "image");
Mat output = net.forward("Openpose/concat_stage7");
output = output.reshape(1, 57);
List <Point> points = new List <Point>();
for (int i = 0; i < BODY_PARTS.Count; i++)
{
Mat heatMap = output.row(i).reshape(1, 46);
Core.MinMaxLocResult result = Core.minMaxLoc(heatMap);
heatMap.Dispose();
double x = (frameWidth * result.maxLoc.x) / 46;
double y = (frameHeight * result.maxLoc.y) / 46;
if (result.maxVal > 0.3)
{
points.Add(new Point(x, y));
}
else
{
points.Add(null);
}
}
for (int i = 0; i < POSE_PAIRS.GetLength(0); i++)
{
string partFrom = POSE_PAIRS[i, 0];
string partTo = POSE_PAIRS[i, 1];
int idFrom = BODY_PARTS[partFrom];
int idTo = BODY_PARTS[partTo];
if (points[idFrom] != null && points[idTo] != null)
{
Imgproc.line(img, points[idFrom], points[idTo], new Scalar(0, 255, 0), 3);
Imgproc.ellipse(img, points[idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
Imgproc.ellipse(img, points[idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
}
}
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
private void Run()
{
Mat src = Imgcodecs.imread(pca_test1_jpg_filepath);
#if !UNITY_WSA_10_0
if (src.empty())
{
Debug.LogError("pca_test1.jpg is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
#endif
Debug.Log("src.ToString() " + src.ToString());
// Convert image to grayscale
Mat gray = new Mat();
Imgproc.cvtColor(src, gray, Imgproc.COLOR_BGR2GRAY);
// Convert image to binary
Mat bw = new Mat();
Imgproc.threshold(gray, bw, 50, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
// Find all the contours in the thresholded image
Mat hierarchy = new Mat();
List <MatOfPoint> contours = new List <MatOfPoint> ();
Imgproc.findContours(bw, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_NONE);
for (int i = 0; i < contours.Count; ++i)
{
// Calculate the area of each contour
double area = Imgproc.contourArea(contours [i]);
// Ignore contours that are too small or too large
if (area < 1e2 || 1e5 < area)
{
continue;
}
// Draw each contour only for visualisation purposes
Imgproc.drawContours(src, contours, i, new Scalar(0, 0, 255), 2);
//Construct a buffer used by the pca analysis
List <Point> pts = contours [i].toList();
int sz = pts.Count;
Mat data_pts = new Mat(sz, 2, CvType.CV_64FC1);
for (int p = 0; p < data_pts.rows(); ++p)
{
data_pts.put(p, 0, pts [p].x);
data_pts.put(p, 1, pts [p].y);
}
Mat mean = new Mat();
Mat eigenvectors = new Mat();
Core.PCACompute(data_pts, mean, eigenvectors, 1);
Debug.Log("mean.dump() " + mean.dump());
Debug.Log("eigenvectors.dump() " + eigenvectors.dump());
Point cntr = new Point(mean.get(0, 0) [0], mean.get(0, 1) [0]);
Point vec = new Point(eigenvectors.get(0, 0) [0], eigenvectors.get(0, 1) [0]);
drawAxis(src, cntr, vec, new Scalar(255, 255, 0), 150);
data_pts.Dispose();
mean.Dispose();
eigenvectors.Dispose();
}
Imgproc.cvtColor(src, src, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(src.cols(), src.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(src, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}
private void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(scenetext01_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("text/scenetext01.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
}
#endif
//# for visualization
Mat vis = new Mat();
img.copyTo(vis);
Imgproc.cvtColor(vis, vis, Imgproc.COLOR_BGR2RGB);
//# Extract channels to be processed individually
List <Mat> channels = new List <Mat> ();
Text.computeNMChannels(img, channels);
//# Append negative channels to detect ER- (bright regions over dark background)
int cn = channels.Count;
for (int i = 0; i < cn; i++)
{
channels.Add(new Scalar(255) - channels [i]);
}
//# Apply the default cascade classifier to each independent channel (could be done in parallel)
Debug.Log("Extracting Class Specific Extremal Regions from " + channels.Count + " channels ...");
Debug.Log(" (...) this may take a while (...)");
foreach (var channel in channels)
{
ERFilter er1 = Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 16, 0.00015f, 0.13f, 0.2f, true, 0.1f);
ERFilter er2 = Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);
List <MatOfPoint> regions = new List <MatOfPoint> ();
Text.detectRegions(channel, er1, er2, regions);
MatOfRect matOfRects = new MatOfRect();
Text.erGrouping(img, channel, regions, matOfRects);
// Text.erGrouping (img, channel, regions, matOfRects, Text.ERGROUPING_ORIENTATION_ANY, Utils.getFilePath ("text/trained_classifier_erGrouping.xml"), 0.5f);
List <OpenCVForUnity.Rect> rects = matOfRects.toList();
//#Visualization
foreach (var rect in rects)
{
Imgproc.rectangle(vis, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 0, 0), 2);
Imgproc.rectangle(vis, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 255, 255), 1);
}
}
Texture2D texture = new Texture2D(vis.cols(), vis.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(vis, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/COCO_val2014_000000000589.jpg is not loaded.The image file can be downloaded here: \"https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/examples/media/COCO_val2014_000000000589.jpg\" folder. ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(caffemodel_filepath) || string.IsNullOrEmpty(prototxt_filepath))
{
Debug.LogError("model file is not loaded. The model and prototxt file can be downloaded here: \"http://posefs1.perception.cs.cmu.edu/OpenPose/models/pose/mpi/pose_iter_160000.caffemodel\",\"https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/openpose_pose_mpi_faster_4_stages.prototxt\". Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(prototxt_filepath, caffemodel_filepath);
//Intel's Deep Learning Inference Engine backend is supported on Windows 64bit platform only. Please refer to ReadMe.pdf for the setup procedure.
//net.setPreferableBackend (Dnn.DNN_BACKEND_INFERENCE_ENGINE);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat input = Dnn.blobFromImage(img, 1.0 / 255, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.setInput(input);
// TickMeter tm = new TickMeter ();
// tm.start ();
Mat output = net.forward();
// tm.stop ();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
output = output.reshape(1, 16);
float[] data = new float[46 * 46];
List <Point> points = new List <Point> ();
for (int i = 0; i < BODY_PARTS.Count; i++)
{
output.get(i, 0, data);
Mat heatMap = new Mat(1, data.Length, CvType.CV_32FC1);
heatMap.put(0, 0, data);
//Originally, we try to find all the local maximums. To simplify a sample
//we just find a global one. However only a single pose at the same time
//could be detected this way.
Core.MinMaxLocResult result = Core.minMaxLoc(heatMap);
heatMap.Dispose();
double x = (frameWidth * (result.maxLoc.x % 46)) / 46;
double y = (frameHeight * (result.maxLoc.x / 46)) / 46;
if (result.maxVal > 0.1)
{
points.Add(new Point(x, y));
}
else
{
points.Add(null);
}
}
for (int i = 0; i < POSE_PAIRS.GetLength(0); i++)
{
string partFrom = POSE_PAIRS [i, 0];
string partTo = POSE_PAIRS [i, 1];
int idFrom = BODY_PARTS [partFrom];
int idTo = BODY_PARTS [partTo];
if (points [idFrom] != null && points [idTo] != null)
{
Imgproc.line(img, points [idFrom], points [idTo], new Scalar(0, 255, 0), 3);
Imgproc.ellipse(img, points [idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
Imgproc.ellipse(img, points [idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
}
}
MatOfDouble timings = new MatOfDouble();
long t = net.getPerfProfile(timings);
Debug.Log("t: " + t);
Debug.Log("timings.dump(): " + timings.dump());
double freq = Core.getTickFrequency() / 1000;
Debug.Log("freq: " + freq);
Imgproc.putText(img, (t / freq) + "ms", new Point(10, img.height() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar(0, 0, 255), 2);
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
classNames = readClassNames(classes_filepath);
if (classNames == null)
{
Debug.LogError(classes_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
classColors = new List <Scalar> ();
for (int i = 0; i < classNames.Count; i++)
{
classColors.Add(new Scalar(UnityEngine.Random.Range(0, 255), UnityEngine.Random.Range(0, 255), UnityEngine.Random.Range(0, 255)));
}
Mat img = Imgcodecs.imread(image_filepath);
if (img.empty())
{
Debug.LogError(image_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(height, width, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(model_filepath) || string.IsNullOrEmpty(config_filepath))
{
Debug.LogError(model_filepath + " or " + config_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
net = Dnn.readNetFromTensorflow(model_filepath, config_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
float frameWidth = img.cols();
float frameHeight = img.rows();
Mat blob = Dnn.blobFromImage(img, 1.0, new Size(width, height), new Scalar(0, 0, 0), true, false);
net.setInput(blob);
List <Mat> outputBlobs = new List <Mat> ();
List <string> outputName = new List <string> ();
outputName.Add("detection_out_final");
outputName.Add("detection_masks");
net.forward(outputBlobs, outputName);
Mat boxes = outputBlobs [0];
Mat masks = outputBlobs [1];
//int numClasses = masks.size (1);
//int numDetections = boxes.size (2);
Debug.Log("boxes.size(0) " + boxes.size(0));
Debug.Log("boxes.size(1) " + boxes.size(1));
Debug.Log("boxes.size(2) " + boxes.size(2));
Debug.Log("boxes.size(3) " + boxes.size(3));
Debug.Log("masks.size(0) " + masks.size(0));
Debug.Log("masks.size(1) " + masks.size(1));
Debug.Log("masks.size(2) " + masks.size(2));
Debug.Log("masks.size(3) " + masks.size(3));
//reshape from 4D to two 2D.
float[] data = new float[boxes.size(3)];
boxes = boxes.reshape(1, (int)boxes.total() / boxes.size(3));
// Debug.Log ("boxes.ToString() " + boxes.ToString ());
//reshape from 4D to two 2D.
float[] mask_data = new float[masks.size(2) * masks.size(3)];
masks = masks.reshape(1, (int)masks.total() / (masks.size(2) * masks.size(3)));
// Debug.Log ("masks.ToString(): " + masks.ToString ());
for (int i = 0; i < boxes.rows(); i++)
{
boxes.get(i, 0, data);
float score = data [2];
if (score > thr)
{
int class_id = (int)(data [1]);
float left = (float)(data [3] * frameWidth);
float top = (float)(data [4] * frameHeight);
float right = (float)(data [5] * frameWidth);
float bottom = (float)(data [6] * frameHeight);
left = (int)Mathf.Max(0, Mathf.Min(left, frameWidth - 1));
top = (int)Mathf.Max(0, Mathf.Min(top, frameHeight - 1));
right = (int)Mathf.Max(0, Mathf.Min(right, frameWidth - 1));
bottom = (int)Mathf.Max(0, Mathf.Min(bottom, frameHeight - 1));
Debug.Log("class_id: " + class_id + " class_name " + classNames [class_id] + " left: " + left + " top: " + top + " right: " + right + " bottom: " + bottom);
//draw masks
masks.get((i * 90) + class_id, 0, mask_data);
Mat objectMask = new Mat(15, 15, CvType.CV_32F);
Utils.copyToMat <float> (mask_data, objectMask);
Imgproc.resize(objectMask, objectMask, new Size(right - left + 1, bottom - top + 1));
Core.compare(objectMask, new Scalar(mask_thr), objectMask, Core.CMP_GT);
// Debug.Log ("objectMask.ToString(): " + objectMask.ToString ());
// Debug.Log ("objectMask.dump(): " + objectMask.dump ());
Mat roi = new Mat(img, new OpenCVForUnity.CoreModule.Rect(new Point(left, top), new Point(right + 1, bottom + 1)));
Mat coloredRoi = new Mat(roi.size(), CvType.CV_8UC3);
Imgproc.rectangle(coloredRoi, new Point(0, 0), new Point(coloredRoi.width(), coloredRoi.height()), classColors [class_id], -1);
Core.addWeighted(coloredRoi, 0.7, roi, 0.3, 0, coloredRoi);
// Debug.Log ("coloredRoi.ToString(): " + coloredRoi.ToString ());
// Debug.Log ("roi.ToString(): " + roi.ToString ());
coloredRoi.copyTo(roi, objectMask);
coloredRoi.Dispose();
objectMask.Dispose();
//draw boxes
Imgproc.rectangle(img, new Point(left, top), new Point(right, bottom), new Scalar(0, 255, 0), 2);
string label = score.ToString();
if (classNames != null && classNames.Count != 0)
{
if (class_id < (int)classNames.Count)
{
label = classNames [class_id] + ": " + label;
}
}
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
top = Mathf.Max(top, (int)labelSize.height);
Imgproc.rectangle(img, new Point(left, top - labelSize.height),
new Point(left + labelSize.width, top + baseLine [0]), Scalar.all(255), Core.FILLED);
Imgproc.putText(img, label, new Point(left, top), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255));
}
}
boxes.Dispose();
masks.Dispose();
blob.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
net.Dispose();
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image_filepath, Imgcodecs.IMREAD_COLOR);
Mat colorized = new Mat(img.rows(), img.cols(), img.type());
if (img.empty())
{
Debug.LogError(image_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
Net net = null;
if (string.IsNullOrEmpty(caffemodel_filepath) || string.IsNullOrEmpty(prototxt_filepath))
{
Debug.LogError(caffemodel_filepath + " or " + prototxt_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
net = Dnn.readNetFromCaffe(prototxt_filepath, caffemodel_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
// setup additional layers:
int[] sz = new int[] { 2, 313, 1, 1 };
Mat pts_in_hull = new Mat(sz, CvType.CV_32F);
pts_in_hull.put(new int[] { 0, 0, 0, 0 }, hull_pts);
Layer class8_ab = net.getLayer(new DictValue("class8_ab"));
List <Mat> blobs = class8_ab.get_blobs();
blobs.Add(pts_in_hull);
class8_ab.set_blobs(blobs);
Layer conv8_313_rh = net.getLayer(new DictValue("conv8_313_rh"));
blobs = conv8_313_rh.get_blobs();
blobs.Add(new Mat(1, 313, CvType.CV_32F, new Scalar(2.606)));
conv8_313_rh.set_blobs(blobs);
// extract L channel and subtract mean
Mat img_32F = new Mat();
Mat lab = new Mat();
Mat L = new Mat();
Mat input = new Mat();
img.convertTo(img_32F, CvType.CV_32F, 1.0 / 255);
Imgproc.cvtColor(img_32F, lab, Imgproc.COLOR_BGR2Lab);
Core.extractChannel(lab, L, 0);
Imgproc.resize(L, input, new Size(inWidth, inHeight));
Core.subtract(input, new Scalar(50.0), input);
// run the L channel through the network
Mat inputBlob = Dnn.blobFromImage(input);
net.setInput(inputBlob);
Mat result = net.forward();
// retrieve the calculated a,b channels from the network output
Mat result_a = new Mat(result, new Range[] { new Range(0, 1), new Range(0, 1), new Range(0, result.size(2)), new Range(0, result.size(3)) });
Mat result_b = new Mat(result, new Range[] { new Range(0, 1), new Range(1, 2), new Range(0, result.size(2)), new Range(0, result.size(3)) });
result_a = result_a.reshape(1, result.size(2));
result_b = result_b.reshape(1, result.size(2));
Mat a = new Mat(img.size(), CvType.CV_32F);
Mat b = new Mat(img.size(), CvType.CV_32F);
Imgproc.resize(result_a, a, img.size());
Imgproc.resize(result_b, b, img.size());
// merge, and convert back to BGR
List <Mat> chn = new List <Mat>();
chn.Add(L); chn.Add(a); chn.Add(b);
Core.merge(chn, lab);
Imgproc.cvtColor(lab, img_32F, Imgproc.COLOR_Lab2BGR);
img_32F.convertTo(colorized, CvType.CV_8U, 255.0);
MatOfDouble timings = new MatOfDouble();
long t = net.getPerfProfile(timings);
double freq = Core.getTickFrequency() / 1000;
Debug.Log("Inference time " + (t / freq) + "ms");
Imgproc.putText(colorized, (t / freq) + "ms", new Point(10, img.height() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2);
Imgproc.putText(img, "gray", new Point(10, 20), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2);
Imgproc.putText(colorized, "colorized", new Point(10, 20), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2);
}
Imgproc.cvtColor(colorized, colorized, Imgproc.COLOR_BGR2RGB);
Mat display = new Mat(img.rows() * 2, img.cols(), img.type());
using (Mat gray = new Mat(img.rows(), img.cols(), CvType.CV_8UC1))
using (Mat displayUpperHalf = new Mat(display, new Range(0, img.rows())))
using (Mat displayLowerHalf = new Mat(display, new Range(img.rows(), display.rows())))
{
Imgproc.cvtColor(img, gray, Imgproc.COLOR_BGR2GRAY);
Imgproc.cvtColor(gray, img, Imgproc.COLOR_GRAY2RGB);
img.copyTo(displayUpperHalf);
colorized.copyTo(displayLowerHalf);
}
Texture2D texture = new Texture2D(display.cols(), display.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(display, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(display.width(), display.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = display.width();
float imageHeight = display.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Utils.setDebugMode(false);
}
public async Task <List <DetectedObject> > DetectAsync(CameraFrame frame)
{
if (frame == null)
{
throw new ArgumentNullException(nameof(frame));
}
Imgcodecs.imwrite(Application.persistentDataPath + "/testB.jpg", frame.Mat);
try
{
Debug.Log($"Enter PredictImageAsync with mat {frame.Mat}");
var imageWidth = frame.Width;
var imageHeight = frame.Height;
Mat rgb = new Mat(imageWidth, imageHeight, CvType.CV_8UC3);
if (frame.Format == ColorFormat.Grayscale)
{
Imgproc.cvtColor(frame.Mat, rgb, Imgproc.COLOR_GRAY2RGB);
Debug.Log($"Converted gray2rgb to {rgb}");
}
else
{
frame.Mat.copyTo(rgb);
}
//Mat rgba = new Mat();
//Imgproc.cvtColor(rgb, rgba, Imgproc.COLOR_RGB2RGBA);
float newHeight = 416.0f / imageWidth * imageHeight;
Mat resized = new Mat(416, 416, CvType.CV_8UC3);
Imgproc.resize(rgb, resized, new Size(416, newHeight), 0.5, 0.5, Imgproc.INTER_LINEAR);
//Imgproc.resize(rgb, resized, new Size(targetWidth, targetHeight), 0.5, 0.5, Imgproc.INTER_LINEAR);
Debug.Log($"Resized {resized}");
Mat resizedBorder = new Mat();
Core.copyMakeBorder(resized, resizedBorder, 0, (int)(416 - newHeight), 0, 0, Core.BORDER_CONSTANT, new Scalar(0, 0, 0));
/*Mat rgba = new Mat();
* Imgproc.cvtColor(resizedBorder, rgba, Imgproc.COLOR_RGB2RGBA);*/
Texture2D texture = new Texture2D(416, 416, TextureFormat.RGB24, false);
Utils.matToTexture2D(resizedBorder, texture, true);
//texture.Apply();
Color32[] pixels32 = texture.GetPixels32();
byte[] encodeArrayToJPG = ImageConversion.EncodeArrayToJPG(pixels32, GraphicsFormat.R8G8B8A8_UInt, 416, 416);
File.WriteAllBytes(Application.persistentDataPath + "/testA.jpg", encodeArrayToJPG);
using (var tensor = TransformInput(pixels32, ImageNetSettings.imageWidth, ImageNetSettings.imageHeight))
{
var inputs = new Dictionary <string, Tensor>();
inputs.Add(ModelSettings.ModelInput, tensor);
//yield return StartCoroutine(worker.StartManualSchedule(inputs));
//var output = engine.Execute(inputs).PeekOutput();
var output = engine.Execute(inputs).PeekOutput(ModelSettings.ModelOutput);
var results = outputParser.ParseOutputs(output, MINIMUM_CONFIDENCE);
var boxes = outputParser.FilterBoundingBoxes(results, 10, MINIMUM_CONFIDENCE);
foreach (var box in boxes)
{
Debug.Log($"{box.tagName}, {box.probability}, {box.boundingBox.left},{box.boundingBox.top},{box.boundingBox.width},{box.boundingBox.height},");
}
List <DetectedObject> detectedObjects = boxes.Select(prediction => CreateDetectedObject(frame, prediction, (int)newHeight)).ToList();
int count = 0;
foreach (var detectedObject in detectedObjects)
{
count++;
Mat clone = frame.Mat.clone();
Imgproc.rectangle(clone, detectedObject.Rect.tl(), detectedObject.Rect.br(), new Scalar(255, 255, 255), 10, 1, 0);
Imgcodecs.imwrite(Application.persistentDataPath + "/clone-" + count + ".jpg", clone);
}
}
}
catch (Exception e)
{
Debug.LogException(e);
throw e;
}
return(new List <DetectedObject>());
}
void Start()
{
srcMat = Imgcodecs.imread(Application.dataPath + "/Textures/sample.jpg", 1); //背景图
Imgproc.cvtColor(srcMat, srcMat, Imgproc.COLOR_BGR2RGB);
}
/// <summary>
/// Raises the load button click event.
/// </summary>
public void LoadModel()
{
string loadDirectoryPath = Path.Combine(Application.persistentDataPath, saveDirectoryName);
if (!Directory.Exists(loadDirectoryPath))
{
Debug.Log("load failure. saved train data file does not exist.");
return;
}
// Restart everything!
dispose();
if (facerecAlgorithm == "FaceRecognizer.Fisherfaces")
{
model = FisherFaceRecognizer.create();
}
else if (facerecAlgorithm == "FaceRecognizer.Eigenfaces")
{
model = EigenFaceRecognizer.create();
}
if (model == null)
{
Debug.LogError("ERROR: The FaceRecognizer algorithm [" + facerecAlgorithm + "] is not available in your version of OpenCV. Please update to OpenCV v2.4.1 or newer.");
m_mode = R_MODES.MODE_DETECTION;
return;
}
// load the train data.
model.read(Path.Combine(loadDirectoryPath, "traindata.yml"));
int maxLabel = (int)Core.minMaxLoc(model.getLabels()).maxVal;
if (maxLabel < 0)
{
Debug.Log("load failure.");
model.Dispose();
return;
}
// Restore the save data.
#if UNITY_WEBGL && !UNITY_EDITOR
string format = "jpg";
#else
string format = "png";
#endif
m_numPersons = maxLabel + 1;
personsNames = new string[m_numPersons];
for (int i = 0; i < m_numPersons; ++i)
{
personsNames[i] = GameManager.instance.personsNames[i];
m_latestFaces.Add(i);
preprocessedFaces.Add(Imgcodecs.imread(Path.Combine(loadDirectoryPath, "preprocessedface" + i + "." + format), 0));
if (preprocessedFaces[i].total() == 0)
{
preprocessedFaces[i] = new Mat(faceHeight, faceWidth, CvType.CV_8UC1, new Scalar(128));
}
faceLabels.Add(i);
}
// go to the recognition mode!
m_mode = R_MODES.MODE_RECOGNITION;
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
List <string> classNames = readClassNames(coco_names_filepath);
#if !UNITY_WSA_10_0
if (classNames == null)
{
Debug.LogError("class names list file is not loaded.The model and class names list can be downloaded here: \"https://github.com/pjreddie/darknet/tree/master/data/coco.names\".Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
#endif
Mat img = Imgcodecs.imread(person_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/person.jpg is not loaded.The image file can be downloaded here: \"https://github.com/pjreddie/darknet/blob/master/data/person.jpg\".Please copy to \"Assets/StreamingAssets/dnn/\" folder. ");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(tiny_yolo_cfg_filepath) || string.IsNullOrEmpty(tiny_yolo_weights_filepath))
{
Debug.LogError("model file is not loaded. the cfg-file and weights-file can be downloaded here: https://github.com/pjreddie/darknet/blob/master/cfg/tiny-yolo.cfg and https://pjreddie.com/media/files/tiny-yolo.weights. Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
//! [Initialize network]
net = Dnn.readNetFromDarknet(tiny_yolo_cfg_filepath, tiny_yolo_weights_filepath);
//! [Initialize network]
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
//! [Resizing without keeping aspect ratio]
Mat resized = new Mat();
Imgproc.resize(img, resized, new Size(network_width, network_height));
//! [Resizing without keeping aspect ratio]
//! [Prepare blob]
Mat inputBlob = Dnn.blobFromImage(resized, 1 / 255.0, new Size(), new Scalar(0), true, true); //Convert Mat to batch of images
//! [Prepare blob]
//! [Set input blob]
net.setInput(inputBlob, "data"); //set the network input
//! [Set input blob]
TickMeter tm = new TickMeter();
tm.start();
//! [Make forward pass]
Mat detectionMat = net.forward("detection_out"); //compute output
//! [Make forward pass]
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
Debug.Log("detectionMat.ToString(): " + detectionMat.ToString());
float[] position = new float[5];
float[] confidences = new float[80];
float confidenceThreshold = 0.24f;
for (int i = 0; i < detectionMat.rows(); i++)
{
detectionMat.get(i, 0, position);
detectionMat.get(i, 5, confidences);
int maxIdx = confidences.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I;
float confidence = confidences [maxIdx];
if (confidence > confidenceThreshold)
{
float x = position [0];
float y = position [1];
float width = position [2];
float height = position [3];
int xLeftBottom = (int)((x - width / 2) * img.cols());
int yLeftBottom = (int)((y - height / 2) * img.rows());
int xRightTop = (int)((x + width / 2) * img.cols());
int yRightTop = (int)((y + height / 2) * img.rows());
Debug.Log("confidence: " + confidence);
Debug.Log(" " + xLeftBottom
+ " " + yLeftBottom
+ " " + xRightTop
+ " " + yRightTop);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop),
new Scalar(0, 255, 0), 2);
if (maxIdx < classNames.Count)
{
string label = classNames [maxIdx] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(img, new Point(xLeftBottom, yLeftBottom),
new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255), Core.FILLED);
Imgproc.putText(img, label, new Point(xLeftBottom, yLeftBottom + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
}
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
private void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat frame = Imgcodecs.imread(scenetext01_jpg_filepath);
#if !UNITY_WSA_10_0
if (frame.empty())
{
Debug.LogError("text/scenetext01.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
}
#endif
Mat binaryMat = new Mat();
Mat maskMat = new Mat();
List <MatOfPoint> regions = new List <MatOfPoint> ();
ERFilter er_filter1 = Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);
ERFilter er_filter2 = Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);
Mat transition_p = new Mat(62, 62, CvType.CV_64FC1);
// string filename = "OCRHMM_transitions_table.xml";
// FileStorage fs(filename, FileStorage::READ);
// fs["transition_probabilities"] >> transition_p;
// fs.release();
//Load TransitionProbabilitiesData.
transition_p.put(0, 0, GetTransitionProbabilitiesData(OCRHMM_transitions_table_xml_filepath));
Mat emission_p = Mat.eye(62, 62, CvType.CV_64FC1);
string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
OCRHMMDecoder decoder = OCRHMMDecoder.create(
OCRHMM_knn_model_data_xml_gz_filepath,
voc, transition_p, emission_p);
//Text Detection
Imgproc.cvtColor(frame, frame, Imgproc.COLOR_BGR2RGB);
Imgproc.cvtColor(frame, binaryMat, Imgproc.COLOR_RGB2GRAY);
Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
Core.absdiff(binaryMat, new Scalar(255), maskMat);
Text.detectRegions(binaryMat, er_filter1, er_filter2, regions);
Debug.Log("regions.Count " + regions.Count);
MatOfRect groups_rects = new MatOfRect();
List <OpenCVForUnity.Rect> rects = new List <OpenCVForUnity.Rect> ();
Text.erGrouping(frame, binaryMat, regions, groups_rects);
for (int i = 0; i < regions.Count; i++)
{
regions [i].Dispose();
}
regions.Clear();
rects.AddRange(groups_rects.toList());
groups_rects.Dispose();
//Text Recognition (OCR)
List <Mat> detections = new List <Mat> ();
for (int i = 0; i < (int)rects.Count; i++)
{
Mat group_img = new Mat();
maskMat.submat(rects [i]).copyTo(group_img);
Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0));
detections.Add(group_img);
}
Debug.Log("detections.Count " + detections.Count);
//#Visualization
for (int i = 0; i < rects.Count; i++)
{
Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0), 2);
Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 255, 255), 1);
string output = decoder.run(detections [i], 0);
if (!string.IsNullOrEmpty(output))
{
Debug.Log("output " + output);
Imgproc.putText(frame, output, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 255), 1, Imgproc.LINE_AA, false);
}
}
Texture2D texture = new Texture2D(frame.cols(), frame.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(frame, texture);
// Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false);
//
// Utils.matToTexture2D (detections [0], texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
for (int i = 0; i < detections.Count; i++)
{
detections [i].Dispose();
}
binaryMat.Dispose();
maskMat.Dispose();
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(dnn004545_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("dnn/004545.jpg is not loaded.The image file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD/blob/master/images/004545.jpg\".Please copy to \"Assets/StreamingAssets/dnn/\" folder. ");
img = new Mat(375, 500, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
#endif
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(MobileNetSSD_deploy_caffemodel_filepath) || string.IsNullOrEmpty(MobileNetSSD_deploy_prototxt_filepath))
{
Debug.LogError("model file is not loaded.The model and prototxt file can be downloaded here: \"https://github.com/chuanqi305/MobileNet-SSD\".Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
else
{
net = Dnn.readNetFromCaffe(MobileNetSSD_deploy_prototxt_filepath, MobileNetSSD_deploy_caffemodel_filepath);
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Mat blob = Dnn.blobFromImage(img, inScaleFactor, new Size(inWidth, inHeight), new Scalar(meanVal, meanVal, meanVal), false, false);
net.setInput(blob);
TickMeter tm = new TickMeter();
tm.start();
Mat prob = net.forward();
prob = prob.reshape(1, (int)prob.total() / 7);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
float[] data = new float[7];
float confidenceThreshold = 0.2f;
for (int i = 0; i < prob.rows(); i++)
{
prob.get(i, 0, data);
float confidence = data [2];
if (confidence > confidenceThreshold)
{
int class_id = (int)(data [1]);
float left = data [3] * img.cols();
float top = data [4] * img.rows();
float right = data [5] * img.cols();
float bottom = data [6] * img.rows();
Debug.Log("class_id: " + class_id);
Debug.Log("Confidence: " + confidence);
Debug.Log(" " + left
+ " " + top
+ " " + right
+ " " + bottom);
Imgproc.rectangle(img, new Point(left, top), new Point(right, bottom),
new Scalar(0, 255, 0), 2);
string label = classNames [class_id] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
top = Mathf.Max(top, (float)labelSize.height);
Imgproc.rectangle(img, new Point(left, top),
new Point(left + labelSize.width, top + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255), Core.FILLED);
Imgproc.putText(img, label, new Point(left, top + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
prob.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
if (!string.IsNullOrEmpty(classes))
{
classNames = readClassNames(classes_filepath);
if (classNames == null)
{
Debug.LogError(classes_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
}
else if (classesList.Count > 0)
{
classNames = classesList;
}
Mat img = Imgcodecs.imread(input_filepath);
if (img.empty())
{
Debug.LogError(input_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(config_filepath) || string.IsNullOrEmpty(model_filepath))
{
Debug.LogError(config_filepath + " or " + model_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
//! [Initialize network]
net = Dnn.readNet(model_filepath, config_filepath);
//! [Initialize network]
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
outBlobNames = getOutputsNames(net);
//for (int i = 0; i < outBlobNames.Count; i++)
//{
// Debug.Log("names [" + i + "] " + outBlobNames[i]);
//}
outBlobTypes = getOutputsTypes(net);
//for (int i = 0; i < outBlobTypes.Count; i++)
//{
// Debug.Log("types [" + i + "] " + outBlobTypes[i]);
//}
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : img.cols(),
inpHeight > 0 ? inpHeight : img.rows());
Mat blob = Dnn.blobFromImage(img, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(img, img, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
postprocess(img, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
net.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
void Start()
{
srcMat = Imgcodecs.imread(Application.dataPath + "/Textures/lena.jpg", 1); //512
Imgproc.cvtColor(srcMat, srcMat, Imgproc.COLOR_BGR2RGB);
Imgproc.resize(srcMat, srcMat, new Size(512, 512));
Texture2D src_t2d = new Texture2D(srcMat.width(), srcMat.height());
Utils.matToTexture2D(srcMat, src_t2d);
Sprite src_sp = Sprite.Create(src_t2d, new UnityEngine.Rect(0, 0, src_t2d.width, src_t2d.height), Vector2.zero);
m_srcImage.sprite = src_sp;
m_srcImage.rectTransform.offsetMin = new Vector2(0, 0);
m_srcImage.rectTransform.offsetMax = new Vector2(srcMat.width(), srcMat.height());
m_srcImage.rectTransform.anchoredPosition = Vector2.zero;
Mat mask = Mat.zeros(srcMat.size(), CvType.CV_8UC1); //注意CvType必须是CV_8UC1
Point p0 = new Point(0, 0);
Point p1 = new Point(0, 256);
Point p2 = new Point(256, 0);
MatOfPoint pts1 = new MatOfPoint(new Point[3] {
p0, p1, p2
});
MatOfPoint2f srcTri = new MatOfPoint2f(new Point[3] {
p0, p1, p2
});
Point p3 = new Point(256, 0);
Point p4 = new Point(512, 0);
Point p5 = new Point(512, 256);
Point p6 = new Point(256, 64);
MatOfPoint pts2 = new MatOfPoint(new Point[3] {
p3, p4, p5
});
MatOfPoint2f dstTri = new MatOfPoint2f(new Point[3] {
p0, p1, p6
});
List <MatOfPoint> contour = new List <MatOfPoint>()
{
pts1
};
for (int i = 0; i < contour.Count; i++)
{
//轮廓提取
Imgproc.drawContours(mask, contour, i, new Scalar(255), -1); //全部放到mask上
}
srcMat.copyTo(mask, mask);
Mat warpMat = Imgproc.getAffineTransform(srcTri, dstTri);
Mat warpImage = Mat.zeros(mask.size(), mask.type());
Imgproc.warpAffine(mask, warpImage, warpMat, warpImage.size());
//------------------------------------------------//
/*
* // Offset points by left top corner of the respective rectangles
* OpenCVForUnity.Rect r1 = Imgproc.boundingRect(pts1);
* OpenCVForUnity.Rect r2 = Imgproc.boundingRect(pts2);
* MatOfPoint2f t1Rect = new MatOfPoint2f();
* MatOfPoint2f t2Rect = new MatOfPoint2f();
* MatOfPoint t2RectInt = new MatOfPoint();
* for (int i = 0; i < 3; i++)
* {
* t1Rect.push_back(new Mat((int)pts1.toList()[i].x - r1.x, (int)pts1.toList()[i].y - r1.y, 0));
* t2Rect.push_back(new Mat((int)pts2.toList()[i].x - r2.x, (int)pts2.toList()[i].y - r2.y, 0));
* t2RectInt.push_back(new Mat((int)pts2.toList()[i].x - r2.x, (int)pts2.toList()[i].y - r2.y, 0)); // for fillConvexPoly
* }
*
* MatOfPoint PointArray = new MatOfPoint();
* dstMat = Mat.zeros(srcMat.size(), CvType.CV_8UC3);
* PointArray.fromList(new List<Point>()
* {
* new Point(50,10),
* new Point(300,12),
* new Point(350,250),
* new Point(9,250),
* });
* Debug.Log(PointArray);
* Imgproc.fillConvexPoly(dstMat, PointArray, new Scalar(255, 0, 0), 4, 0);
*/
//------------------------------------------------//
Texture2D dst_t2d = new Texture2D(warpImage.width(), warpImage.height());
Utils.matToTexture2D(warpImage, dst_t2d);
Sprite sp = Sprite.Create(dst_t2d, new UnityEngine.Rect(0, 0, dst_t2d.width, dst_t2d.height), Vector2.zero);
m_roiImage.sprite = sp;
m_roiImage.preserveAspect = true;
m_roiImage.rectTransform.offsetMin = new Vector2(0, 0);
m_roiImage.rectTransform.offsetMax = new Vector2(srcMat.width(), srcMat.height());
m_roiImage.rectTransform.anchoredPosition = Vector2.zero;
}