public bool descriptorsORB_Old(Mat RGB, Mat cameraFeed, string targetName)//找出特徵的顏色方法三(可運行但效率不佳放棄) { if (RGB == null) { Debug.Log("RGB Mat is Null"); return(false); } //將傳入的RGB存入Src Mat SrcMat = new Mat(); RGB.copyTo(SrcMat); //比對樣本 Texture2D imgTexture = Resources.Load(targetName) as Texture2D; // Texture2D imgTexture2 = Resources.Load("lenaK") as Texture2D; //Texture2D轉Mat Mat img1Mat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3); Utils.texture2DToMat(imgTexture, img1Mat); //創建 ORB的特徵點裝置 FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB); DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.ORB); //產生存放特徵點Mat MatOfKeyPoint keypoints1 = new MatOfKeyPoint(); Mat descriptors1 = new Mat(); MatOfKeyPoint keypointsSrc = new MatOfKeyPoint(); Mat descriptorsSrc = new Mat(); //找特徵點圖1 detector.detect(img1Mat, keypoints1); extractor.compute(img1Mat, keypoints1, descriptors1); //找特徵點圖Src detector.detect(SrcMat, keypointsSrc); extractor.compute(SrcMat, keypointsSrc, descriptorsSrc); DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMINGLUT); MatOfDMatch matches = new MatOfDMatch(); matcher.match(descriptors1, descriptorsSrc, matches); DMatch[] arrayDmatch = matches.toArray(); for (int i = arrayDmatch.Length - 1; i >= 0; i--) { // Debug.Log("match " + i + ": " + arrayDmatch[i].distance); } //做篩選 double max_dist = 0; double min_dist = 100; //-- Quick calculation of max and min distances between keypoints double dist = new double(); for (int i = 0; i < matches.rows(); i++) { dist = arrayDmatch[i].distance; if (dist < min_dist) { min_dist = dist; } if (dist > max_dist) { max_dist = dist; } } Debug.Log("Max dist :" + max_dist); Debug.Log("Min dist :" + min_dist); //只畫好的點 List <DMatch> matchesGoodList = new List <DMatch>(); for (int i = 0; i < matches.rows(); i++) { //if (arrayDmatch[i].distance < RateDist.value * min_dist) //{ // //Debug.Log("match " + i + ": " + arrayDmatch[i].distance); // matchesGoodList.Add(arrayDmatch[i]); //} } MatOfDMatch matchesGood = new MatOfDMatch(); matchesGood.fromList(matchesGoodList); //Draw Keypoints Features2d.drawKeypoints(SrcMat, keypointsSrc, SrcMat); //做輸出的轉換予宣告 Mat resultImg = new Mat(); // Features2d.drawMatches(img1Mat, keypoints1, SrcMat, keypointsSrc, matchesGood, resultImg); List <Point> P1 = new List <Point>(); // List<Point> P2 = new List<Point>(); List <Point> pSrc = new List <Point>(); Debug.Log("MatchCount" + matchesGoodList.Count); for (int i = 0; i < matchesGoodList.Count; i++) { P1.Add(new Point(keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.x, keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.y)); pSrc.Add(new Point(keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.x, keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.y)); //Debug.Log("ID = " + matchesGoodList[i].queryIdx ); //Debug.Log("x,y =" + (int)keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.x + "," + (int)keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.y); //Debug.Log("x,y =" + (int)keypoints2.toArray()[matchesGoodList[i].trainIdx].pt.x + "," + (int)keypoints2.toArray()[matchesGoodList[i].trainIdx].pt.y); } MatOfPoint2f p2fTarget = new MatOfPoint2f(P1.ToArray()); MatOfPoint2f p2fSrc = new MatOfPoint2f(pSrc.ToArray()); Mat matrixH = Calib3d.findHomography(p2fTarget, p2fSrc, Calib3d.RANSAC, 3); List <Point> srcPointCorners = new List <Point>(); srcPointCorners.Add(new Point(0, 0)); srcPointCorners.Add(new Point(img1Mat.width(), 0)); srcPointCorners.Add(new Point(img1Mat.width(), img1Mat.height())); srcPointCorners.Add(new Point(0, img1Mat.height())); Mat originalRect = Converters.vector_Point2f_to_Mat(srcPointCorners); List <Point> srcPointCornersEnd = new List <Point>(); srcPointCornersEnd.Add(new Point(0, img1Mat.height())); srcPointCornersEnd.Add(new Point(0, 0)); srcPointCornersEnd.Add(new Point(img1Mat.width(), 0)); srcPointCornersEnd.Add(new Point(img1Mat.width(), img1Mat.height())); Mat changeRect = Converters.vector_Point2f_to_Mat(srcPointCornersEnd); Core.perspectiveTransform(originalRect, changeRect, matrixH); List <Point> srcPointCornersSave = new List <Point>(); Converters.Mat_to_vector_Point(changeRect, srcPointCornersSave); if ((srcPointCornersSave[2].x - srcPointCornersSave[0].x) < 5 || (srcPointCornersSave[2].y - srcPointCornersSave[0].y) < 5) { Debug.Log("Match Out Put image is to small"); SrcMat.copyTo(cameraFeed); SrcMat.release(); Imgproc.putText(cameraFeed, "X-S", new Point(10, 50), 0, 1, new Scalar(255, 255, 255), 2); return(false); } // Features2d.drawMatches(img1Mat, keypoints1, SrcMat, keypointsSrc, matchesGood, resultImg); Imgproc.line(SrcMat, srcPointCornersSave[0], srcPointCornersSave[1], new Scalar(255, 0, 0), 3); Imgproc.line(SrcMat, srcPointCornersSave[1], srcPointCornersSave[2], new Scalar(255, 0, 0), 3); Imgproc.line(SrcMat, srcPointCornersSave[2], srcPointCornersSave[3], new Scalar(255, 0, 0), 3); Imgproc.line(SrcMat, srcPointCornersSave[3], srcPointCornersSave[0], new Scalar(255, 0, 0), 3); SrcMat.copyTo(cameraFeed); keypoints1.release(); img1Mat.release(); SrcMat.release(); return(true); }
//============================================================ //=================以下為沒有再使用的函式===================== //============================================================ //找出特徵的顏色方法三(ORB特徵點比對) public bool descriptorsORB(Mat RGB, Mat cameraFeed, string targetName) { if (RGB == null) { Debug.Log("RGB Mat is Null"); return(false); } //將傳入的RGB存入Src Mat SrcMat = new Mat(); RGB.copyTo(SrcMat); //比對樣本載入 Texture2D imgTexture = Resources.Load(targetName) as Texture2D; //Texture2D轉Mat Mat targetMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3); Utils.texture2DToMat(imgTexture, targetMat); //創建 ORB的特徵點裝置 FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB); DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.ORB); //產生存放特徵點Mat MatOfKeyPoint keypointsTarget = new MatOfKeyPoint(); Mat descriptorsTarget = new Mat(); MatOfKeyPoint keypointsSrc = new MatOfKeyPoint(); Mat descriptorsSrc = new Mat(); //找特徵點圖Target detector.detect(targetMat, keypointsTarget); extractor.compute(targetMat, keypointsTarget, descriptorsTarget); //找特徵點圖Src detector.detect(SrcMat, keypointsSrc); extractor.compute(SrcMat, keypointsSrc, descriptorsSrc); //創建特徵點比對物件 DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMINGLUT); MatOfDMatch matches = new MatOfDMatch(); //丟入兩影像的特徵點 matcher.match(descriptorsTarget, descriptorsSrc, matches); DMatch[] arrayDmatch = matches.toArray(); //做篩選 double max_dist = 0; double min_dist = 100; //-- Quick calculation of max and min distances between keypoints double dist = new double(); for (int i = 0; i < matches.rows(); i++) { dist = arrayDmatch[i].distance; if (dist < min_dist) { min_dist = dist; } if (dist > max_dist) { max_dist = dist; } } Debug.Log("Max dist :" + max_dist); Debug.Log("Min dist :" + min_dist); List <DMatch> matchesGoodList = new List <DMatch>(); MatOfDMatch matchesGood = new MatOfDMatch(); matchesGood.fromList(matchesGoodList); //Draw Keypoints Features2d.drawKeypoints(SrcMat, keypointsSrc, SrcMat); List <Point> pTarget = new List <Point>(); List <Point> pSrc = new List <Point>(); Debug.Log("MatchCount" + matchesGoodList.Count); for (int i = 0; i < matchesGoodList.Count; i++) { pTarget.Add(new Point(keypointsTarget.toArray()[matchesGoodList[i].queryIdx].pt.x, keypointsTarget.toArray()[matchesGoodList[i].queryIdx].pt.y)); pSrc.Add(new Point(keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.x, keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.y)); } MatOfPoint2f p2fTarget = new MatOfPoint2f(pTarget.ToArray()); MatOfPoint2f p2fSrc = new MatOfPoint2f(pSrc.ToArray()); Mat matrixH = Calib3d.findHomography(p2fTarget, p2fSrc, Calib3d.RANSAC, 3); List <Point> srcPointCorners = new List <Point>(); srcPointCorners.Add(new Point(0, 0)); srcPointCorners.Add(new Point(targetMat.width(), 0)); srcPointCorners.Add(new Point(targetMat.width(), targetMat.height())); srcPointCorners.Add(new Point(0, targetMat.height())); Mat originalRect = Converters.vector_Point2f_to_Mat(srcPointCorners); List <Point> srcPointCornersEnd = new List <Point>(); srcPointCornersEnd.Add(new Point(0, targetMat.height())); srcPointCornersEnd.Add(new Point(0, 0)); srcPointCornersEnd.Add(new Point(targetMat.width(), 0)); srcPointCornersEnd.Add(new Point(targetMat.width(), targetMat.height())); Mat changeRect = Converters.vector_Point2f_to_Mat(srcPointCornersEnd); Core.perspectiveTransform(originalRect, changeRect, matrixH); List <Point> srcPointCornersSave = new List <Point>(); Converters.Mat_to_vector_Point(changeRect, srcPointCornersSave); if ((srcPointCornersSave[2].x - srcPointCornersSave[0].x) < 5 || (srcPointCornersSave[2].y - srcPointCornersSave[0].y) < 5) { Debug.Log("Match Out Put image is to small"); SrcMat.copyTo(cameraFeed); SrcMat.release(); Imgproc.putText(cameraFeed, targetName, srcPointCornersSave[0], 0, 1, new Scalar(255, 255, 255), 2); return(false); } //畫出框框 Imgproc.line(SrcMat, srcPointCornersSave[0], srcPointCornersSave[1], new Scalar(255, 0, 0), 3); Imgproc.line(SrcMat, srcPointCornersSave[1], srcPointCornersSave[2], new Scalar(255, 0, 0), 3); Imgproc.line(SrcMat, srcPointCornersSave[2], srcPointCornersSave[3], new Scalar(255, 0, 0), 3); Imgproc.line(SrcMat, srcPointCornersSave[3], srcPointCornersSave[0], new Scalar(255, 0, 0), 3); //畫中心 Point middlePoint = new Point((srcPointCornersSave[0].x + srcPointCornersSave[2].x) / 2, (srcPointCornersSave[0].y + srcPointCornersSave[2].y) / 2); Imgproc.line(SrcMat, middlePoint, middlePoint, new Scalar(0, 0, 255), 10); SrcMat.copyTo(cameraFeed); keypointsTarget.release(); targetMat.release(); SrcMat.release(); return(true); }
public void UpdateAttitude(Mat mat) { int LandmarksCount = 0; int MatchsCount = 0; using (MatOfKeyPoint keypoints = new MatOfKeyPoint()) using (Mat descriptors = new Mat()) { detector.detect(mat, keypoints); extractor.compute(mat, keypoints, descriptors); var trainPoints = keypoints.toArray(); List <List <Vector3> > newLandmarks = new List <List <Vector3> >(); foreach (var keyPoint in trainPoints) { var keyVectorL = new List <Vector3>(); keyVectorL.Add(ARCameraManager.Instance.ToVector(keyPoint)); newLandmarks.Add(keyVectorL); LandmarksCount++; } if (Landmarks.Count > 0) { List <Vector3> FromVectorL = new List <Vector3>(); List <Vector3> ToVectorL = new List <Vector3>(); using (MatOfDMatch matches = new MatOfDMatch()) using (MatOfDMatch crossMatches = new MatOfDMatch()) { matcher.match(MapDescriptors, descriptors, matches); matcher.match(descriptors, MapDescriptors, crossMatches); var matchL = matches.toArray(); var crossMatchL = crossMatches.toArray(); int i = 0; foreach (DMatch match in matchL) { bool flag = false; foreach (DMatch crossMatch in crossMatchL) { if (match.trainIdx == crossMatch.queryIdx && match.queryIdx == crossMatch.trainIdx) { flag = true; MatchsCount++; } } if (match.distance > MatchFilter) { flag = false; } if (flag) { var trainVectors = newLandmarks[match.trainIdx]; var queryVectors = Landmarks[match.queryIdx]; FromVectorL.Add(trainVectors[0]); //ToVectorL.Add(queryVectors.ToArray().Median()); START double[] queryPointsX = new double[queryVectors.Count]; double[] queryPointsY = new double[queryVectors.Count]; for (int j = 0; j < queryVectors.Count; j++) { var queryPoint = ARCameraManager.Instance.toPoint(queryVectors[j], Attitude); queryPointsX[j] = queryPoint.x; queryPointsY[j] = queryPoint.y; } ToVectorL.Add(Attitude * ARCameraManager.Instance.ToVector(new Point(queryPointsX.Median(), queryPointsY.Median()))); //ToVectorL.Add(queryVectors.ToArray().Median()); END newLandmarks[match.trainIdx].AddRange(queryVectors.ToArray()); } i++; } Quaternion newAttitude; float error = ARCameraManager.Instance.LMedS(FromVectorL, ToVectorL, out newAttitude); _matchTestEvent.Invoke(FromVectorL.Count); FromVectorL.Clear(); ToVectorL.Clear(); if (error > 0 && LMedSFilter > error) { Attitude = newAttitude; _trackingTestEvent.Invoke(Attitude); //ARCameraManager.Instance.UpdateCameraPosture(Attitude); if (debugMode) { Debug.Log(string.Format("Attitude = {0}\nError = {1}", Attitude, error)); } } foreach (var newLandmark in newLandmarks) { newLandmark[0] = Attitude * newLandmark[0]; } } } MapDescriptors.Dispose(); Landmarks.Clear(); Landmarks = newLandmarks; MapDescriptors = descriptors.clone(); } float now = Time.time; if (debugMode) { Debug.Log(string.Format("time : {0} Landmarks : {1}, Matchs : {2}.", 1 / (Time.time - startime), LandmarksCount, MatchsCount)); } startime = Time.time; }
// Update is called once per frame public void UpdateScreen(Mat mat) { using (MatOfKeyPoint keypoints = new MatOfKeyPoint()) using (Mat descriptors = new Mat()) { detector.detect(mat, keypoints); extractor.compute(mat, keypoints, descriptors); int matchCount = 0; var trainPoints = keypoints.toArray(); var newBuffPointL = new List <List <Point> >(); var newBuffColorL = new List <Scalar>(); foreach (var keyPoint in trainPoints) { var points = new List <Point>(); points.Add(keyPoint.pt); newBuffPointL.Add(points); Scalar color = new Scalar(255, 225, 225, 100); var x = Random.Range(0, 256); switch (Random.Range(0, 6)) { case 0: color = new Scalar(255, 0, x, 100); break; case 1: color = new Scalar(0, 255, x, 100); break; case 2: color = new Scalar(0, x, 255, 100); break; case 3: color = new Scalar(225, x, 0, 100); break; case 4: color = new Scalar(x, 0, 255, 100); break; case 5: color = new Scalar(x, 255, 0, 100); break; } newBuffColorL.Add(color); } if (buffPointL.Count > 0) { using (MatOfDMatch matches = new MatOfDMatch()) using (MatOfDMatch crossMatches = new MatOfDMatch()) { matcher.match(buffDescriptors, descriptors, matches); matcher.match(descriptors, buffDescriptors, crossMatches); var matchL = matches.toArray(); var crossMatchL = crossMatches.toArray(); int i = 0; foreach (DMatch match in matchL) { bool flag = false; foreach (DMatch crossMatch in crossMatchL) { if (match.trainIdx == crossMatch.queryIdx && match.queryIdx == crossMatch.trainIdx) { flag = true; } } if (match.distance > filter) { flag = false; } if (flag) { var trainPoint = trainPoints[match.trainIdx]; var queryPoints = buffPointL[match.queryIdx]; int a = (int)trainPoint.pt.x / mat.width() * 250; int b = (int)trainPoint.pt.y / mat.height() * 250; Scalar color = buffColorL[match.queryIdx]; Imgproc.circle(mat, trainPoint.pt, 4, color, -1); Point startPoint = trainPoint.pt; foreach (Point queryPoint in queryPoints) { Imgproc.line(mat, startPoint, queryPoint, color, 2); Imgproc.circle(mat, queryPoint, 4, color, -1); startPoint = queryPoint; newBuffPointL[match.trainIdx].Add(queryPoint); } newBuffColorL[match.trainIdx] = buffColorL[match.queryIdx]; matchCount++; } i++; } } } buffDescriptors.Dispose(); buffPointL = newBuffPointL; buffColorL = newBuffColorL; buffDescriptors = descriptors.clone(); text = string.Format("Matching Count : {0}.", matchCount); DestroyImmediate(tex); tex = new Texture2D(ARCameraManager.Instance.Width, ARCameraManager.Instance.Height); OpenCVForUnity.Utils.matToTexture2D(mat, tex); ARCameraManager.Instance.UpdateScreenTexture(tex); } }
public void Tracking(Mat mat) { using (MatOfKeyPoint keypoints = new MatOfKeyPoint()) using (Mat descriptors = new Mat()) { detector.detect(mat, keypoints); extractor.compute(mat, keypoints, descriptors); var trainPoints = keypoints.toArray(); List <List <Vector3> > newLandmarks = new List <List <Vector3> >(); List <List <Mat> > newDescriptors = new List <List <Mat> >(); for (int i = 0; i < trainPoints.Length; i++) { var keyVectorL = new List <Vector3>(); keyVectorL.Add(ARCameraManager.Instance.ToVector(trainPoints[i])); var DescriptorL = new List <Mat>(); DescriptorL.Add(descriptors.clone().row(i)); newLandmarks.Add(keyVectorL); newDescriptors.Add(DescriptorL); } List <Vector3> FromVectorL = new List <Vector3>(); List <Vector3> ToVectorL = new List <Vector3>(); List <int> FinalizingL = new List <int>(); bool finLMedS = false; if (FinalizingLandmarks.Count > 0) { using (MatOfDMatch matchesFinal = new MatOfDMatch()) using (MatOfDMatch crossMatchesFinal = new MatOfDMatch()) { matcher.match(FinalizingLandmarkDescriptors, descriptors, matchesFinal); matcher.match(descriptors, FinalizingLandmarkDescriptors, crossMatchesFinal); var matchLFinal = matchesFinal.toArray(); var crossMatchLFinal = crossMatchesFinal.toArray(); int i = 0; foreach (DMatch match in matchLFinal) { bool flag = false; foreach (DMatch crossMatch in crossMatchLFinal) { if (match.trainIdx == crossMatch.queryIdx && match.queryIdx == crossMatch.trainIdx) { flag = true; } } if (match.distance > MatchFilter) { flag = false; } if (flag) { FromVectorL.Add(newLandmarks[match.trainIdx][0]); ToVectorL.Add(FinalizingLandmarks[match.queryIdx]); FinalizingL.Add(match.trainIdx); newLandmarks[match.trainIdx][0] = FinalizingLandmarks[match.queryIdx]; newDescriptors[match.trainIdx][0] = FinalizingLandmarkDescriptors.row(match.queryIdx); } i++; } Quaternion newAttitude; float error = ARCameraManager.Instance.LMedS(FromVectorL, ToVectorL, out newAttitude); if (error > 0 && LMedSFilter > error) { Attitude = newAttitude; _trackingEvent.Invoke(Attitude); _matchingEvent.Invoke(FromVectorL.Count); //ARCameraManager.Instance.UpdateCameraPosture(Attitude); Debug.Log(string.Format("Attitude = {0}\nError = {1}\nFinalizMatch = {2}\nAccuracy = {3}", Attitude, error, FinalizingL.Count, 100 * FinalizingL.Count / FromVectorL.Count)); finLMedS = true; } } } if (ProvisioningLandmarks.Count > 0) { using (MatOfDMatch matches = new MatOfDMatch()) using (MatOfDMatch crossMatches = new MatOfDMatch()) { Mat optimisationDescriptors = OptimisationDescriptors; matcher.match(optimisationDescriptors, descriptors, matches); matcher.match(descriptors, optimisationDescriptors, crossMatches); var matchL = matches.toArray(); var crossMatchL = crossMatches.toArray(); int i = 0; foreach (DMatch match in matchL) { bool flag = false; foreach (DMatch crossMatch in crossMatchL) { if (match.trainIdx == crossMatch.queryIdx && match.queryIdx == crossMatch.trainIdx) { flag = true; } } if (match.distance > MatchFilter) { flag = false; } if (flag) { if (FinalizingL.IndexOf(match.trainIdx) < 0) { var trainVectors = newLandmarks[match.trainIdx]; var queryVectors = ProvisioningLandmarks[match.queryIdx]; Vector3 queryVector; int filter = OptimisationLandmark(queryVectors, Attitude, out queryVector); if (filter > 0) { if ((filter > SufficientCount) && (matchL.Length * FinalizedPercentage < FinalizingL.Count || matchL.Length * FinalizedPercentage > FinalizingLandmarks.Count)) { FinalizingLandmarks.Add(queryVector); if (FinalizingLandmarkDescriptors != null) { FinalizingLandmarkDescriptors.push_back(optimisationDescriptors.row(match.queryIdx)); } else { FinalizingLandmarkDescriptors = optimisationDescriptors.row(match.queryIdx); } Debug.Log(string.Format("Finalizing :Landmark = {0}\nDescriptors = {1}\nCount ALL = {2}", queryVector, optimisationDescriptors.row(match.queryIdx).ToStringMat(), FinalizingLandmarks.Count)); } else { FromVectorL.Add(trainVectors[0]); ToVectorL.Add(queryVector); newLandmarks[match.trainIdx].AddRange(queryVectors.ToArray()); newDescriptors[match.trainIdx].AddRange(ProvisioningLandmarkDescriptors[match.queryIdx].ToArray()); } } } } i++; } } } if (FromVectorL.Count == ToVectorL.Count && ToVectorL.Count > 0) { Quaternion newAttitude; float error = ARCameraManager.Instance.LMedS(FromVectorL, ToVectorL, out newAttitude); if ((error > 0 && LMedSFilter > error) && (!finLMedS)) { Attitude = newAttitude; _trackingEvent.Invoke(Attitude); //ARCameraManager.Instance.UpdateCameraPosture(Attitude); Debug.Log(string.Format("Attitude = {0}\nError = {1}\nFinalizMatch = {2}\nAccuracy = {3}", Attitude, error, FinalizingL.Count, 100 * FinalizingL.Count / FromVectorL.Count)); } for (int i = 0; i < newLandmarks.Count; i++) { if (FinalizingL.IndexOf(i) < 0) { newLandmarks[i][0] = Attitude * newLandmarks[i][0]; } } } _matchingEvent.Invoke(FromVectorL.Count); FromVectorL.Clear(); ToVectorL.Clear(); ProvisioningLandmarks.Clear(); ProvisioningLandmarks = newLandmarks; ProvisioningLandmarkDescriptors = newDescriptors; } }