void Rectify(ref Point[] face_point_array, int i)
{
Debug.Log("R: Starting");
homoMat_array[i] = new Mat(480, 640, CvType.CV_8UC1);
reg_point_array[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
reg_point_array[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
reg_point_array[2] = new Point(0.0, 0.0);
reg_point_array[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
Debug.Log("R: reg_point_array populated");
MatOfPoint2f srcPoints = new MatOfPoint2f(face_point_array);
MatOfPoint2f regPoints = new MatOfPoint2f(reg_point_array);
// Debug.Log("R: src and reg points instantiated");
Debug.LogFormat("Rectify Face Points; {0} \n {1} \n {2} \n {3}",
face_point_array[0], face_point_array[1], face_point_array[2], face_point_array[3]);
// Creating the H Matrix
Mat Homo_Mat = Calib3d.findHomography(srcPoints, regPoints);
Debug.Log("R: H Matrix Instantiated");
Imgproc.warpPerspective(cached_initMat, homoMat_array[i], Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
Debug.Log("R: image rectified");
}
void HomographyTransform(int i)
{
// Init homography result Mat
homoMat_array[i] = new Mat(480, 640, CvType.CV_8UC1);
// Init regular point array
reg_point_array[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
reg_point_array[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
reg_point_array[2] = new Point(0.0, 0.0);
reg_point_array[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
// Extract face_points corresponding with reg_points
Point[] out_point_array = new Point[4];
for (int j = 0; j < 4; j++) // j :: face point count
{
int src_i = face_index[i, j];
out_point_array[j] = proj_point_array[src_i];
}
MatOfPoint2f regPoints = new MatOfPoint2f(reg_point_array);
MatOfPoint2f outPoints = new MatOfPoint2f(out_point_array);
Mat Homo_Mat = Calib3d.findHomography(regPoints, outPoints);
Imgproc.warpPerspective(rectMat_array[i], homoMat_array[i], Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
}
// Update is called once per frame
void Update()
{
MatDisplay.SetCameraFoV(41.5f);
Image cameraImage = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888);
if (cameraImage == null)
{
return;
}
if (_cameraImageMat == null)
{
//First frame -> generate Mat with same dimensions as camera feed
_cameraImageMat = new Mat(cameraImage.Height, cameraImage.Width, CvType.CV_8UC4);
}
_cameraImageMat.put(0, 0, cameraImage.Pixels); // transferring image data to Mat
if (FingerPointTarget.GetComponent <ImageTargetBehaviour>().CurrentStatus != TrackableBehaviour.Status.TRACKED)
{
MatDisplay.DisplayMat(_cameraImageMat, MatDisplaySettings.FULL_BACKGROUND);
return;
}
FindHomographyPoints(out var matDst, out var matObj);
var H = Calib3d.findHomography(matObj, matDst);
try
{
var bWMat = GetBWSkinColor();
var fingerTipCoor = FindFingerTip(bWMat);
var fingerPointInWorldSpace = FingerPointInWorldSpace(fingerTipCoor);
FingerPlane.position = fingerPointInWorldSpace;
var colorPixelValue = FindPixelValue(_cameraImageMat, Color.position);
var drawPixelValue = FindPixelValue(bWMat, Draw.position);
if ((int)drawPixelValue.First() == 255)
{
//Debug.Log($"{colorPixelValue[0]}, {colorPixelValue[1]}, {colorPixelValue[2]}");
//Debug.Log("Found Draw");
//draw at finger pos
var camMask = PaintCircle(fingerTipCoor);
DrawMaskOnCanvas(camMask, H, colorPixelValue);
}
}
catch
{
}
var blendTex = BlendMats(H, _cameraImageMat, _drawingPlaceMat);
MatDisplay.DisplayMat(blendTex, MatDisplaySettings.FULL_BACKGROUND);
}
// Warps cached_homoMat to outMat
void HomographyTransform(ref Mat homoMat)
{
Corner_AR_Controller Homo_Controller = m_ARSessionManager.GetComponent <Corner_AR_Controller>();
Point[] c2_scrpoints = Homo_Controller.GetScreenpoints(false);
MatOfPoint2f initPoints = new MatOfPoint2f(regPointArray);
MatOfPoint2f currPoints = new MatOfPoint2f(c2_scrpoints);
Mat H = Calib3d.findHomography(initPoints, currPoints);
Imgproc.warpPerspective(homoMat, outMat, H, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
Core.flip(outMat, outMat, 0);
}
private void refresh()
{
var defaultPoints = defaultCornerPoints.Select(e => e.point).ToArray();
var destPoints = CornerPoints.Select(e => e.point).ToArray();
using (var defaultCornerMat = new MatOfPoint2f(defaultPoints))
using (var destCornerMat = new MatOfPoint2f(destPoints))
using (var defaultMat = new MatOfPoint2f(defaultVertices.Select(e => new Point(e.x, e.y)).ToArray()))
using (var destMat = new MatOfPoint2f(meshFilter.mesh.vertices.Select(e => new Point(e.x, e.y)).ToArray()))
{
var h = Calib3d.findHomography(defaultCornerMat, destCornerMat);
OpenCVForUnity.CoreModule.Core.perspectiveTransform(defaultMat, destMat, h);
var vertices = destMat.toList().Select(e => new Vector3((float)e.x, (float)e.y, 0f)).ToList();//resultPoints.Select (e => new Vector3((float)e.x,(float)e.y,0f)).ToList();
meshFilter.mesh.SetVertices(vertices);
}
}
void ComputerVisionAlgo(IntPtr greyscale)
{
Utils.copyToMat(greyscale, imageMat);
// Inverting Image pixel values
MatOfKeyPoint keyMat = new MatOfKeyPoint();
inMat = imageMat;
// inMat = (Mat.ones(imageMat.rows(), imageMat.cols(), CvType.CV_8UC1) * 255) - imageMat;
// Creating Detector (Yellow Circle)
// MatOfKeyPoint keyMat = new MatOfKeyPoint();
// SimpleBlobDetector detector = SimpleBlobDetector.create();
double[] homo_points = m_ARSessionManager.GetComponent <AR_Controller>().GetHomopoints();
// Display Homography Points
// outMat = inMat;
// Imgproc.circle(outMat, new Point(homo_points[0], homo_points[1]), 5, new Scalar(0.0, 0.0, 255.0));
// Imgproc.circle(outMat, new Point(homo_points[2], homo_points[3]), 5, new Scalar(0.0, 0.0, 255.0));
// Imgproc.circle(outMat, new Point(homo_points[4], homo_points[5]), 5, new Scalar(0.0, 0.0, 255.0));
// Imgproc.circle(outMat, new Point(homo_points[6], homo_points[7]), 5, new Scalar(0.0, 0.0, 255.0));
// Creating MatOfPoint2f arrays for Homography Points
Point[] srcPointArray = new Point[4];
for (int i = 0; i < 4; i++)
{
srcPointArray[i] = new Point(homo_points[2 * i], homo_points[(2 * i) + 1]);
}
Point[] dstPointArray = new Point[4];
dstPointArray[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
dstPointArray[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
dstPointArray[2] = new Point(0.0, 0.0);
dstPointArray[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
MatOfPoint2f srcPoints = new MatOfPoint2f(srcPointArray);
MatOfPoint2f dstPoints = new MatOfPoint2f(dstPointArray);
// Creating the H Matrix
Mat Homo_Mat = Calib3d.findHomography(srcPoints, dstPoints);
Debug.Log(Homo_Mat);
Debug.Log(outMat.size());
Imgproc.warpPerspective(inMat, outMat, Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
}
void ComputerVisionAlgo(IntPtr greyscale)
{
// Utils.copyToMat(greyscale, imageMat);
inMat = cached_initMat;
Plane_AR_Controller Homo_Controller = m_ARSessionManager.GetComponent <Plane_AR_Controller>();
Point[] c1_scrpoints = Homo_Controller.GetScreenpoints(true);
Point[] c2_scrpoints = Homo_Controller.GetScreenpoints(false);
MatOfPoint2f initPoints = new MatOfPoint2f(c1_scrpoints);
MatOfPoint2f currPoints = new MatOfPoint2f(c2_scrpoints);
Mat H = Calib3d.findHomography(initPoints, currPoints);
Imgproc.warpPerspective(inMat, outMat, H, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
}
public ImageObject warpImage(List <ImageObject> imageList)
{
Texture2D imgTexture = base64ImageToTexture(imageList[0].image);
//Find the matching keypoints from the winner list.
MatOfPoint2f queryPoints = new MatOfPoint2f();
MatOfPoint2f matchPoints = new MatOfPoint2f();
List <Point> queryPointsList = new List <Point>();
List <Point> matchPointsList = new List <Point>();
foreach (MatOfDMatch match in imageList[1].matches)
{
DMatch[] arrayDmatch = match.toArray();
queryPointsList.Add(imageList[0].keyPoints.toList()[arrayDmatch[0].queryIdx].pt);
matchPointsList.Add(imageList[1].keyPoints.toList()[arrayDmatch[0].trainIdx].pt);
}
queryPoints.fromList(queryPointsList);
matchPoints.fromList(matchPointsList);
//Calculate the homography of the best matching image
//Mat homography = Calib3d.findHomography(queryPoints, matchPoints, Calib3d.RANSAC, 5.0);
Mat homography = Calib3d.findHomography(queryPoints, matchPoints, Calib3d.RANSAC, 3.0);
Mat tempResultImg = new Mat();
Imgproc.warpPerspective(imageList[0].imageMat, tempResultImg, homography, imageList[1].imageMat.size());
//Show (red) annotations only
ImageProcessor imageProcessor = new ImageProcessor();
Mat resultImg = imageProcessor.ShowAnnotationsOnly(tempResultImg);
//Show image
Texture2D tempTexture = new Texture2D(imageList[1].imageMat.cols(), imageList[1].imageMat.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(resultImg, tempTexture);
//Make black pixels transparent
Texture2D texture = imageProcessor.removeColor(Color.black, tempTexture);
return(new ImageObject(Convert.ToBase64String(texture.EncodeToPNG()), imageList[1].index));
//return new ImageObject(Convert.ToBase64String(tempTexture.EncodeToPNG()), imageList[1].index);
}
void Rectify(ref Point[] facePointArray, ref Mat cachedMat)
{
regPointArray[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
regPointArray[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
regPointArray[2] = new Point(0.0, 0.0);
regPointArray[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
MatOfPoint2f srcPoints = new MatOfPoint2f(facePointArray);
MatOfPoint2f regPoints = new MatOfPoint2f(regPointArray);
Debug.LogFormat("Rectify Face Points; {0} \n {1} \n {2} \n {3}",
facePointArray[0], facePointArray[1], facePointArray[2], facePointArray[3]);
// Creating the H Matrix
Mat Homo_Mat = Calib3d.findHomography(srcPoints, regPoints);
Imgproc.warpPerspective(imageMat, cachedMat, Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
}
void Rectify(ref Point[] face_point_array, int i)
{
rectMat_array[i] = new Mat(360, 640, CvType.CV_8UC1);
Point[] reg_point_array = new Point[4];
reg_point_array[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
reg_point_array[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
reg_point_array[2] = new Point(0.0, 0.0);
reg_point_array[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
MatOfPoint2f srcPoints = new MatOfPoint2f(face_point_array);
MatOfPoint2f regPoints = new MatOfPoint2f(reg_point_array);
// Debug.LogFormat("Rectify Face Points; {0} \n {1} \n {2} \n {3}",
// face_point_array[0], face_point_array[1], face_point_array[2], face_point_array[3]);
// Creating the H Matrix
Mat Homo_Mat = Calib3d.findHomography(srcPoints, regPoints);
Imgproc.warpPerspective(cached_initMat, rectMat_array[i], Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
}
// Detects Blobs with Detector Framework and stores Top-down view into cached_homoMat
void BlobDetection()
{
SimpleBlobDetector detector = SimpleBlobDetector.create();
// Core.flip(cached_initMat, imageMat, 0);
cached_initMat = imageMat;
keyMat = new MatOfKeyPoint();
detector.detect(imageMat, keyMat);
// Features2d.drawKeypoints(imageMat, keyMat, outMat);
if (keyMat.rows() < 4)
{
return;
}
for (int i = 0; i < 4; i++)
{
srcPointArray[i] = new Point(keyMat.get(i, 0)[0], keyMat.get(i, 0)[1]);
}
SortPoints();
regPointArray[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
regPointArray[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
regPointArray[2] = new Point(0.0, 0.0);
regPointArray[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
MatOfPoint2f srcPoints = new MatOfPoint2f(srcPointArray);
MatOfPoint2f regPoints = new MatOfPoint2f(regPointArray);
// Creating the H Matrix
Mat Homo_Mat = Calib3d.findHomography(srcPoints, regPoints);
Imgproc.warpPerspective(imageMat, cached_homoMat, Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
}
void Update()
{
//Access camera image provided by Vuforia
Image camImg = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888);
if (camImg != null)
{
if (camImageMat == null)
{
//First time -> instantiate camera image specific data
camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4); //Note: rows=height, cols=width
}
camImageMat.put(0, 0, camImg.Pixels);
//Replace with your own projection matrix. This approach only uses fy.
cam.fieldOfView = 2 * Mathf.Atan(camImg.Height * 0.5f / fy) * Mathf.Rad2Deg;
Vector3 worldPnt1 = corner1.transform.position;
Vector3 worldPnt2 = corner2.transform.position;
Vector3 worldPnt3 = corner3.transform.position;
Vector3 worldPnt4 = corner4.transform.position;
//See lecture slides
Matrix4x4 Rt = cam.transform.worldToLocalMatrix;
Matrix4x4 A = Matrix4x4.identity;
A.m00 = fx;
A.m11 = fy;
A.m02 = cx;
A.m12 = cy;
Matrix4x4 worldToImage = A * Rt;
Vector3 hUV1 = worldToImage.MultiplyPoint3x4(worldPnt1);
Vector3 hUV2 = worldToImage.MultiplyPoint3x4(worldPnt2);
Vector3 hUV3 = worldToImage.MultiplyPoint3x4(worldPnt3);
Vector3 hUV4 = worldToImage.MultiplyPoint3x4(worldPnt4);
//hUV are the image coordinates in 2D homogeneous coordinates, we need to normalize, i.e., divide by Z
Vector2 uv1 = new Vector2(hUV1.x, hUV1.y) / hUV1.z;
Vector2 uv2 = new Vector2(hUV2.x, hUV2.y) / hUV2.z;
Vector2 uv3 = new Vector2(hUV3.x, hUV3.y) / hUV3.z;
Vector2 uv4 = new Vector2(hUV4.x, hUV4.y) / hUV4.z;
//don't forget to alloc before putting values into a MatOfPoint2f
imagePoints.put(0, 0, uv1.x, camImg.Height - uv1.y);
imagePoints.put(1, 0, uv2.x, camImg.Height - uv2.y);
imagePoints.put(2, 0, uv3.x, camImg.Height - uv3.y);
imagePoints.put(3, 0, uv4.x, camImg.Height - uv4.y);
//Debug draw points
Point imgPnt1 = new Point(imagePoints.get(0, 0));
Point imgPnt2 = new Point(imagePoints.get(1, 0));
Point imgPnt3 = new Point(imagePoints.get(2, 0));
Point imgPnt4 = new Point(imagePoints.get(3, 0));
Imgproc.circle(camImageMat, imgPnt1, 5, new Scalar(255, 0, 0, 255));
Imgproc.circle(camImageMat, imgPnt2, 5, new Scalar(0, 255, 0, 255));
Imgproc.circle(camImageMat, imgPnt3, 5, new Scalar(0, 0, 255, 255));
Imgproc.circle(camImageMat, imgPnt4, 5, new Scalar(255, 255, 0, 255));
Scalar lineCl = new Scalar(200, 120, 0, 160);
Imgproc.line(camImageMat, imgPnt1, imgPnt2, lineCl);
Imgproc.line(camImageMat, imgPnt2, imgPnt3, lineCl);
Imgproc.line(camImageMat, imgPnt3, imgPnt4, lineCl);
Imgproc.line(camImageMat, imgPnt4, imgPnt1, lineCl);
var destPoints = new MatOfPoint2f(); // Creating a destination
destPoints.alloc(4);
destPoints.put(0, 0, width, 0);
destPoints.put(1, 0, width, height);
destPoints.put(2, 0, 0, height);
destPoints.put(3, 0, 0, 0);
var homography = Calib3d.findHomography(imagePoints, destPoints); // Finding the image
Imgproc.warpPerspective(camImageMat, destPoints, homography, new Size(camImageMat.width(), camImageMat.height()));
unwarpedTexture = unwarpedTextureClean;
MatDisplay.MatToTexture(destPoints, ref unwarpedTexture); // Take output and transform into texture
if (Input.GetKey("space"))
{
fish.GetComponent <Renderer>().material.mainTexture = unwarpedTexture;
}
else
{
fish.GetComponent <Renderer>().material.mainTexture = tex;
}
MatDisplay.DisplayMat(destPoints, MatDisplaySettings.BOTTOM_LEFT);
MatDisplay.DisplayMat(camImageMat, MatDisplaySettings.FULL_BACKGROUND);
}
}
/// <summary>
/// Refines the matches with homography.
/// </summary>
/// <returns><c>true</c>, if matches with homography was refined, <c>false</c> otherwise.</returns>
/// <param name="queryKeypoints">Query keypoints.</param>
/// <param name="trainKeypoints">Train keypoints.</param>
/// <param name="reprojectionThreshold">Reprojection threshold.</param>
/// <param name="matches">Matches.</param>
/// <param name="homography">Homography.</param>
static bool refineMatchesWithHomography
(
MatOfKeyPoint queryKeypoints,
MatOfKeyPoint trainKeypoints,
float reprojectionThreshold,
MatOfDMatch matches,
Mat homography
)
{
//Debug.Log ("matches " + matches.ToString ());
int minNumberMatchesAllowed = 8;
List <KeyPoint> queryKeypointsList = queryKeypoints.toList();
List <KeyPoint> trainKeypointsList = trainKeypoints.toList();
List <DMatch> matchesList = matches.toList();
if (matchesList.Count < minNumberMatchesAllowed)
{
return(false);
}
// Prepare data for cv::findHomography
List <Point> srcPointsList = new List <Point>(matchesList.Count);
List <Point> dstPointsList = new List <Point>(matchesList.Count);
for (int i = 0; i < matchesList.Count; i++)
{
srcPointsList.Add(trainKeypointsList[matchesList[i].trainIdx].pt);
dstPointsList.Add(queryKeypointsList[matchesList[i].queryIdx].pt);
}
// Find homography matrix and get inliers mask
using (MatOfPoint2f srcPoints = new MatOfPoint2f())
using (MatOfPoint2f dstPoints = new MatOfPoint2f())
using (MatOfByte inliersMask = new MatOfByte(new byte[srcPointsList.Count]))
{
srcPoints.fromList(srcPointsList);
dstPoints.fromList(dstPointsList);
//Debug.Log ("srcPoints " + srcPoints.ToString ());
//Debug.Log ("dstPoints " + dstPoints.ToString ());
Calib3d.findHomography(srcPoints,
dstPoints,
Calib3d.FM_RANSAC,
reprojectionThreshold,
inliersMask, 2000, 0.955).copyTo(homography);
if (homography.rows() != 3 || homography.cols() != 3)
{
return(false);
}
//Debug.Log ("homography " + homography.ToString ());
//Debug.Log ("inliersMask " + inliersMask.dump ());
List <byte> inliersMaskList = inliersMask.toList();
List <DMatch> inliers = new List <DMatch>();
for (int i = 0; i < inliersMaskList.Count; i++)
{
if (inliersMaskList[i] == 1)
{
inliers.Add(matchesList[i]);
}
}
matches.fromList(inliers);
//Debug.Log ("matches " + matches.ToString ());
}
return(matchesList.Count > minNumberMatchesAllowed);
}
public ImageString MatchFeatures(string base64image, List <string> base64imageList)
{
List <MatOfDMatch> winnerMatches = new List <MatOfDMatch>();
MatOfKeyPoint winnerKeyPoints = new MatOfKeyPoint();
Mat winnerImage = new Mat();
int winnerIndex = -1;
int winnerValue = 0;
Texture2D imgTexture = base64ImageToTexture(base64image);
List <Texture2D> imgTextures = new List <Texture2D>();
for (int i = 0; i < base64imageList.Count; i++)
{
imgTextures.Add(base64ImageToTexture(base64imageList[i]));
}
//Create Mat from texture
Mat img1Mat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3);
Utils.texture2DToMat(imgTexture, img1Mat);
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
Mat descriptors1 = new Mat();
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.ORB);
//Detect keypoints and compute descriptors from photo.
detector.detect(img1Mat, keypoints1);
extractor.compute(img1Mat, keypoints1, descriptors1);
Debug.Log("Billede features: " + descriptors1.rows());
if (descriptors1.rows() < 10)
{
Debug.Log("ARRRRRRGH der er ikke mange descripters i mit original-billede");
return(new ImageString(base64image, winnerIndex));
}
//Run through each image in list
for (int i = 0; i < imgTextures.Count; i++)
{
Texture2D imgTexture2 = imgTextures[i];
//Create Mat from texture
Mat img2Mat = new Mat(imgTexture2.height, imgTexture2.width, CvType.CV_8UC3);
Utils.texture2DToMat(imgTexture2, img2Mat);
//Find keypoints and descriptors from image in list
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
Mat descriptors2 = new Mat();
detector.detect(img2Mat, keypoints2);
extractor.compute(img2Mat, keypoints2, descriptors2);
//Match photo with image from list
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMINGLUT);
Debug.Log("Billede2 features: " + descriptors2.rows());
if (descriptors2.rows() < 10)
{
Debug.Log("ARRRRRRGH der er ikke mange descripters i mit test billede: " + i);
continue;
}
List <MatOfDMatch> matchList = new List <MatOfDMatch>();
matcher.knnMatch(descriptors1, descriptors2, matchList, 2);
//Find the good matches and put them ind a list
List <MatOfDMatch> good = new List <MatOfDMatch>();
foreach (MatOfDMatch match in matchList)
{
DMatch[] arrayDmatch = match.toArray();
if (arrayDmatch[0].distance < 0.7f * arrayDmatch[1].distance)
{
good.Add(match);
}
}
//Find the best match image based on the good lists
if (good.Count > winnerThreshold && good.Count > winnerValue)
{
winnerImage = img2Mat;
winnerMatches = good;
winnerKeyPoints = keypoints2;
winnerIndex = i;
winnerValue = good.Count;
}
}
Debug.Log("The winner is image: " + winnerIndex + " with a value of: " + winnerValue);
//If no winner just return the original image
if (winnerIndex == -1)
{
Debug.Log("No winner");
return(new ImageString(base64image, winnerIndex));
}
Debug.Log("No winner");
//Find the matching keypoints from the winner list.
MatOfPoint2f queryPoints = new MatOfPoint2f();
MatOfPoint2f matchPoints = new MatOfPoint2f();
List <Point> queryPointsList = new List <Point>();
List <Point> matchPointsList = new List <Point>();
foreach (MatOfDMatch match in winnerMatches)
{
DMatch[] arrayDmatch = match.toArray();
queryPointsList.Add(keypoints1.toList()[arrayDmatch[0].queryIdx].pt);
matchPointsList.Add(winnerKeyPoints.toList()[arrayDmatch[0].trainIdx].pt);
}
queryPoints.fromList(queryPointsList);
matchPoints.fromList(matchPointsList);
//Calculate the homography of the best matching image
Mat homography = Calib3d.findHomography(queryPoints, matchPoints, Calib3d.RANSAC, 5.0);
Mat resultImg = new Mat();
Imgproc.warpPerspective(img1Mat, resultImg, homography, winnerImage.size());
//Show image
Texture2D texture = new Texture2D(winnerImage.cols(), winnerImage.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(resultImg, texture);
return(new ImageString(Convert.ToBase64String(texture.EncodeToPNG()), winnerIndex));
}
public bool descriptorsORB_Old(Mat RGB, Mat cameraFeed, string targetName)//找出特徵的顏色方法三(可運行但效率不佳放棄)
{
if (RGB == null)
{
Debug.Log("RGB Mat is Null");
return(false);
}
//將傳入的RGB存入Src
Mat SrcMat = new Mat();
RGB.copyTo(SrcMat);
//比對樣本
Texture2D imgTexture = Resources.Load(targetName) as Texture2D;
// Texture2D imgTexture2 = Resources.Load("lenaK") as Texture2D;
//Texture2D轉Mat
Mat img1Mat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3);
Utils.texture2DToMat(imgTexture, img1Mat);
//創建 ORB的特徵點裝置
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.ORB);
//產生存放特徵點Mat
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
Mat descriptors1 = new Mat();
MatOfKeyPoint keypointsSrc = new MatOfKeyPoint();
Mat descriptorsSrc = new Mat();
//找特徵點圖1
detector.detect(img1Mat, keypoints1);
extractor.compute(img1Mat, keypoints1, descriptors1);
//找特徵點圖Src
detector.detect(SrcMat, keypointsSrc);
extractor.compute(SrcMat, keypointsSrc, descriptorsSrc);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMINGLUT);
MatOfDMatch matches = new MatOfDMatch();
matcher.match(descriptors1, descriptorsSrc, matches);
DMatch[] arrayDmatch = matches.toArray();
for (int i = arrayDmatch.Length - 1; i >= 0; i--)
{
// Debug.Log("match " + i + ": " + arrayDmatch[i].distance);
}
//做篩選
double max_dist = 0;
double min_dist = 100;
//-- Quick calculation of max and min distances between keypoints
double dist = new double();
for (int i = 0; i < matches.rows(); i++)
{
dist = arrayDmatch[i].distance;
if (dist < min_dist)
{
min_dist = dist;
}
if (dist > max_dist)
{
max_dist = dist;
}
}
Debug.Log("Max dist :" + max_dist);
Debug.Log("Min dist :" + min_dist);
//只畫好的點
List <DMatch> matchesGoodList = new List <DMatch>();
for (int i = 0; i < matches.rows(); i++)
{
//if (arrayDmatch[i].distance < RateDist.value * min_dist)
//{
// //Debug.Log("match " + i + ": " + arrayDmatch[i].distance);
// matchesGoodList.Add(arrayDmatch[i]);
//}
}
MatOfDMatch matchesGood = new MatOfDMatch();
matchesGood.fromList(matchesGoodList);
//Draw Keypoints
Features2d.drawKeypoints(SrcMat, keypointsSrc, SrcMat);
//做輸出的轉換予宣告
Mat resultImg = new Mat();
// Features2d.drawMatches(img1Mat, keypoints1, SrcMat, keypointsSrc, matchesGood, resultImg);
List <Point> P1 = new List <Point>();
// List<Point> P2 = new List<Point>();
List <Point> pSrc = new List <Point>();
Debug.Log("MatchCount" + matchesGoodList.Count);
for (int i = 0; i < matchesGoodList.Count; i++)
{
P1.Add(new Point(keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.x, keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.y));
pSrc.Add(new Point(keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.x, keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.y));
//Debug.Log("ID = " + matchesGoodList[i].queryIdx );
//Debug.Log("x,y =" + (int)keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.x + "," + (int)keypoints1.toArray()[matchesGoodList[i].queryIdx].pt.y);
//Debug.Log("x,y =" + (int)keypoints2.toArray()[matchesGoodList[i].trainIdx].pt.x + "," + (int)keypoints2.toArray()[matchesGoodList[i].trainIdx].pt.y);
}
MatOfPoint2f p2fTarget = new MatOfPoint2f(P1.ToArray());
MatOfPoint2f p2fSrc = new MatOfPoint2f(pSrc.ToArray());
Mat matrixH = Calib3d.findHomography(p2fTarget, p2fSrc, Calib3d.RANSAC, 3);
List <Point> srcPointCorners = new List <Point>();
srcPointCorners.Add(new Point(0, 0));
srcPointCorners.Add(new Point(img1Mat.width(), 0));
srcPointCorners.Add(new Point(img1Mat.width(), img1Mat.height()));
srcPointCorners.Add(new Point(0, img1Mat.height()));
Mat originalRect = Converters.vector_Point2f_to_Mat(srcPointCorners);
List <Point> srcPointCornersEnd = new List <Point>();
srcPointCornersEnd.Add(new Point(0, img1Mat.height()));
srcPointCornersEnd.Add(new Point(0, 0));
srcPointCornersEnd.Add(new Point(img1Mat.width(), 0));
srcPointCornersEnd.Add(new Point(img1Mat.width(), img1Mat.height()));
Mat changeRect = Converters.vector_Point2f_to_Mat(srcPointCornersEnd);
Core.perspectiveTransform(originalRect, changeRect, matrixH);
List <Point> srcPointCornersSave = new List <Point>();
Converters.Mat_to_vector_Point(changeRect, srcPointCornersSave);
if ((srcPointCornersSave[2].x - srcPointCornersSave[0].x) < 5 || (srcPointCornersSave[2].y - srcPointCornersSave[0].y) < 5)
{
Debug.Log("Match Out Put image is to small");
SrcMat.copyTo(cameraFeed);
SrcMat.release();
Imgproc.putText(cameraFeed, "X-S", new Point(10, 50), 0, 1, new Scalar(255, 255, 255), 2);
return(false);
}
// Features2d.drawMatches(img1Mat, keypoints1, SrcMat, keypointsSrc, matchesGood, resultImg);
Imgproc.line(SrcMat, srcPointCornersSave[0], srcPointCornersSave[1], new Scalar(255, 0, 0), 3);
Imgproc.line(SrcMat, srcPointCornersSave[1], srcPointCornersSave[2], new Scalar(255, 0, 0), 3);
Imgproc.line(SrcMat, srcPointCornersSave[2], srcPointCornersSave[3], new Scalar(255, 0, 0), 3);
Imgproc.line(SrcMat, srcPointCornersSave[3], srcPointCornersSave[0], new Scalar(255, 0, 0), 3);
SrcMat.copyTo(cameraFeed);
keypoints1.release();
img1Mat.release();
SrcMat.release();
return(true);
}
//============================================================
//=================以下為沒有再使用的函式=====================
//============================================================
//找出特徵的顏色方法三(ORB特徵點比對)
public bool descriptorsORB(Mat RGB, Mat cameraFeed, string targetName)
{
if (RGB == null)
{
Debug.Log("RGB Mat is Null");
return(false);
}
//將傳入的RGB存入Src
Mat SrcMat = new Mat();
RGB.copyTo(SrcMat);
//比對樣本載入
Texture2D imgTexture = Resources.Load(targetName) as Texture2D;
//Texture2D轉Mat
Mat targetMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3);
Utils.texture2DToMat(imgTexture, targetMat);
//創建 ORB的特徵點裝置
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.ORB);
//產生存放特徵點Mat
MatOfKeyPoint keypointsTarget = new MatOfKeyPoint();
Mat descriptorsTarget = new Mat();
MatOfKeyPoint keypointsSrc = new MatOfKeyPoint();
Mat descriptorsSrc = new Mat();
//找特徵點圖Target
detector.detect(targetMat, keypointsTarget);
extractor.compute(targetMat, keypointsTarget, descriptorsTarget);
//找特徵點圖Src
detector.detect(SrcMat, keypointsSrc);
extractor.compute(SrcMat, keypointsSrc, descriptorsSrc);
//創建特徵點比對物件
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMINGLUT);
MatOfDMatch matches = new MatOfDMatch();
//丟入兩影像的特徵點
matcher.match(descriptorsTarget, descriptorsSrc, matches);
DMatch[] arrayDmatch = matches.toArray();
//做篩選
double max_dist = 0;
double min_dist = 100;
//-- Quick calculation of max and min distances between keypoints
double dist = new double();
for (int i = 0; i < matches.rows(); i++)
{
dist = arrayDmatch[i].distance;
if (dist < min_dist)
{
min_dist = dist;
}
if (dist > max_dist)
{
max_dist = dist;
}
}
Debug.Log("Max dist :" + max_dist);
Debug.Log("Min dist :" + min_dist);
List <DMatch> matchesGoodList = new List <DMatch>();
MatOfDMatch matchesGood = new MatOfDMatch();
matchesGood.fromList(matchesGoodList);
//Draw Keypoints
Features2d.drawKeypoints(SrcMat, keypointsSrc, SrcMat);
List <Point> pTarget = new List <Point>();
List <Point> pSrc = new List <Point>();
Debug.Log("MatchCount" + matchesGoodList.Count);
for (int i = 0; i < matchesGoodList.Count; i++)
{
pTarget.Add(new Point(keypointsTarget.toArray()[matchesGoodList[i].queryIdx].pt.x, keypointsTarget.toArray()[matchesGoodList[i].queryIdx].pt.y));
pSrc.Add(new Point(keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.x, keypointsSrc.toArray()[matchesGoodList[i].trainIdx].pt.y));
}
MatOfPoint2f p2fTarget = new MatOfPoint2f(pTarget.ToArray());
MatOfPoint2f p2fSrc = new MatOfPoint2f(pSrc.ToArray());
Mat matrixH = Calib3d.findHomography(p2fTarget, p2fSrc, Calib3d.RANSAC, 3);
List <Point> srcPointCorners = new List <Point>();
srcPointCorners.Add(new Point(0, 0));
srcPointCorners.Add(new Point(targetMat.width(), 0));
srcPointCorners.Add(new Point(targetMat.width(), targetMat.height()));
srcPointCorners.Add(new Point(0, targetMat.height()));
Mat originalRect = Converters.vector_Point2f_to_Mat(srcPointCorners);
List <Point> srcPointCornersEnd = new List <Point>();
srcPointCornersEnd.Add(new Point(0, targetMat.height()));
srcPointCornersEnd.Add(new Point(0, 0));
srcPointCornersEnd.Add(new Point(targetMat.width(), 0));
srcPointCornersEnd.Add(new Point(targetMat.width(), targetMat.height()));
Mat changeRect = Converters.vector_Point2f_to_Mat(srcPointCornersEnd);
Core.perspectiveTransform(originalRect, changeRect, matrixH);
List <Point> srcPointCornersSave = new List <Point>();
Converters.Mat_to_vector_Point(changeRect, srcPointCornersSave);
if ((srcPointCornersSave[2].x - srcPointCornersSave[0].x) < 5 || (srcPointCornersSave[2].y - srcPointCornersSave[0].y) < 5)
{
Debug.Log("Match Out Put image is to small");
SrcMat.copyTo(cameraFeed);
SrcMat.release();
Imgproc.putText(cameraFeed, targetName, srcPointCornersSave[0], 0, 1, new Scalar(255, 255, 255), 2);
return(false);
}
//畫出框框
Imgproc.line(SrcMat, srcPointCornersSave[0], srcPointCornersSave[1], new Scalar(255, 0, 0), 3);
Imgproc.line(SrcMat, srcPointCornersSave[1], srcPointCornersSave[2], new Scalar(255, 0, 0), 3);
Imgproc.line(SrcMat, srcPointCornersSave[2], srcPointCornersSave[3], new Scalar(255, 0, 0), 3);
Imgproc.line(SrcMat, srcPointCornersSave[3], srcPointCornersSave[0], new Scalar(255, 0, 0), 3);
//畫中心
Point middlePoint = new Point((srcPointCornersSave[0].x + srcPointCornersSave[2].x) / 2, (srcPointCornersSave[0].y + srcPointCornersSave[2].y) / 2);
Imgproc.line(SrcMat, middlePoint, middlePoint, new Scalar(0, 0, 255), 10);
SrcMat.copyTo(cameraFeed);
keypointsTarget.release();
targetMat.release();
SrcMat.release();
return(true);
}
void Update()
{
// Camera image from Vuforia
Image camImg = CameraDevice.Instance.GetCameraImage(PIXEL_FORMAT.RGBA8888);
if (camImg != null && camImg.Height > 0)
{
if (camImageMat == null)
{
// Vuforia seems to enforce a resolution of width=640px for any camera
Debug.Log("rows: " + camImg.Height + ", cols: " + camImg.Width);
camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4);
}
// Put Vuforia camera feed pixels into OpenCV display matrix
camImageMat.put(0, 0, camImg.Pixels);
// DEBUG TEST: In OpenCV, we operate in screen coordinates (pixels),
// and we know the resolution of the Vuforia camera
// Here, we draw a red circle in screen space using OpenCV
//Imgproc.circle(camImageMat, new Point(300, 200), 20, new Scalar(255, 0, 0, 128));
//---- <THIS IS WHERE THE CORNER PROJECTION BEGINS> ----
// Get corner's position in world coordinates
Matrix4x4 m1 = corner1.transform.localToWorldMatrix;
Matrix4x4 m2 = corner2.transform.localToWorldMatrix;
Matrix4x4 m3 = corner3.transform.localToWorldMatrix;
Matrix4x4 m4 = corner4.transform.localToWorldMatrix;
Vector3 worldPnt1 = m1.MultiplyPoint3x4(corner1.transform.position);
Vector3 worldPnt2 = m2.MultiplyPoint3x4(corner2.transform.position);
Vector3 worldPnt3 = m3.MultiplyPoint3x4(corner3.transform.position);
Vector3 worldPnt4 = m4.MultiplyPoint3x4(corner4.transform.position);
// Matrix that goes from world to the camera coordinate system
Matrix4x4 Rt = cam.transform.worldToLocalMatrix;
// Camera intrinsics
Matrix4x4 A = Matrix4x4.identity;
A.m00 = fx;
A.m11 = fy;
A.m02 = cx;
A.m12 = cy;
//see cheat sheet
Matrix4x4 worldToImage = A * Rt;
Vector3 hUV1 = worldToImage.MultiplyPoint3x4(worldPnt1);
Vector3 hUV2 = worldToImage.MultiplyPoint3x4(worldPnt2);
Vector3 hUV3 = worldToImage.MultiplyPoint3x4(worldPnt3);
Vector3 hUV4 = worldToImage.MultiplyPoint3x4(worldPnt4);
// Remember that we dealing with homogeneous coordinates.
// Here we normalize them to get Image coordinates
Vector2 uv1 = new Vector2(hUV1.x, hUV1.y) / hUV1.z;
Vector2 uv2 = new Vector2(hUV2.x, hUV2.y) / hUV2.z;
Vector2 uv3 = new Vector2(hUV3.x, hUV3.y) / hUV3.z;
Vector2 uv4 = new Vector2(hUV4.x, hUV4.y) / hUV4.z;
// We flip the v-coordinate of our image points to make the Unity (Vuforia) data compatible with OpenCV
// Remember that in OpenCV the (0,0) pos is in the top left corner in contrast to the bottom left corner
float maxV = camImg.Height - 1; // The -1 is because pixel coordinates are 0-indexed
imagePoints.put(0, 0, uv1.x, maxV - uv1.y);
imagePoints.put(1, 0, uv2.x, maxV - uv2.y);
imagePoints.put(2, 0, uv3.x, maxV - uv3.y);
imagePoints.put(3, 0, uv4.x, maxV - uv4.y);
Point imgPnt1 = new Point(imagePoints.get(0, 0));
Point imgPnt2 = new Point(imagePoints.get(1, 0));
Point imgPnt3 = new Point(imagePoints.get(2, 0));
Point imgPnt4 = new Point(imagePoints.get(3, 0));
//For debug. Show if impPnti found the right position in img coordinate
Imgproc.circle(camImageMat, imgPnt1, 10, new Scalar(255, 0, 0, 200), 5);
Imgproc.circle(camImageMat, imgPnt2, 20, new Scalar(255, 255, 0, 255), 5);
Imgproc.circle(camImageMat, imgPnt3, 30, new Scalar(0, 255, 0, 255), 5);
Imgproc.circle(camImageMat, imgPnt4, 40, new Scalar(0, 0, 255, 255), 4);
MatOfPoint2f unwarpPoints;
unwarpPoints = new MatOfPoint2f();
unwarpPoints.alloc(4);
//according to the resolution
unwarpPoints.put(0, 0, 0, 0);
unwarpPoints.put(1, 0, 0, 442);
unwarpPoints.put(2, 0, 442, 442);
unwarpPoints.put(3, 0, 442, 0);
//compute homography matrix
Mat H = Calib3d.findHomography(imagePoints, unwarpPoints);
Mat Hinv = H.inv();
Mat dst = new Mat(442, 442, CvType.CV_8UC4);
texMat = MatDisplay.LoadRGBATexture("/models/dog_tex.png");
Imgproc.warpPerspective(texMat, dst, Hinv, new Size(442, 442));
// MatDisplay.MatToTexture(dst, ref tex);
//rd.material.mainTexture = tex;
//Debug.Log(imgPnt2);
//Debug.Log(imgPnt2);
//---- </THIS IS WHERE THE CORNER PROJECTION ENDS> ----
// Display the Mat that includes video feed and debug points
// Do not forget to disable Vuforia's video background and change your aspect ratio to 4:3!
MatDisplay.DisplayMat(camImageMat, MatDisplaySettings.FULL_BACKGROUND);
//---- MATCH INTRINSICS OF REAL CAMERA AND PROJECTION MATRIX OF VIRTUAL CAMERA ----
// See lecture slides for why this formular works.
cam.fieldOfView = 2 * Mathf.Atan(camImg.Height * 0.5f / fy) * Mathf.Rad2Deg;
}
}
IEnumerator ProcessImage(XRCpuImage image, Vector3 viewportScaling)
{
// Create the async conversion request.
XRCpuImage.ConversionParams conv_params = new XRCpuImage.ConversionParams
{
// Use the full image.
inputRect = new RectInt(0, 0, image.width, image.height),
// Downsample by 2.
outputDimensions = new Vector2Int(image.width, image.height),
// Color image format.
outputFormat = TextureFormat.RGBA32,
// Flip across the Y axis.
transformation = XRCpuImage.Transformation.MirrorY
};
var request = image.ConvertAsync(conv_params);
// Wait for the conversion to complete.
while (!request.status.IsDone())
{
yield return(null);
}
// Check status to see if the conversion completed successfully.
if (request.status != XRCpuImage.AsyncConversionStatus.Ready)
{
// Something went wrong.
Debug.LogErrorFormat("Request failed with status {0}", request.status);
// Dispose even if there is an error.
request.Dispose();
yield break;
}
// Image data is ready. Let's apply it to a Texture2D.
var rawData = request.GetData <byte>();
// Create a texture if necessary.
if (m_Texture == null)
{
m_Texture = new Texture2D(
request.conversionParams.outputDimensions.x,
request.conversionParams.outputDimensions.y,
request.conversionParams.outputFormat,
false);
}
// Copy the image data into the texture.
m_Texture.LoadRawTextureData(rawData);
m_Texture.Apply();
Debug.Log("TEX: " + m_Texture.height + "h " + m_Texture.width + "w");
Debug.Log("Screen: " + m_Texture.height + "h " + m_Texture.width + "w");
Mat inputMat = new Mat(image.height, image.width, CvType.CV_8UC4);
Mat outputMat = new Mat(1500, 1500, CvType.CV_8UC4);
Utils.fastTexture2DToMat(m_Texture, inputMat);
if (tex2d == null)
{
tex2d = new Texture2D(1500,
1500, conv_params.outputFormat, false);
}
Debug.Log("positionAnchor");
Debug.Log(positionAnchor);
Debug.Log("anchorRef");
Debug.Log(anchorRef);
int counter = 0;
Point[] srcPointsVec = new Point[4];
foreach (var point in anchorRef.getWorldPoints())
{
Vector3 screenPoint = mainCam.WorldToScreenPoint(point);
srcPointsVec[counter] = new Point(screenPoint.y * viewportScaling.y / 3,
100 - screenPoint.x * viewportScaling.x / 3);
counter += 1;
}
MatOfPoint2f srcPoints = new MatOfPoint2f(new[]
{
srcPointsVec[0],
srcPointsVec[1],
srcPointsVec[2],
srcPointsVec[3]
});
MatOfPoint2f dstPoints = new MatOfPoint2f(new[]
{
new Point(195 * 1.25, 0),
new Point(0, 0),
new Point(0, 280 * 1.25),
new Point(195 * 1.25, 280 * 1.25),
});
Mat H = Calib3d.findHomography(srcPoints, dstPoints);
Imgproc.warpPerspective(inputMat, outputMat, H, new Size(1500, 1500));
Utils.fastMatToTexture2D(outputMat, tex2d);
if (websocket.State == WebSocketState.Open && canProcess)
{
websocket.Send(ImageConversion.EncodeToJPG(tex2d, 50));
canProcess = false;
}
inputMat.Dispose();
inputMat = null;
outputMat.Dispose();
outputMat = null;
request.Dispose();
}
