void UpdateCirclePatternSize()
{
if (_state == State.Initiating || _state == State.BlindCalibration)
{
_circlePatternSize = defaultCirclesPatternSize;
}
else
{
const int circlePatternSizeYMin = 7;
const int desiredPixelsPerCirclePatternSegment = 25; // 50px is recommended, but for a 720p camera, this will give too fex dots.
float desiredCirclePatternNumAspect = _chessPatternSize.x / (_chessPatternSize.y) / 2f; // 3f / 4f / 2f;
float patternDistance = Vector3.Distance(_mainCamera.transform.position, _circlePatternTransform.position);
float patternHeight = _chessPatternTransform.localScale.y;
float viewHeightAtPatternPosition = Mathf.Tan(_mainCamera.fieldOfView * Mathf.Deg2Rad * 0.5f) * patternDistance * 2;
int circlePatternPixelHeight = (int)((patternHeight / viewHeightAtPatternPosition) * _cameraTexture.height);
int optimalPatternSizeY = Mathf.Max(circlePatternSizeYMin, Mathf.FloorToInt(circlePatternPixelHeight / (float)desiredPixelsPerCirclePatternSegment));
int optimalPatternSizeX = Mathf.FloorToInt(optimalPatternSizeY * desiredCirclePatternNumAspect);
_circlePatternSize = TrackingToolsHelper.GetClosestValidPatternSize(new Vector2Int(optimalPatternSizeX, optimalPatternSizeY), TrackingToolsHelper.PatternType.AsymmetricCircleGrid);
}
_circlePatternPointCount = _circlePatternSize.x * _circlePatternSize.y;
if (_circlePointsProjectorRenderImageMat != null && _circlePointsProjectorRenderImageMat.rows() == _circlePatternPointCount)
{
return;
}
if (_circlePointsProjectorRenderImageMat != null)
{
_circlePointsProjectorRenderImageMat.release();
}
if (_circlePointsRealModelMat != null)
{
_circlePointsRealModelMat.release();
}
if (_circlePointsDetectedWorldMat != null)
{
_circlePointsDetectedWorldMat.release();
}
_circlePointsProjectorRenderImageMat.alloc(_circlePatternPointCount);
_circlePointsRealModelMat.alloc(_circlePatternPointCount);
_circlePointsDetectedWorldMat.alloc(_circlePatternPointCount);
// Render pattern to texture.
_circlePatternBorderSizeUV = TrackingToolsHelper.RenderPattern(_circlePatternSize, TrackingToolsHelper.PatternType.AsymmetricCircleGrid, 2048, ref _circlePatternTexture, ref _patternRenderMaterial, circlePatternBorder, true);
_circlePatternBoardMaterial.mainTexture = _circlePatternTexture;
// Update transform to match.
float circleTextureAspect = _circlePatternTexture.width / (float)_circlePatternTexture.height;
float borderProportion = (_circlePatternSize.y - 1 + 2f) / (_circlePatternSize.y - 1f); // Asymmetric patttern tiles are half the height.
_circlePatternTransform.localScale = new Vector3(circleTextureAspect, 1, 0) * _chessPatternTransform.localScale.y * borderProportion;
if (_state == State.TrackedCalibration || _state == State.Testing)
{
_circlePatternTransform.localPosition = -Vector3.right * (_chessCirclePatternCenterOffset / 1000f);
}
}
// Start is called before the first frame update
void Start()
{
//Anders magic number
_drawingPlaceMat = new Mat(100, 150, CvType.CV_8UC4);
_drawingPlaceMat.setTo(new Scalar(255, 255, 255));
_imagePoints = new MatOfPoint2f();
_imagePoints.alloc(4);
}
void Start()
{
fishRd = GameObject.Find("fish").GetComponents <SkinnedMeshRenderer>()[0];
// OpenCV matrices need to be allocated first to indicate size
imagePoints = new MatOfPoint2f();
imagePoints.alloc(4);
/*
* corner1 = GameObject.Find("corner1");
* corner2 = GameObject.Find("corner2");
* corner3 = GameObject.Find("corner2");
* corner4 = GameObject.Find("corner2");
*/
}
void Start()
{
fish = GameObject.Find("Fish");
corner1 = GameObject.Find("Corner1");
corner2 = GameObject.Find("Corner2");
corner3 = GameObject.Find("Corner3");
corner4 = GameObject.Find("Corner4");
imagePoints = new MatOfPoint2f();
imagePoints.alloc(4);
tex = new Texture2D(2, 2);
fileData = File.ReadAllBytes("Assets/_Templates/assignment2b/fish_tex.png");
tex.LoadImage(fileData);
unwarpedTextureClean = new Texture2D(width, height, TextureFormat.RGBA32, false);
}
//private Renderer rd;
//public Texture2D tex;
void Start()
{
texMat = MatDisplay.LoadRGBATexture("/models/dog_tex.png");
// OpenCV matrices need to be allocated first to indicate size
imagePoints = new MatOfPoint2f();
imagePoints.alloc(4);
// rd = GetComponents<SkinnedMeshRenderer>()[0];
/*
* corner1 = GameObject.Find("corner1");
* corner2 = GameObject.Find("corner2");
* corner3 = GameObject.Find("corner2");
* corner4 = GameObject.Find("corner2");
*/
}
void Update()
{
//Access camera image provided by Vuforia
Image camImg = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888);
if (camImg != null)
{
if (camImageMat == null)
{
//First time -> instantiate camera image specific data
camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4); //Note: rows=height, cols=width
}
camImageMat.put(0, 0, camImg.Pixels);
//Replace with your own projection matrix. This approach only uses fy.
cam.fieldOfView = 2 * Mathf.Atan(camImg.Height * 0.5f / fy) * Mathf.Rad2Deg;
Vector3 worldPnt1 = corner1.transform.position;
Vector3 worldPnt2 = corner2.transform.position;
Vector3 worldPnt3 = corner3.transform.position;
Vector3 worldPnt4 = corner4.transform.position;
//See lecture slides
Matrix4x4 Rt = cam.transform.worldToLocalMatrix;
Matrix4x4 A = Matrix4x4.identity;
A.m00 = fx;
A.m11 = fy;
A.m02 = cx;
A.m12 = cy;
Matrix4x4 worldToImage = A * Rt;
Vector3 hUV1 = worldToImage.MultiplyPoint3x4(worldPnt1);
Vector3 hUV2 = worldToImage.MultiplyPoint3x4(worldPnt2);
Vector3 hUV3 = worldToImage.MultiplyPoint3x4(worldPnt3);
Vector3 hUV4 = worldToImage.MultiplyPoint3x4(worldPnt4);
//hUV are the image coordinates in 2D homogeneous coordinates, we need to normalize, i.e., divide by Z
Vector2 uv1 = new Vector2(hUV1.x, hUV1.y) / hUV1.z;
Vector2 uv2 = new Vector2(hUV2.x, hUV2.y) / hUV2.z;
Vector2 uv3 = new Vector2(hUV3.x, hUV3.y) / hUV3.z;
Vector2 uv4 = new Vector2(hUV4.x, hUV4.y) / hUV4.z;
//don't forget to alloc before putting values into a MatOfPoint2f
imagePoints.put(0, 0, uv1.x, camImg.Height - uv1.y);
imagePoints.put(1, 0, uv2.x, camImg.Height - uv2.y);
imagePoints.put(2, 0, uv3.x, camImg.Height - uv3.y);
imagePoints.put(3, 0, uv4.x, camImg.Height - uv4.y);
//Debug draw points
Point imgPnt1 = new Point(imagePoints.get(0, 0));
Point imgPnt2 = new Point(imagePoints.get(1, 0));
Point imgPnt3 = new Point(imagePoints.get(2, 0));
Point imgPnt4 = new Point(imagePoints.get(3, 0));
Imgproc.circle(camImageMat, imgPnt1, 5, new Scalar(255, 0, 0, 255));
Imgproc.circle(camImageMat, imgPnt2, 5, new Scalar(0, 255, 0, 255));
Imgproc.circle(camImageMat, imgPnt3, 5, new Scalar(0, 0, 255, 255));
Imgproc.circle(camImageMat, imgPnt4, 5, new Scalar(255, 255, 0, 255));
Scalar lineCl = new Scalar(200, 120, 0, 160);
Imgproc.line(camImageMat, imgPnt1, imgPnt2, lineCl);
Imgproc.line(camImageMat, imgPnt2, imgPnt3, lineCl);
Imgproc.line(camImageMat, imgPnt3, imgPnt4, lineCl);
Imgproc.line(camImageMat, imgPnt4, imgPnt1, lineCl);
var destPoints = new MatOfPoint2f(); // Creating a destination
destPoints.alloc(4);
destPoints.put(0, 0, width, 0);
destPoints.put(1, 0, width, height);
destPoints.put(2, 0, 0, height);
destPoints.put(3, 0, 0, 0);
var homography = Calib3d.findHomography(imagePoints, destPoints); // Finding the image
Imgproc.warpPerspective(camImageMat, destPoints, homography, new Size(camImageMat.width(), camImageMat.height()));
unwarpedTexture = unwarpedTextureClean;
MatDisplay.MatToTexture(destPoints, ref unwarpedTexture); // Take output and transform into texture
if (Input.GetKey("space"))
{
fish.GetComponent <Renderer>().material.mainTexture = unwarpedTexture;
}
else
{
fish.GetComponent <Renderer>().material.mainTexture = tex;
}
MatDisplay.DisplayMat(destPoints, MatDisplaySettings.BOTTOM_LEFT);
MatDisplay.DisplayMat(camImageMat, MatDisplaySettings.FULL_BACKGROUND);
}
}
void Awake()
{
// Create UI.
if (!_containerUI)
{
_containerUI = new GameObject("CameraPoser").AddComponent <RectTransform>();
_containerUI.transform.SetParent(_canvas.transform);
}
CanvasGroup wrapperGroup = _containerUI.GetComponent <CanvasGroup>();
if (!wrapperGroup)
{
wrapperGroup = _containerUI.gameObject.AddComponent <CanvasGroup>();
}
wrapperGroup.alpha = _alpha;
Image backgroundImage = new GameObject("Background").AddComponent <Image>();
backgroundImage.transform.SetParent(_containerUI.transform);
_rawImageUI = new GameObject("CameraImage").AddComponent <RawImage>();
_rawImageUI.transform.SetParent(_containerUI.transform);
_rawImageUI.uvRect = _flipTexture ? new UnityEngine.Rect(0, 1, 1, -1) : new UnityEngine.Rect(0, 0, 1, 1);
_rawImageRect = _rawImageUI.GetComponent <RectTransform>();
_uiMaterial = new Material(Shader.Find("Hidden/SingleChannelTexture"));
_rawImageUI.material = _uiMaterial;
_aspectFitterUI = _rawImageUI.gameObject.AddComponent <AspectRatioFitter>();
_aspectFitterUI.aspectMode = AspectRatioFitter.AspectMode.HeightControlsWidth;
backgroundImage.color = Color.black;
ExpandRectTransform(_containerUI);
ExpandRectTransform(backgroundImage.GetComponent <RectTransform>());
ExpandRectTransform(_rawImageRect);
_userPointRects = new RectTransform[pointCount];
_userPointImages = new Image[pointCount];
for (int p = 0; p < pointCount; p++)
{
GameObject pointObject = new GameObject("Point" + p);
pointObject.transform.SetParent(_rawImageRect);
Image pointImage = pointObject.AddComponent <Image>();
pointImage.color = Color.cyan;
RectTransform pointRect = pointObject.GetComponent <RectTransform>();
pointRect.sizeDelta = Vector2.one * 5;
SetAnchoredPosition(pointRect, defaultPointPositions[p]);
pointRect.anchoredPosition = Vector3.zero;
Text pointLabel = new GameObject("Label").AddComponent <Text>();
pointLabel.text = p.ToString();
pointLabel.transform.SetParent(pointRect);
pointLabel.rectTransform.anchoredPosition = Vector2.zero;
pointLabel.rectTransform.sizeDelta = new Vector2(_fontSize, _fontSize) * 2;
pointLabel.font = _font;
pointLabel.fontSize = _fontSize;
_userPointRects[p] = pointRect;
_userPointImages[p] = pointImage;
}
// Hide.
//if( !_interactable ) _containerUI.transform.gameObject.SetActive( false );
// Prepare OpenCV.
_noDistCoeffs = new MatOfDouble(new double[] { 0, 0, 0, 0 });
_rVec = new Mat();
_tVec = new Mat();
_anchorPointsImage = new Point[pointCount];
_anchorPointsWorld = new Point3[pointCount];
_anchorPointsImageMat = new MatOfPoint2f();
_anchorPointsWorldMat = new MatOfPoint3f();
_anchorPointsImageMat.alloc(pointCount);
_anchorPointsWorldMat.alloc(pointCount);
for (int p = 0; p < pointCount; p++)
{
_anchorPointsImage[p] = new Point();
_anchorPointsWorld[p] = new Point3();
}
// Load files.
if (Intrinsics.TryLoadFromFile(_intrinsicsFileName, out _intrinsics))
{
enabled = false;
return;
}
LoadCircleAnchorPoints();
// Update variables.
if (!Application.isEditor)
{
OnValidate();
}
}
void Update()
{
// Camera image from Vuforia
Image camImg = CameraDevice.Instance.GetCameraImage(PIXEL_FORMAT.RGBA8888);
if (camImg != null && camImg.Height > 0)
{
if (camImageMat == null)
{
// Vuforia seems to enforce a resolution of width=640px for any camera
Debug.Log("rows: " + camImg.Height + ", cols: " + camImg.Width);
camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4);
}
// Put Vuforia camera feed pixels into OpenCV display matrix
camImageMat.put(0, 0, camImg.Pixels);
// DEBUG TEST: In OpenCV, we operate in screen coordinates (pixels),
// and we know the resolution of the Vuforia camera
// Here, we draw a red circle in screen space using OpenCV
//Imgproc.circle(camImageMat, new Point(300, 200), 20, new Scalar(255, 0, 0, 128));
//---- <THIS IS WHERE THE CORNER PROJECTION BEGINS> ----
// Get corner's position in world coordinates
Matrix4x4 m1 = corner1.transform.localToWorldMatrix;
Matrix4x4 m2 = corner2.transform.localToWorldMatrix;
Matrix4x4 m3 = corner3.transform.localToWorldMatrix;
Matrix4x4 m4 = corner4.transform.localToWorldMatrix;
Vector3 worldPnt1 = m1.MultiplyPoint3x4(corner1.transform.position);
Vector3 worldPnt2 = m2.MultiplyPoint3x4(corner2.transform.position);
Vector3 worldPnt3 = m3.MultiplyPoint3x4(corner3.transform.position);
Vector3 worldPnt4 = m4.MultiplyPoint3x4(corner4.transform.position);
// Matrix that goes from world to the camera coordinate system
Matrix4x4 Rt = cam.transform.worldToLocalMatrix;
// Camera intrinsics
Matrix4x4 A = Matrix4x4.identity;
A.m00 = fx;
A.m11 = fy;
A.m02 = cx;
A.m12 = cy;
//see cheat sheet
Matrix4x4 worldToImage = A * Rt;
Vector3 hUV1 = worldToImage.MultiplyPoint3x4(worldPnt1);
Vector3 hUV2 = worldToImage.MultiplyPoint3x4(worldPnt2);
Vector3 hUV3 = worldToImage.MultiplyPoint3x4(worldPnt3);
Vector3 hUV4 = worldToImage.MultiplyPoint3x4(worldPnt4);
// Remember that we dealing with homogeneous coordinates.
// Here we normalize them to get Image coordinates
Vector2 uv1 = new Vector2(hUV1.x, hUV1.y) / hUV1.z;
Vector2 uv2 = new Vector2(hUV2.x, hUV2.y) / hUV2.z;
Vector2 uv3 = new Vector2(hUV3.x, hUV3.y) / hUV3.z;
Vector2 uv4 = new Vector2(hUV4.x, hUV4.y) / hUV4.z;
// We flip the v-coordinate of our image points to make the Unity (Vuforia) data compatible with OpenCV
// Remember that in OpenCV the (0,0) pos is in the top left corner in contrast to the bottom left corner
float maxV = camImg.Height - 1; // The -1 is because pixel coordinates are 0-indexed
imagePoints.put(0, 0, uv1.x, maxV - uv1.y);
imagePoints.put(1, 0, uv2.x, maxV - uv2.y);
imagePoints.put(2, 0, uv3.x, maxV - uv3.y);
imagePoints.put(3, 0, uv4.x, maxV - uv4.y);
Point imgPnt1 = new Point(imagePoints.get(0, 0));
Point imgPnt2 = new Point(imagePoints.get(1, 0));
Point imgPnt3 = new Point(imagePoints.get(2, 0));
Point imgPnt4 = new Point(imagePoints.get(3, 0));
//For debug. Show if impPnti found the right position in img coordinate
Imgproc.circle(camImageMat, imgPnt1, 10, new Scalar(255, 0, 0, 200), 5);
Imgproc.circle(camImageMat, imgPnt2, 20, new Scalar(255, 255, 0, 255), 5);
Imgproc.circle(camImageMat, imgPnt3, 30, new Scalar(0, 255, 0, 255), 5);
Imgproc.circle(camImageMat, imgPnt4, 40, new Scalar(0, 0, 255, 255), 4);
MatOfPoint2f unwarpPoints;
unwarpPoints = new MatOfPoint2f();
unwarpPoints.alloc(4);
//according to the resolution
unwarpPoints.put(0, 0, 0, 0);
unwarpPoints.put(1, 0, 0, 442);
unwarpPoints.put(2, 0, 442, 442);
unwarpPoints.put(3, 0, 442, 0);
//compute homography matrix
Mat H = Calib3d.findHomography(imagePoints, unwarpPoints);
Mat Hinv = H.inv();
Mat dst = new Mat(442, 442, CvType.CV_8UC4);
texMat = MatDisplay.LoadRGBATexture("/models/dog_tex.png");
Imgproc.warpPerspective(texMat, dst, Hinv, new Size(442, 442));
// MatDisplay.MatToTexture(dst, ref tex);
//rd.material.mainTexture = tex;
//Debug.Log(imgPnt2);
//Debug.Log(imgPnt2);
//---- </THIS IS WHERE THE CORNER PROJECTION ENDS> ----
// Display the Mat that includes video feed and debug points
// Do not forget to disable Vuforia's video background and change your aspect ratio to 4:3!
MatDisplay.DisplayMat(camImageMat, MatDisplaySettings.FULL_BACKGROUND);
//---- MATCH INTRINSICS OF REAL CAMERA AND PROJECTION MATRIX OF VIRTUAL CAMERA ----
// See lecture slides for why this formular works.
cam.fieldOfView = 2 * Mathf.Atan(camImg.Height * 0.5f / fy) * Mathf.Rad2Deg;
}
}
