/// <summary>
/// called whenever a screenshot was taken by the screenshot manager
/// </summary>
/// <param name="sender"> the sender of the event </param>
/// <param name="e"> the photograph event parameters </param>
private void OnScreenshotTaken(object sender, EventArgs e)
{
// Save ref for latest picture taken
screenshotManager.GetLatestPicture(out imageAsTextureTmp, out cameraToWorldMatrixTmp, out projectionMatrixTmp);
cameraPositionResultTmp = new CameraPositionResult();
cameraPositionResultTmp.cameraToWorldMatrix = cameraToWorldMatrixTmp;
cameraPositionResultTmp.projectionMatrix = projectionMatrixTmp;
//start analyzing image
analysingScreenshot = true;
#if (!UNITY_EDITOR)
if (screenshotsTakeable)
{
//id
switch (currentRequestCause)
{
case RequestCause.REGULAR:
if (regularCameraPositionResultList.Count > 9)
{
// removing game object
Destroy(regularCameraPositionResultList[0].textHighlightObject);
regularCameraPositionResultList.RemoveAt(0);
}
cameraPositionResultTmp.id = currentId;
regularCameraPositionResultList.Insert(regularCameraPositionResultList.Count, cameraPositionResultTmp);
// Debug.Log("!!!!!RegularLIST Count" + regularCameraPositionResultList.Count);
apiManager.AnalyzeImageAsync(RequestType.LOCAL, new Picture(imageAsTextureTmp));
break;
case RequestCause.USERINITIATED:
screenshotsTakeable = false;
this.screenshotManager._screenshotsTakeable = false;
if (initiatedCameraPositionResultList.Count > 9)
{
// removing game object
Destroy(initiatedCameraPositionResultList[0].visualizedTextObject);
initiatedCameraPositionResultList.RemoveAt(0);
}
cameraPositionResultTmp.id = currentId;
initiatedCameraPositionResultList.Insert(initiatedCameraPositionResultList.Count, cameraPositionResultTmp);
// Debug.Log("!!!!!InitiatedLIST Count" + initiatedCameraPositionResultList.Count);
apiManager.AnalyzeImageAsync(RequestType.REMOTE, new Picture(imageAsTextureTmp));
break;
}
}
#endif
}
/// <summary>
/// called when the application is started
/// </summary>
void Start()
{
// link managers
apiManager = ApiManager.Instance;
screenshotManager = ScreenshotManager.Instance;
visualTextManager = VisualTextManager.Instance;
gesturesManager = GesturesManager.Instance;
// Link IconObjects, deactivate animation
loadingIcon = Instantiate(loadingIcon);
loadingIcon.SetActive(false);
cursor = GameObject.Find("CursorVisual");
cursorSize = cursor.transform.localScale;
// subscribe to events
screenshotManager.ScreenshotTaken += OnScreenshotTaken;
apiManager.ImageAnalysed += onImageAnalysed;
gesturesManager.Tapped += onTapped;
gesturesManager.DoubleTapped += onDoubleTapped;
visualTextManager.VisualizedTextFocused += OnVisualizedTextFocused;
visualTextManager.VisualizedTextUnfocused += OnVisualizedTextUnfocused;
analysingScreenshot = false;
timeCounter = 0;
timeInterval = 1;
currentRequestCause = RequestCause.REGULAR;
nextRequestCause = RequestCause.REGULAR;
screenshotsTakeable = true;
cameraPositionResultTmp = new CameraPositionResult();
currentId = 0;
visualizedTextFocused = false;
focusedVisualizedTextTmp = "";
dialogPanel = DialogPanel.Instance();
}
internal void hightlightTextLocation(List <CameraPositionResult> regularCameraPositionResultList, CameraPositionResult cameraPositionResult)
{
float ImageWidth = 896; // Camera.main.pixelWidth;
float ImageHeight = 504; // Camera.main.pixelHeight;
var ocrResult = cameraPositionResult.ocrResult; //new OcrResult("hi", new Rect(ImageWidth / 2, ImageHeight / 2, 0, 0));
var headPosition = Camera.main.transform.position;
float textX = ocrResult.BoundingBox.x;
float textY = ocrResult.BoundingBox.y;
float textWidth = ocrResult.BoundingBox.width;
float textHeight = ocrResult.BoundingBox.height;
var gazeDirection = Camera.main.transform.forward;
//Debug.Log(ocrResult.BoundingBox);
//Debug.Log("textWidth: " + textWidth + "; textHeight: " + textHeight + "; camHeight: " + ImageHeight + "; camWidtht: " + ImageWidth);
Vector3[] WorldSpaceCenter = convert2DtoWorld(textX, textY, ImageWidth, ImageHeight, cameraPositionResult.cameraToWorldMatrix, cameraPositionResult.projectionMatrix);
RaycastHit hitCenter;
if (Physics.Raycast(WorldSpaceCenter[0], WorldSpaceCenter[1], out hitCenter))
{
// Debug.Log("Raycasts hit!");
// Lisa:
for (int i = 0; i < regularCameraPositionResultList.Count; i++)
{
if (regularCameraPositionResultList[i].textHighlightObject)
{
float vdistance = (hitCenter.point - regularCameraPositionResultList[i].textHighlightObject.transform.position).magnitude;
if (vdistance <= 0.4)
{
//Destroy(regularCameraPositionResultList[i].textHighlightObject);
//regularCameraPositionResultList.RemoveAt(i);
//Destroy(regularCameraPositionResultList[i].textHighlightObject);
regularCameraPositionResultList.RemoveAt(regularCameraPositionResultList.Count - 1);
return;
}
}
}
//Debug.Log("parent of dot" + textHighlight.);
GameObject newHighlight = Instantiate(textHighlight);
// saving ref to text highlight object for later
cameraPositionResult.textHighlightObject = newHighlight;
//set Position
newHighlight.transform.position = hitCenter.point;
//set Rotation
Quaternion toQuat = Camera.main.transform.localRotation;
toQuat.x = 0;
toQuat.z = 0;
newHighlight.transform.rotation = toQuat;
}
}
internal void visualizeText(List <CameraPositionResult> initiatedCameraPositionResultList, CameraPositionResult cameraPositionResult)
{
float ImageWidth = 896; // Camera.main.pixelWidth;
float ImageHeight = 504; // Camera.main.pixelHeight;
var ocrResult = cameraPositionResult.ocrResult; //new OcrResult("hi", new Rect(ImageWidth / 2, ImageHeight / 2, 0, 0));
var headPosition = Camera.main.transform.position;
float textX = ocrResult.BoundingBox.x;
float textY = ocrResult.BoundingBox.y;
float textWidth = ocrResult.BoundingBox.width;
float textHeight = ocrResult.BoundingBox.height;
var gazeDirection = Camera.main.transform.forward;
Debug.Log(ocrResult.BoundingBox);
// Debug.Log("textWidth: " + textWidth + "; textHeight: " + textHeight + "; camHeight: " + ImageHeight + "; camWidtht: " + ImageWidth);
Vector3[] WorldSpaceCenter = convert2DtoWorld(textX + (textWidth / 2), textY + (textHeight / 2), ImageWidth, ImageHeight, cameraPositionResult.cameraToWorldMatrix, cameraPositionResult.projectionMatrix);
Vector3[] WorldSpaceTopLeft = convert2DtoWorld(textX, textY, ImageWidth, ImageHeight, cameraPositionResult.cameraToWorldMatrix, cameraPositionResult.projectionMatrix);
Vector3[] WorldSpaceTopRight = convert2DtoWorld(textX + textWidth, textY, ImageWidth, ImageHeight, cameraPositionResult.cameraToWorldMatrix, cameraPositionResult.projectionMatrix);
Vector3[] WorldSpaceBotLeft = convert2DtoWorld(textX, textY + textHeight, ImageWidth, ImageHeight, cameraPositionResult.cameraToWorldMatrix, cameraPositionResult.projectionMatrix);
Vector3[] WorldSpaceBotRight = convert2DtoWorld(textX + textWidth, textY + textHeight, ImageWidth, ImageHeight, cameraPositionResult.cameraToWorldMatrix, cameraPositionResult.projectionMatrix);
var CameraToWorld = cameraPositionResult.cameraToWorldMatrix;
var Projection = cameraPositionResult.projectionMatrix;
RaycastHit hitCenter, hitTopLeft, hitTopRight, hitBotLeft, hitBotRight, hitGaze;
if (Physics.Raycast(WorldSpaceCenter[0], WorldSpaceCenter[1], out hitCenter) && Physics.Raycast(WorldSpaceTopLeft[0], WorldSpaceTopLeft[1], out hitTopLeft) && Physics.Raycast(WorldSpaceBotLeft[0], WorldSpaceTopRight[1], out hitTopRight) && Physics.Raycast(WorldSpaceBotLeft[0], WorldSpaceBotLeft[1], out hitBotLeft) && Physics.Raycast(WorldSpaceBotRight[0], WorldSpaceBotRight[1], out hitBotRight) && Physics.Raycast(headPosition, gazeDirection, out hitGaze))
// if (Physics.Raycast(headPosition, WorldSpaceCenter[1], out hitCenter) && Physics.Raycast(headPosition, gazeDirection, out hitGaze))
{
//Debug.Log("Raycasts hit!");
// Lisa:
for (int i = 0; i < initiatedCameraPositionResultList.Count; i++)
{
if (initiatedCameraPositionResultList[i].visualizedTextObject)
{
float vdistance = (hitCenter.point - initiatedCameraPositionResultList[i].visualizedTextObject.transform.position).magnitude;
if (vdistance <= 0.4)
{
Destroy(initiatedCameraPositionResultList[i].visualizedTextObject);
initiatedCameraPositionResultList.RemoveAt(i);
}
}
}
GameObject newArea = Instantiate(visualTextCanvas);
// saving ref to created game object for later
cameraPositionResult.visualizedTextObject = newArea;
// Text visualText = newArea.transform.Find("Text").gameObject.GetComponent<Text>();
//set Text
newArea.transform.Find("Text").gameObject.GetComponent <Text>().text = ocrResult.Text;
//set Position
newArea.transform.position = hitCenter.point;
//opposite direction
Vector3 opposite = (WorldSpaceCenter[1] * -0.5f).normalized;
//newArea.transform.Translate(opposite);
//set Rotation
Quaternion toQuat = Camera.main.transform.localRotation;
toQuat.x = 0;
toQuat.z = 0;
newArea.transform.rotation = toQuat;
//set Size
//desired size
float distance = (hitCenter.point - headPosition).magnitude;;
float targetWidth = (hitTopLeft.point - hitTopRight.point).magnitude;
float targetHeight = (hitTopLeft.point - hitBotLeft.point).magnitude;
//current size
Vector3 oldScale = newArea.transform.localScale;
RectTransform textBox = newArea.transform.GetComponent <RectTransform>();
float currentWidth = oldScale.x * textBox.rect.width;
float currentHeight = oldScale.y * textBox.rect.height;
float scaleWidth = targetWidth / currentWidth;
float scaleHeight = targetHeight / currentHeight;
Vector3 newScale = new Vector3(scaleWidth * oldScale.x, scaleHeight * oldScale.y, oldScale.z);
newArea.transform.localScale = newScale;
}
}
