/// <summary> /// Call the Computer Vision Service to submit the image. /// </summary> public IEnumerator AnalyseLastImageCaptured(string imagePath) { Debug.Log("Analyzing..."); ObjectRecognitionManager.Instance.PrimaryText = "Analyzing . . ."; ObjectRecognitionManager.Instance.Update_DebugDisplay(); WWWForm webForm = new WWWForm(); using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(predictionEndpoint, webForm)) { // Gets a byte array out of the saved image imageBytes = GetImageAsByteArray(imagePath); unityWebRequest.SetRequestHeader("Content-Type", "application/octet-stream"); unityWebRequest.SetRequestHeader("Prediction-Key", predictionKey); // The upload handler will help uploading the byte array with the request unityWebRequest.uploadHandler = new UploadHandlerRaw(imageBytes); unityWebRequest.uploadHandler.contentType = "application/octet-stream"; // The download handler will help receiving the analysis from Azure unityWebRequest.downloadHandler = new DownloadHandlerBuffer(); // Send the request yield return(unityWebRequest.SendWebRequest()); string jsonResponse = unityWebRequest.downloadHandler.text; Debug.Log("response: " + jsonResponse); ObjectRecognitionManager.Instance.PrimaryText = "Receive response"; ObjectRecognitionManager.Instance.Update_DebugDisplay(); //// The response will be in JSON format, therefore it needs to be deserialized //AnalysisRootObject analysisRootObject = new AnalysisRootObject(); //analysisRootObject = JsonConvert.DeserializeObject<AnalysisRootObject>(jsonResponse); //ObjectRecognitionManager.Instance.ObjectAnalysisResult(analysisRootObject); // The response will be in JSON format, therefore it needs to be deserialized AnalysisRootObject analysisRootObject = new AnalysisRootObject(); analysisRootObject = JsonUtility.FromJson <AnalysisRootObject>(jsonResponse); //analysisRootObject = JsonConvert.DeserializeObject<AnalysisRootObject>(jsonResponse); ObjectRecognitionManager.Instance.PrimaryText = "predictions id: " + analysisRootObject.id; ObjectRecognitionManager.Instance.Update_DebugDisplay(); Debug.Log("predictions id: " + analysisRootObject.id); ObjectRecognitionManager.Instance.ObjectAnalysisResult(analysisRootObject); } }
public void PlaceLabels(AnalysisRootObject analysisObject) { if (analysisObject.predictions != null) { foreach (var prediction in analysisObject.predictions) { if (prediction.probability > probabilityThreshold) { // LABEL FOR TAG Transform newLabel = Instantiate(label.transform, cursor.transform.position, transform.rotation); newLabel.transform.localScale = new Vector3(0.005f, 0.005f, 0.005f); // LABEL FOR PROBABILITY Transform newProbaLabel = Instantiate(probaLabel.transform, cursor.transform.position, transform.rotation); newProbaLabel.transform.localScale = new Vector3(0.005f, 0.005f, 0.005f); quadRenderer = quad.GetComponent <Renderer>() as Renderer; Bounds quadBounds = quadRenderer.bounds; // Position the label as close as possible to the Bounding Box of the prediction // At this point it will not consider depth newLabel.transform.parent = quad.transform; newProbaLabel.transform.parent = quad.transform; newLabel.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, prediction.boundingBox); newProbaLabel.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, prediction.boundingBox); // Cast a ray from the user's head to the currently placed label, it should hit the object detected by the Service. // At that point it will reposition the label where the ray HL sensor collides with the object, // (using the HL spatial tracking) //Debug.Log("Repositioning Label"); Vector3 headPosition = Camera.main.transform.position; RaycastHit objHitInfo; Vector3 objDirection = newLabel.position; if (Physics.Raycast(headPosition, objDirection, out objHitInfo, 30.0f, SpatialMapping.PhysicsRaycastMask)) { newLabel.position = objHitInfo.point; } newProbaLabel.position = new Vector3(newLabel.position.x, newLabel.position.y - 0.05f, newLabel.position.z); // Set the tag text newLabel.GetComponent <TextMesh>().text = prediction.tagName; // Set the probability text newProbaLabel.GetComponent <TextMesh>().text = prediction.probability.ToString().Substring(0, 4); } } } // Reset the color of the cursor cursor.GetComponent <Renderer>().material.color = Color.green; // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); }
/// <summary> /// Set the Tags as Text of the last label created. /// </summary> public void FinaliseLabel(AnalysisRootObject analysisObject) { if (analysisObject.predictions != null) { lastLabelPlacedText = lastLabelPlaced.GetComponent <TextMesh>(); // Sort the predictions to locate the highest one List <Prediction> sortedPredictions = new List <Prediction>(); sortedPredictions = analysisObject.predictions.OrderBy(p => p.probability).ToList(); Prediction bestPrediction = new Prediction(); bestPrediction = sortedPredictions[sortedPredictions.Count - 1]; if (bestPrediction.probability > probabilityThreshold) { quadRenderer = quad.GetComponent <Renderer>() as Renderer; Bounds quadBounds = quadRenderer.bounds; // Position the label as close as possible to the Bounding Box of the prediction // At this point it will not consider depth lastLabelPlaced.transform.parent = quad.transform; lastLabelPlaced.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); //create rect //rect.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); rect.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); rect.transform.localScale = new Vector3((float)rectHeight * 0.8f, (float)rectWidth * 0.8f, 0.8f); Instantiate(rect, cursor.transform.position, transform.rotation); // Set the tag text lastLabelPlacedText.text = bestPrediction.tagName; // Cast a ray from the user's head to the currently placed label, it should hit the object detected by the Service. // At that point it will reposition the label where the ray HL sensor collides with the object, // (using the HL spatial tracking) Debug.Log("Repositioning Label"); Vector3 headPosition = Camera.main.transform.position; RaycastHit objHitInfo; Vector3 objDirection = lastLabelPlaced.position; if (Physics.Raycast(headPosition, objDirection, out objHitInfo, 30.0f, SpatialMapping.PhysicsRaycastMask)) { lastLabelPlaced.position = objHitInfo.point; } } } // Reset the color of the cursor cursor.GetComponent <Renderer>().material.color = Color.green; // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); }
public void ObjectAnalysisResult(AnalysisRootObject analysisObject) { Debug.Log("Showing result . . ."); PrimaryText = "Showing result . . ."; Update_DebugDisplay(); if (analysisObject.predictions != null) { Debug.Log("Prediction is"); PrimaryText = "Prediction is . . ."; Update_DebugDisplay(); // Sort the predictions to locate the highest one List <Prediction> sortedPredictions = new List <Prediction>(); sortedPredictions = analysisObject.predictions.OrderBy(p => p.probability).ToList(); Prediction bestPrediction = new Prediction(); bestPrediction = sortedPredictions[sortedPredictions.Count - 1]; if (bestPrediction.probability > probabilityThreshold) { //PrimaryText = bestPrediction.tagName + " : " + bestPrediction.probability.ToString(); PrimaryText = "Take a picture to be analyzed"; Update_DebugDisplay(); GameObject resultPanel = Instantiate(panelReference, CameraCache.Main.transform.position, CameraCache.Main.transform.rotation); Vector3 toPosition = CameraCache.Main.transform.position + CameraCache.Main.transform.forward; resultPanel.transform.position = toPosition; PredictionResultData resultData = new PredictionResultData(); resultData.objectName = bestPrediction.tagName; resultData.confidentResult = bestPrediction.probability.ToString(); StateManagement.Instance.lastestPredictionResult = resultData; resultPanels.Add(resultPanel); } } else { Debug.Log("Object is unknown"); PrimaryText = "Object is unknown"; Update_DebugDisplay(); } // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); }
/// <summary> /// Call the Computer Vision Service to submit the image. /// </summary> public IEnumerator AnalyseLastImageCaptured(string imagePath) { Debug.Log("Analyzing..."); WWWForm webForm = new WWWForm(); using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(predictionEndpoint, webForm)) { // Gets a byte array out of the saved image imageBytes = GetImageAsByteArray(imagePath); unityWebRequest.SetRequestHeader("Content-Type", "application/octet-stream"); unityWebRequest.SetRequestHeader("Prediction-Key", predictionKey); // The upload handler will help uploading the byte array with the request unityWebRequest.uploadHandler = new UploadHandlerRaw(imageBytes); unityWebRequest.uploadHandler.contentType = "application/octet-stream"; // The download handler will help receiving the analysis from Azure unityWebRequest.downloadHandler = new DownloadHandlerBuffer(); // Send the request yield return(unityWebRequest.SendWebRequest()); string jsonResponse = unityWebRequest.downloadHandler.text; Debug.Log("response: " + jsonResponse); // Create a texture. Texture size does not matter, since // LoadImage will replace with the incoming image size. Texture2D tex = new Texture2D(1, 1); tex.LoadImage(imageBytes); SceneOrganiser.Instance.quadRenderer.material.SetTexture("_MainTex", tex); // The response will be in JSON format, therefore it needs to be deserialized AnalysisRootObject analysisRootObject = new AnalysisRootObject(); analysisRootObject = JsonConvert.DeserializeObject <AnalysisRootObject>(jsonResponse); SceneOrganiser.Instance.FinaliseLabel(analysisRootObject); SceneOrganiser.Instance.PingOnSuccessful(); } }
public IEnumerator AnalyseLastImageCaptured(string imagePath) { Debug.Log("Analyzing..."); //WWWForm是一个HelperClass辅助类,用来生成表单数据,用来存储要访问服务器的数据 WWWForm webForm = new WWWForm(); //post方法包括uri和WWWForm表单BodyData两部分 using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(predictionEndpoint, webForm)) { imageBytes = GetImageAsByteArray(imagePath); //设定请求头 unityWebRequest.SetRequestHeader("Content-Type", "application/octet-stream"); unityWebRequest.SetRequestHeader("Prediction-Key", predictionKey); //上传数据体至服务器端,需要指定请求头的contentType unityWebRequest.uploadHandler = new UploadHandlerRaw(imageBytes); unityWebRequest.uploadHandler.contentType = "application/octet-stream"; //download handler帮助接收服务器端分析的数据 unityWebRequest.downloadHandler = new DownloadHandlerBuffer(); //yield只能建立在IEnumerator类中执行 yield return(unityWebRequest.SendWebRequest()); string jsonResponse = unityWebRequest.downloadHandler.text; Debug.Log("response: " + jsonResponse); Texture2D tex = new Texture2D(1, 1); tex.LoadImage(imageBytes); SceneOrganiser.Instance.quadRenderer.material.SetTexture("_MainTex", tex); AnalysisRootObject analysisRootObject = new AnalysisRootObject(); analysisRootObject = JsonConvert.DeserializeObject <AnalysisRootObject>(jsonResponse); //Azure服务对最新照片分析完毕后,触发FinaliseLabel()方法,正确放置标签文字 SceneOrganiser.Instance.FinaliseLabel(analysisRootObject); } }
/// <summary> /// Set the Tags as Text of the last label created. /// </summary> //public void FinaliseLabel(AnalysisRootObject analysisObject) //{ // if (analysisObject.predictions != null) // { // lastLabelPlacedText = lastLabelPlaced.GetComponent<TextMesh>(); // // Sort the predictions to locate the highest one // List<Prediction> sortedPredictions = new List<Prediction>(); // sortedPredictions = analysisObject.predictions.OrderBy(p => p.probability).ToList(); // Prediction bestPrediction = new Prediction(); // bestPrediction = sortedPredictions[sortedPredictions.Count - 1]; // if (bestPrediction.probability > probabilityThreshold) // { // quadRenderer = quad.GetComponent<Renderer>() as Renderer; // Bounds quadBounds = quadRenderer.bounds; // // Position the label as close as possible to the Bounding Box of the prediction // // At this point it will not consider depth // lastLabelPlaced.transform.parent = quad.transform; // lastLabelPlaced.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); // // Set the tag text // lastLabelPlacedText.text = bestPrediction.tagName; // // Cast a ray from the user's head to the currently placed label, it should hit the object detected by the Service. // // At that point it will reposition the label where the ray HL sensor collides with the object, // // (using the HL spatial tracking) // Debug.Log("Repositioning Label"); // Vector3 headPosition = Camera.main.transform.position; // RaycastHit objHitInfo; // Vector3 objDirection = lastLabelPlaced.position; // if (Physics.Raycast(headPosition, objDirection, out objHitInfo, 30.0f, SpatialMapping.PhysicsRaycastMask)) // { // lastLabelPlaced.position = objHitInfo.point; // } // } // } // // Reset the color of the cursor // cursor.GetComponent<Renderer>().material.color = Color.green; // // Stop the analysis process // ImageCapture.Instance.ResetImageCapture(); //} public void FinaliseLabel(AnalysisRootObject analysisObject) { if (analysisObject.predictions != null && analysisObject.predictions.ToList().Count != 0) { lastLabelPlacedText = lastLabelPlaced.GetComponent <TextMesh>(); quadRenderer = quad.GetComponent <Renderer>() as Renderer; Bounds quadBounds = quadRenderer.bounds; // Position the label as close as possible to the Bounding Box of the prediction // At this point it will not consider depth QRObject qRObject = new QRObject(); qRObject = (analysisObject.predictions.ToList())[0]; lastLabelPlaced.transform.parent = quad.transform; lastLabelPlaced.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, qRObject.boundingBox); // Set the tag text lastLabelPlacedText.text = qRObject.tagName; // Cast a ray from the user's head to the currently placed label, it should hit the object detected by the Service. // At that point it will reposition the label where the ray HL sensor collides with the object, // (using the HL spatial tracking) Debug.Log("Repositioning Label"); Vector3 headPosition = Camera.main.transform.position; RaycastHit objHitInfo; Vector3 objDirection = lastLabelPlaced.position; if (Physics.Raycast(headPosition, objDirection, out objHitInfo, 30.0f, SpatialMapping.PhysicsRaycastMask)) { lastLabelPlaced.position = objHitInfo.point; } } // Reset the color of the cursor cursor.GetComponent <Renderer>().material.color = Color.green; // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); }
/// <summary> /// Set the Tags as Text of the last label created. /// </summary> public void ShowTag(AnalysisRootObject analysisObject) { if (analysisObject.predictions != null) { // Sort the predictions to locate the highest one. Note that for demo purposes, we are just going to show that tag alone. List <Prediction> sortedPredictions = new List <Prediction>(); sortedPredictions = analysisObject.predictions.OrderBy(p => p.probability).ToList(); Prediction bestPrediction = new Prediction(); bestPrediction = sortedPredictions[sortedPredictions.Count - 1]; if (bestPrediction.probability > probabilityThreshold) { quadRenderer = quad.GetComponent <Renderer>() as Renderer; Bounds quadBounds = quadRenderer.bounds; // Position the label as close as possible to the Bounding Box of the prediction // At this point it will not consider depth this.tagFound.SetActive(true); this.tagFound.transform.parent = quad.transform; this.tagFound.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); this.tagName.GetComponent <Text>().text = bestPrediction.tagName; this.ShowObjectIdentified(); } else { this.ShowNoObjectsIdentified(); } } else { this.ShowNoObjectsIdentified(); } // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); }
//1.设置标签文字为最高概率预测 //2.调用CalculateBoundingBoxPosition方法,在场景中放置标签 //3.运用Raycast,方向为Camera至BoundingBox的方向,与物理层产生碰撞,即可获得深度信息,调整标签深度 //4.充值Capture过程,使用户可以捕获新的照片 public void FinaliseLabel(AnalysisRootObject analysisRootObject) { if (analysisRootObject.predictions != null) { lastLabelPlacedText = lastLabelPlaced.GetComponent <TextMesh>(); List <Prediction> sortedPredictions = new List <Prediction>(); //OrderBy默认升序排序 sortedPredictions = analysisRootObject.predictions.OrderBy(p => p.probability).ToList(); Prediction bestPrediction = new Prediction(); bestPrediction = sortedPredictions[sortedPredictions.Count - 1]; if (bestPrediction.probability > probabilityThreshold) { quadRenderer = quad.GetComponent <Renderer>() as Renderer; //Bounds边界盒 Bounds quadBounds = quadRenderer.bounds; lastLabelPlaced.transform.parent = quad.transform; //设定相对于父组件的相对位置。标签文字这时是在Quad上的 //lastLabelPlaced的parent是quad,故localPosition是针对于quad来说的。localPosition为父子组件之间中心的距离 lastLabelPlaced.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); lastLabelPlacedText.text = bestPrediction.tagName; /* * //Cube半透明 * Debug.Log("Placing The Box."); * box = GameObject.CreatePrimitive(PrimitiveType.Cube); * float BoxTransparency = 0.5f; * box.GetComponent<Renderer>().material = new Material(Shader.Find("Legacy Shaders/Transparent/Diffuse")); * box.GetComponent<Renderer>().material.color = new Color(1, 1, 1, BoxTransparency); * box.transform.parent = quad.transform; * box.transform.localPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); * * //注:localScale相对于父物体缩放比例在(0,1)之间 * //貌似正确的Box大小 * box.transform.localScale = new Vector3((float)(bestPrediction.boundingBox.width/(2*quad.transform.localScale.x)),(float)(bestPrediction.boundingBox.height/(2*quad.transform.localScale.y)), 0.05f); * Debug.LogFormat(@"The Local Scale X of Quad:{0} The Local Scale Y of Quad:{1} ", quad.transform.localScale.x, quad.transform.localScale.y); * Debug.Log("Firstly The Label Position: " + lastLabelPlaced.transform.position); * Debug.Log("Firstly The Box Position: " + box.transform.position); */ Debug.Log("Repositioning Label..."); Vector3 headPosition = Camera.main.transform.position; RaycastHit objHitInfo; Vector3 objDirection = lastLabelPlaced.position; if (Physics.Raycast(headPosition, objDirection, out objHitInfo, 30.0f, SpatialMapping.PhysicsRaycastMask)) { //设定世界坐标位置 lastLabelPlaced.position = objHitInfo.point; /* * box.transform.position = objHitInfo.point; * Debug.Log("After Raycast The Label Position: " + lastLabelPlaced.position); * Debug.Log("After Raycast The Box Position: " + box.transform.position); */ } } } cursor.GetComponent <Renderer>().material.color = Color.green; ImageCapture.Instance.ResetImageCapture(); }
/// <summary> /// Call the Computer Vision Service to submit the image. /// </summary> public IEnumerator AnalyseLastImageCaptured() { Debug.Log("Analyzing..."); WWWForm webForm = new WWWForm(); using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(visionAnalysisEndpoint, webForm)) { // gets a byte array out of the saved image imageBytes = GetImageAsByteArray(imagePath); Debug.Log(imageBytes); unityWebRequest.SetRequestHeader("Content-Type", "application/octet-stream"); unityWebRequest.SetRequestHeader(ocpApimSubscriptionKeyHeader, authorizationKey); // the download handler will help receiving the analysis from Azure unityWebRequest.downloadHandler = new DownloadHandlerBuffer(); // the upload handler will help uploading the byte array with the request unityWebRequest.uploadHandler = new UploadHandlerRaw(imageBytes); unityWebRequest.uploadHandler.contentType = "application/octet-stream"; yield return(unityWebRequest.SendWebRequest()); long responseCode = unityWebRequest.responseCode; Debug.Log(unityWebRequest.responseCode); try { string jsonResponse = null; jsonResponse = unityWebRequest.downloadHandler.text; // Create a texture. Texture size does not matter, since // LoadImage will replace with the incoming image size. //Texture2D tex = new Texture2D(1, 1); //tex.LoadImage(imageBytes); //ResultsLabel.instance.quadRenderer.material.SetTexture("_MainTex", tex); //AnalysisRootObject analysisRootObject = new AnalysisRootObject(); //analysisRootObject = JsonConvert.DeserializeObject<AnalysisRootObject>(jsonResponse); //ResultsLabel.instance.FinaliseLabel(analysisRootObject); // // The response will be in Json format // // therefore it needs to be deserialized into the classes AnalysedObject and TagData AnalysisRootObject analysedObject = new AnalysisRootObject(); analysedObject = JsonUtility.FromJson <AnalysisRootObject>(jsonResponse); if (analysedObject.tags == null) { Debug.Log("analysedObject.tagData is null"); } else { Dictionary <string, float> tagsDictionary = new Dictionary <string, float>(); foreach (TagData td in analysedObject.tags) { TagData tag = td as TagData; tagsDictionary.Add(tag.name, tag.confidence); } Texture2D tex = new Texture2D(1, 1); tex.LoadImage(imageBytes); ResultsLabel.instance.quadRenderer.material.SetTexture("_MainTex", tex); ResultsLabel.instance.SetTagsToLastLabel(tagsDictionary); } } catch (Exception exception) { Debug.Log("Json exception.Message: " + exception.Message); } yield return(null); } }
/// <summary> /// Call the Computer Vision Service to submit the image. /// </summary> public IEnumerator AnalyseLastImageCaptured(string imagePath) { Debug.Log("Analyzing..."); WWWForm webForm = new WWWForm(); using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(predictionEndpoint, webForm)) { System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); sw.Start(); // Gets a byte array out of the saved image imageBytes = GetImageAsByteArray(imagePath); unityWebRequest.SetRequestHeader("Content-Type", "application/octet-stream"); unityWebRequest.SetRequestHeader("Prediction-Key", predictionKey); // The upload handler will help uploading the byte array with the request unityWebRequest.uploadHandler = new UploadHandlerRaw(imageBytes); unityWebRequest.uploadHandler.contentType = "application/octet-stream"; // The download handler will help receiving the analysis from Azure unityWebRequest.downloadHandler = new DownloadHandlerBuffer(); Debug.Log("Time 0 : " + sw.Elapsed); // Send the request yield return(unityWebRequest.SendWebRequest()); Debug.Log("Time 1 : " + sw.Elapsed); string jsonResponse = unityWebRequest.downloadHandler.text; Debug.Log("response: " + jsonResponse); // Create a texture. Texture size does not matter, since // LoadImage will replace with the incoming image size. Debug.Log("Time 2 : " + sw.Elapsed); Texture2D tex = new Texture2D(1, 1); tex.LoadImage(imageBytes); Debug.Log("Time 3 : " + sw.Elapsed); SceneOrganiser.Instance.quadRenderer.material.SetTexture("_MainTex", tex); Debug.Log("Time 4 : " + sw.Elapsed); // The response will be in JSON format, therefore it needs to be deserialized AnalysisRootObject analysisRootObject = new AnalysisRootObject(); analysisRootObject = JsonConvert.DeserializeObject <AnalysisRootObject>(jsonResponse); Debug.Log("Time 5 : " + sw.Elapsed); SceneOrganiser.Instance.FinaliseLabel(analysisRootObject); for (int i = 0; i < analysisRootObject.predictions.Count; i++) { Debug.Log("Prediction " + (i + 1) + " in this frame: " + analysisRootObject.predictions[i].probability + " for object " + analysisRootObject.predictions[i].tagName + " in the bounding box of data " + " left: " + analysisRootObject.predictions[i].boundingBox.left + " top: " + analysisRootObject.predictions[i].boundingBox.top + " width: " + analysisRootObject.predictions[i].boundingBox.width + " height: " + analysisRootObject.predictions[i].boundingBox.height); } sw.Stop(); } }
/// <summary> /// Set the Tags as Text of the last label created. /// </summary> public void FinaliseLabel(AnalysisRootObject analysisObject) { if (analysisObject != null && analysisObject.predictions != null) { lastLabelPlacedText = lastLabelPlaced.GetComponent <TextMesh>(); // Sort the predictions to locate the highest one List <Prediction> sortedPredictions = new List <Prediction>(); sortedPredictions = analysisObject.predictions.OrderBy(p => p.probability).ToList(); //TOO remove below /* quadRenderer = quad.GetComponent<Renderer>() as Renderer; * Bounds quadBounds = quadRenderer.bounds; * lastLabelPlaced.transform.parent = quad.transform; * // Vector3 labelPosition = new Vector3((float)-quadBounds.size.normalized.x/2, (float)-quadBounds.size.normalized.y/2, 0); * Vector3 labelPosition = new Vector3(0, 0, 0); * lastLabelPlaced.transform.localPosition = labelPosition; * lastLabelPlacedText.text = "";//bestPrediction.tagName; * Debug.Log("Repositioning Label"); * Vector3 headPosition = Camera.main.transform.position; * RaycastHit objHitInfo; * Vector3 objDirection = lastLabelPlaced.position; * * if (Physics.Raycast(captureTimeHeadPosition, objDirection, out objHitInfo, 30.0f, captureTimeRaycastMask)) * { * lastLabelPlaced.position = objHitInfo.point; * latestDataPanelPosition = objHitInfo.point; * } * ObjectData data = envData.getObjectData("water bottle"); * data.probability = 0.90; * StartCoroutine(makeDataPanel(data, latestDataPanelPosition, latestDataPanelRotation)); * IEnumerator coroutine = ResetLoadingUI(lastLabelPlacedText, lastLoadingIcon, 5.0f); * StartCoroutine(coroutine); */ //TODO uncomment below Prediction bestPrediction = new Prediction(); if (sortedPredictions.Count > 0) { bestPrediction = sortedPredictions[sortedPredictions.Count - 1]; } else { bestPrediction.probability = 0; } if (bestPrediction.probability > probabilityThreshold) { quadRenderer = quad.GetComponent <Renderer>() as Renderer; Bounds quadBounds = quadRenderer.bounds; // Position the label as close as possible to the Bounding Box of the prediction // At this point it will not consider depth lastLabelPlaced.transform.parent = quad.transform; Vector3 labelPosition = CalculateBoundingBoxPosition(quadBounds, bestPrediction.boundingBox); lastLabelPlaced.transform.localPosition = labelPosition; // Set the tag text lastLabelPlacedText.text = "";//bestPrediction.tagName; // Cast a ray from the user's head to the currently placed label, it should hit the object detected by the Service. // At that point it will reposition the label where the ray HL sensor collides with the object, // (using the HL spatial tracking) Debug.Log("Repositioning Label"); Vector3 headPosition = Camera.main.transform.position; RaycastHit objHitInfo; Vector3 objDirection = lastLabelPlaced.position; if (Physics.Raycast(captureTimeHeadPosition, objDirection, out objHitInfo, 30.0f, captureTimeRaycastMask)) { lastLabelPlaced.position = objHitInfo.point; latestDataPanelPosition = objHitInfo.point; } Debug.Log("Success"); ObjectData data = envData.getObjectData(bestPrediction.tagName); data.probability = bestPrediction.probability; StartCoroutine(MakeDataPanel(data, latestDataPanelPosition, latestDataPanelRotation)); } else { LoadingRotation loadingRotation = lastLoadingIcon.GetComponent <LoadingRotation>(); loadingRotation.Failed(); } IEnumerator coroutine = ResetLoadingUI(lastLabelPlacedText, lastLoadingIcon, 5.0f); StartCoroutine(coroutine); } // Reset the color of the cursor cursor.GetComponent <Renderer>().material.color = Color.yellow; // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); }
/// <summary> /// Set the Tags as Text of the last label created. /// </summary> public void FinaliseLabel(AnalysisRootObject analysisObject) { if (analysisObject.predictions != null) { // Sort the predictions to locate the highest one List <Prediction> sortedPredictions = new List <Prediction>(); sortedPredictions = analysisObject.predictions.OrderByDescending(p => p.probability).ToList(); foreach (Prediction prediction in sortedPredictions) { if (prediction.probability < probabilityThreshold) { Debug.Log("Already at the threshold. Stopping"); break; } if (prediction.tagName != suitcaseLabel) { Debug.Log("Skipping non-suitcase: " + prediction.tagName); break; } GameObject label = CreateLabel(); /// <summary> /// Reference to the last Label positioned /// </summary> Transform labelObj = Instantiate(label.transform, cursor.transform.position, transform.rotation); /// <summary> /// Reference to the last Label positioned /// </summary> TextMesh labelObjText = labelObj.GetComponent <TextMesh>(); // Initialize new label PlaceAnalysisLabel(label, labelObj, labelObjText); quadRenderer = quad.GetComponent <Renderer>() as Renderer; Bounds quadBounds = quadRenderer.bounds; // Position the label as close as possible to the Bounding Box of the prediction // At this point it will not consider depth labelObj.transform.parent = quad.transform; MyRectangle positionRect = CalculateBoundingBoxPosition(quadBounds, prediction.boundingBox); labelObj.transform.localPosition = positionRect.GetCenter(); // Cast a ray from the user's head to the currently placed label, it should hit the object detected by the Service. // At that point it will reposition the label where the ray HL sensor collides with the object, // (using the HL spatial tracking) Debug.Log("Repositioning Label"); Vector3 headPosition = Camera.main.transform.position; RaycastHit objHitInfo; Vector3 objDirection = labelObj.position; string dimensionsString = ""; if (Physics.Raycast(headPosition, objDirection, out objHitInfo, 30.0f, SpatialMapping.PhysicsRaycastMask)) { float widthInMeters = 0; float heightInMeters = 0; labelObj.position = objHitInfo.point; widthInMeters = Vector3.Distance(new Vector3(positionRect.topLeft.x, positionRect.topLeft.y, objHitInfo.distance), new Vector3(positionRect.bottomRight.x, positionRect.topLeft.y, objHitInfo.distance)); heightInMeters = Vector3.Distance(new Vector3(positionRect.topLeft.x, positionRect.topLeft.y, objHitInfo.distance), new Vector3(positionRect.topLeft.x, positionRect.bottomRight.y, objHitInfo.distance)); dimensionsString = " - " + widthInMeters + "x" + heightInMeters; // Draw rectangle around DrawRect(positionRect, objHitInfo.distance); } // Set the tag text labelObjText.text = prediction.tagName + " (" + (prediction.probability * 100) + "%)" + dimensionsString; } } // Reset the color of the cursor cursor.GetComponent <Renderer>().material.color = Color.green; // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); }
/// <summary> /// Call the Computer Vision Service to submit the image. /// </summary> public IEnumerator AnalyseLastImageCaptured(string imagePath, string operation = "label") { Debug.Log("Analyzing..."); WWWForm webForm = new WWWForm(); using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(predictionEndpoint, webForm)) { // Gets a byte array out of the saved image imageBytes = GetImageAsByteArray(imagePath); unityWebRequest.SetRequestHeader("Content-Type", "application/octet-stream"); // The upload handler will help uploading the byte array with the request unityWebRequest.uploadHandler = new UploadHandlerRaw(imageBytes); unityWebRequest.uploadHandler.contentType = "application/octet-stream"; // The download handler will help receiving the analysis from Azure unityWebRequest.downloadHandler = new DownloadHandlerBuffer(); // Send the request yield return(unityWebRequest.SendWebRequest()); string jsonResponse = unityWebRequest.downloadHandler.text; Debug.Log("response: " + jsonResponse); // Create a texture. Texture size does not matter, since // LoadImage will replace with the incoming image size. Texture2D tex = new Texture2D(1, 1); tex.LoadImage(imageBytes); SceneOrganiser.Instance.quadRenderer.material.SetTexture("_MainTex", tex); // The response will be in JSON format, therefore it needs to be deserialized AnalysisRootObject analysisRootObject = new AnalysisRootObject(); analysisRootObject = JsonUtility.FromJson <AnalysisRootObject>(jsonResponse); QRObject qRObject = (analysisRootObject.predictions.ToList())[0]; string data = qRObject.tagName; data = data.Substring(7); switch (operation) { case "label": SceneOrganiser.Instance.FinaliseLabel(analysisRootObject); break; case "get_location": // TODO Validate data, i.e error handling var parameters = data.Split(','); var buid = parameters[0]; var floor_number = int.Parse(parameters[1]); StartCoroutine(navManager.LoadPois(buid, floor_number)); break; case "get_worker": firebaseManager.loadWorker(data); break; default: break; } SceneOrganiser.Instance.FinaliseLabel(analysisRootObject); } }