private void spawnAnnotationFetchingTask(int frameNo, AnnotationsFetcher fetcher, float th, OnAnnotationFetched onAnnotationFetched = null) { fetcher.fetchAnnotation(frameNo, delegate(string jsonArrayString) { var now = System.DateTime.Now; string fetcherName = fetcher.getServiceName(); float threshold = th; int frameNumber = frameNo; string debugString = jsonArrayString.Replace(System.Environment.NewLine, ""); Debug.LogFormat((ILogComponent)this, "fetched {3} annotations JSON for {0}, length {1}: {2}", frameNumber, jsonArrayString.Length, debugString, fetcherName); try { if (onAnnotationFetched != null) { onAnnotationFetched(now, jsonArrayString); } } catch (System.Exception e) { Debug.LogExceptionFormat(e, "caught exception while callback, annotation: {0}", debugString.Substring(0, 100)); } try { Dictionary <int, FrameObjectData> frameObjects = frameBuffer_.Dequeue(); FrameObjectData frameObjectData; if (frameObjects.TryGetValue(frameNumber, out frameObjectData)) { // some pre-processing for the received string // TBD: this needs to be replaced with JSON deserialization, need to update edge too string[] testDebug = jsonArrayString.Split(']'); string formatDebug = testDebug[0] + "]"; string str = "{ \"annotationData\": " + formatDebug + "}"; AnnotationData data = JsonUtility.FromJson <AnnotationData>(str); #if DEVELOPMENT_BUILD for (int i = 0; i < data.annotationData.Length; i++) { Debug.LogFormat((ILogComponent)this, "{7} annotation {0}: label {1} prob {2} xleft {3} xright {4} ytop {5} ybottom {6}", i, data.annotationData[i].label, data.annotationData[i].prob, data.annotationData[i].xleft, data.annotationData[i].xright, data.annotationData[i].ytop, data.annotationData[i].ybottom, fetcherName); } #endif Debug.LogFormat((ILogComponent)this, "processing {5} annotation for frame #{0}, cam pos {1}, cam rot {2}, points {3}, lifetime {4} sec", frameNumber, frameObjectData.camPos, frameObjectData.camRot, frameObjectData.points, Mathf.Abs((float)(frameObjectData.timestamp - timestamp_)), fetcherName); // filter out annotations with probability below threshold List <AnnotationData.ArrayEntry> filteredAnnotations = new List <AnnotationData.ArrayEntry>(); for (int i = 0; i < data.annotationData.Length; ++i) { if (data.annotationData[i].prob >= threshold) { filteredAnnotations.Add(data.annotationData[i]); } } Debug.LogFormat("{0} annotations above threshold (filtered out {0} annotations)", filteredAnnotations.Count, data.annotationData.Length - filteredAnnotations.Count); if (filteredAnnotations.Count > 0) { int boxCount = filteredAnnotations.Count; // TBD: this needs to be pooled BoxData boxData = new BoxData(); // TBD: this needs to be hidden in a method/constructor boxData.frameNumber = frameNumber; boxData.count = boxCount; boxData.points = frameObjectData.points; boxData.numPoints = frameObjectData.numPoints; boxData.cam = frameObjectData.cam; boxData.camPos = frameObjectData.camPos; boxData.camRot = frameObjectData.camRot; boxData.timestamp = frameObjectData.timestamp; boxData.label = new string[boxCount]; boxData.xleft = new float[boxCount]; boxData.xright = new float[boxCount]; boxData.ytop = new float[boxCount]; boxData.ybottom = new float[boxCount]; boxData.prob = new float[boxCount]; for (int i = 0; i < boxCount; i++) { // safety checks if (data.annotationData[i].ytop > 1) { data.annotationData[i].ytop = 1; } if (data.annotationData[i].ybottom < 0) { data.annotationData[i].ybottom = 0; } boxData.label[i] = filteredAnnotations[i].label; // since we flipped the image, flip bbox coordinates again // only y coordinates get flipped though, and it works. // I think this has something to do with how Viewport coordinates // are represented... boxData.xleft[i] = filteredAnnotations[i].xleft; boxData.xright[i] = filteredAnnotations[i].xright; boxData.ytop[i] = 1 - filteredAnnotations[i].ytop; boxData.ybottom[i] = 1 - filteredAnnotations[i].ybottom; boxData.prob[i] = filteredAnnotations[i].prob; Debug.LogFormat((ILogComponent)this, "will render {6}: {0} xleft {1} xright {2} ytop {3} ybottom {4} prob {5}", boxData.label[i], boxData.xleft[i], boxData.xright[i], boxData.ytop[i], boxData.ybottom[i], boxData.prob[i], fetcherName); } //boxBufferToCalc.Enqueue(annoData); boundingBoxBufferToCalc_.Enqueue(boxData); } } else { // frame object was not in the pool, lifetime expired Debug.LogFormat((ILogComponent)this, "received {0} annotations but frame expired", fetcherName); } } catch (System.Exception e) { Debug.LogExceptionFormat(e, "while parsing {0} annotation {1}...", fetcherName, debugString.Substring(0, 100)); } }); }
public void OnImageAvailable(TextureReaderApi.ImageFormatType format, int width, int height, IntPtr pixelBuffer, int bufferSize) { try{ System.DateTime current = System.DateTime.Now; long elapsedTicks = current.Ticks - begin.Ticks; System.TimeSpan elapsedSpan = new System.TimeSpan(elapsedTicks); timestamp = elapsedSpan.TotalSeconds; //Debug.Log("before call to ndnrtc"); int publishedFrameNo = NdnRtc.videoStream.processIncomingFrame(format, width, height, pixelBuffer, bufferSize); Debug.Log("Published frame number: " + publishedFrameNo); if (publishedFrameNo >= 0) { Debug.Log("create frame object frame number: " + publishedFrameNo); Debug.Log("create frame object timestamp: " + timestamp); Debug.Log("create frame object position: " + Frame.Pose.position); Debug.Log("create frame object rotation: " + Frame.Pose.rotation); Debug.Log("create frame object camera: " + camForCalcThread.ToString()); frameMgr.CreateFrameObject(publishedFrameNo, timestamp, Frame.Pose.position, Frame.Pose.rotation, camForCalcThread); //frameMgr.CreateFrameObject (imgBuffer, publishedFrameNo, timestamp, Vector3.zero, Quaternion.identity, offset.m_uOffset, offset.m_vOffset, camForCalcThread); //frameObjectBuffer.Enqueue (frameMgr.frameObjects [publishedFrameNo]); frameBuffer.Enqueue(frameMgr.frameObjects); Debug.Log("frame buffer enqueue: " + publishedFrameNo); // spawn fetching task for annotations of this frame // once successfully received, delegate callback will be called aFetcher_.fetchAnnotation(publishedFrameNo, delegate(string jsonArrayString) { int frameNumber = publishedFrameNo; // storing frame number locally string debuglog = jsonArrayString.Replace(System.Environment.NewLine, " "); Debug.Log("Received annotations JSON (frame " + frameNumber + "): " + debuglog); //Debug.Log("annotations string length: " + jsonArrayString.Length); string[] testDebug = jsonArrayString.Split(']'); string formatDebug = testDebug[0] + "]"; try{ Dictionary <int, FrameObjectData> frameObjects = frameBuffer.Dequeue(); FrameObjectData temp; if (frameObjects.TryGetValue(frameNumber, out temp)) { //AnnotationData[] data = JsonHelper.FromJson<AnnotationData>(jsonArrayString); //try to print out how many characters the jsonArrayString has string str = "{ \"annotationData\": " + formatDebug + "}"; AnnotationData data = JsonUtility.FromJson <AnnotationData>(str); for (int i = 0; i < data.annotationData.Length; i++) { if (data.annotationData[i].prob >= 0.5f) { Debug.Log("test: " + data.annotationData.Length); Debug.Log("test label: " + data.annotationData[i].label + " test xleft: " + data.annotationData[i].xleft + " test xright: " + data.annotationData[i].xright + " test ytop: " + (1 - data.annotationData[i].ytop) + " test ybottom: " + (1 - data.annotationData[i].ybottom)); // Debug.Log("test xleft: " + data.annotationData[i].xleft); // Debug.Log("test xright: " + data.annotationData[i].xright); // Debug.Log("test ytop: " + data.annotationData[i].ytop); // Debug.Log("test ybottom: " + data.annotationData[i].ybottom); // example how to fetch model from content-publisher // Therese, please check this is the right place in code where models should be requested // (prob. model doesn't need to be fetched every frame for same object) //fetchModel(data.annotationData[i].label); } } // FrameObjectData temp; // bool success = frameObjectBuffer.TryDequeue(out temp); // if(success) // //FrameObjectData temp = frameBuffer.Dequeue(); // { // Debug.Log("Frame info: " + frameNumber); // Debug.Log ("Frame info camera position: " + temp.camPos); // Debug.Log ("Frame info camera rotation: " + temp.camRot); // Debug.Log ("Frame info points number: " + temp.numPoints); // Debug.Log ("Frame info points: " + temp.points.ToString()); // } Debug.Log("Frame number annotations: " + frameNumber); Debug.Log("Frame info camera position: " + temp.camPos); Debug.Log("Frame info camera rotation: " + temp.camRot); //Debug.Log ("Frame info points number: " + temp.numPoints); Debug.Log("Frame info points: " + temp.points.ToString()); Debug.Log("test time difference: " + (Mathf.Abs((float)(temp.timestamp - timestamp))) + " frame number: " + publishedFrameNo); //int boxCount = Mathf.Min(data.annotationData.Length, 2); int boxCount = data.annotationData.Length; BoxData annoData = new BoxData(); Debug.Log("box created boxdata"); annoData.frameNumber = frameNumber; annoData.count = boxCount; annoData.points = temp.points; annoData.numPoints = temp.numPoints; annoData.cam = temp.cam; annoData.camPos = temp.camPos; annoData.camRot = temp.camRot; annoData.timestamp = temp.timestamp; annoData.label = new string[boxCount]; annoData.xleft = new float[boxCount]; annoData.xright = new float[boxCount]; annoData.ytop = new float[boxCount]; annoData.ybottom = new float[boxCount]; annoData.prob = new float[boxCount]; for (int i = 0; i < boxCount; i++) { annoData.label[i] = data.annotationData[i].label; annoData.xleft[i] = 1 - data.annotationData[i].xright; annoData.xright[i] = 1 - data.annotationData[i].xleft; annoData.ytop[i] = data.annotationData[i].ybottom; annoData.ybottom[i] = data.annotationData[i].ytop; annoData.prob[i] = data.annotationData[i].prob; } Debug.Log("Received annotations box enqueue"); //boxBufferToCalc.Enqueue(annoData); boundingBoxBufferToCalc.Enqueue(annoData); } else { //frame object was not in the pool, lifetime expired Debug.Log("Received annotations but frame expired"); } } catch (System.Exception e) { Debug.Log("exception caught annotations: " + e); string debug = jsonArrayString.Replace(System.Environment.NewLine, " "); Debug.Log("exception caught string: " + debug); string str = "{ \"annotationData\": " + debug + "}"; Debug.Log("exception caught string with format: " + str); } }); openFaceFetcher_.fetchAnnotation(publishedFrameNo, delegate(string jsonArrayString) { int frameNumber = publishedFrameNo; // storing frame number locally string debuglog = jsonArrayString.Replace(System.Environment.NewLine, " "); Debug.Log("Received OpenFace annotations JSON (frame " + frameNumber + "): " + debuglog); string[] testDebug = jsonArrayString.Split(']'); string formatDebug = testDebug[0] + "]"; try{ Dictionary <int, FrameObjectData> frameObjects = frameBuffer.Dequeue(); FrameObjectData temp; if (frameObjects.TryGetValue(frameNumber, out temp)) { string str = "{ \"annotationData\": " + formatDebug + "}"; AnnotationData data = JsonUtility.FromJson <AnnotationData>(str); for (int i = 0; i < data.annotationData.Length; i++) { //if(data.annotationData[i].prob >= 0.7f) { Debug.Log("openface test: " + data.annotationData.Length); Debug.Log("openface test label: " + data.annotationData[i].label + " test xleft: " + data.annotationData[i].xleft + " test xright: " + data.annotationData[i].xright + " test ytop: " + (data.annotationData[i].ytop) + " test ybottom: " + (data.annotationData[i].ybottom)); // Debug.Log("test xleft: " + data.annotationData[i].xleft); // Debug.Log("test xright: " + data.annotationData[i].xright); // Debug.Log("test ytop: " + data.annotationData[i].ytop); // Debug.Log("test ybottom: " + data.annotationData[i].ybottom); } } //int boxCount = Mathf.Min(data.annotationData.Length, 2); int boxCount = data.annotationData.Length; BoxData annoData = new BoxData(); Debug.Log("box created boxdata"); annoData.frameNumber = frameNumber; annoData.count = boxCount; annoData.points = temp.points; annoData.numPoints = temp.numPoints; annoData.cam = temp.cam; annoData.camPos = temp.camPos; annoData.camRot = temp.camRot; annoData.timestamp = temp.timestamp; annoData.label = new string[boxCount]; annoData.xleft = new float[boxCount]; annoData.xright = new float[boxCount]; annoData.ytop = new float[boxCount]; annoData.ybottom = new float[boxCount]; annoData.prob = new float[boxCount]; for (int i = 0; i < boxCount; i++) { if (data.annotationData[i].ytop > 1) { data.annotationData[i].ytop = 1; } if (data.annotationData[i].ybottom < 0) { data.annotationData[i].ybottom = 0; } annoData.label[i] = data.annotationData[i].label; annoData.xleft[i] = data.annotationData[i].xleft; annoData.xright[i] = data.annotationData[i].xright; annoData.ytop[i] = data.annotationData[i].ytop; annoData.ybottom[i] = data.annotationData[i].ybottom; annoData.prob[i] = 1; } Debug.Log("Received openface annotations box enqueue"); //boxBufferToCalc.Enqueue(annoData); boundingBoxBufferToCalc.Enqueue(annoData); } else { //frame object was not in the pool, lifetime expired Debug.Log("Received openface annotations but frame expired"); } } catch (System.Exception e) { Debug.Log("exception caught openface annotations: " + e); string debug = jsonArrayString.Replace(System.Environment.NewLine, " "); Debug.Log("exception caught openface string: " + debug); string str = "{ \"annotationData\": " + debug + "}"; Debug.Log("exception caught openface string with format: " + str); } }); } else { // frame was dropped by the encoder and was not published } } catch (System.Exception e) { Debug.Log("exception caught video" + e.ToString()); } }