private void spawnAnnotationFetchingTask(int frameNo, AnnotationsFetcher fetcher,
float th,
OnAnnotationFetched onAnnotationFetched = null)
{
fetcher.fetchAnnotation(frameNo, delegate(string jsonArrayString)
{
var now = System.DateTime.Now;
string fetcherName = fetcher.getServiceName();
float threshold = th;
int frameNumber = frameNo;
string debugString = jsonArrayString.Replace(System.Environment.NewLine, "");
Debug.LogFormat((ILogComponent)this, "fetched {3} annotations JSON for {0}, length {1}: {2}",
frameNumber, jsonArrayString.Length, debugString, fetcherName);
try {
if (onAnnotationFetched != null)
{
onAnnotationFetched(now, jsonArrayString);
}
}
catch (System.Exception e) {
Debug.LogExceptionFormat(e, "caught exception while callback, annotation: {0}",
debugString.Substring(0, 100));
}
try
{
Dictionary <int, FrameObjectData> frameObjects = frameBuffer_.Dequeue();
FrameObjectData frameObjectData;
if (frameObjects.TryGetValue(frameNumber, out frameObjectData))
{
// some pre-processing for the received string
// TBD: this needs to be replaced with JSON deserialization, need to update edge too
string[] testDebug = jsonArrayString.Split(']');
string formatDebug = testDebug[0] + "]";
string str = "{ \"annotationData\": " + formatDebug + "}";
AnnotationData data = JsonUtility.FromJson <AnnotationData>(str);
#if DEVELOPMENT_BUILD
for (int i = 0; i < data.annotationData.Length; i++)
{
Debug.LogFormat((ILogComponent)this,
"{7} annotation {0}: label {1} prob {2} xleft {3} xright {4} ytop {5} ybottom {6}",
i, data.annotationData[i].label, data.annotationData[i].prob,
data.annotationData[i].xleft, data.annotationData[i].xright,
data.annotationData[i].ytop, data.annotationData[i].ybottom,
fetcherName);
}
#endif
Debug.LogFormat((ILogComponent)this,
"processing {5} annotation for frame #{0}, cam pos {1}, cam rot {2}, points {3}, lifetime {4} sec",
frameNumber, frameObjectData.camPos, frameObjectData.camRot, frameObjectData.points,
Mathf.Abs((float)(frameObjectData.timestamp - timestamp_)), fetcherName);
// filter out annotations with probability below threshold
List <AnnotationData.ArrayEntry> filteredAnnotations = new List <AnnotationData.ArrayEntry>();
for (int i = 0; i < data.annotationData.Length; ++i)
{
if (data.annotationData[i].prob >= threshold)
{
filteredAnnotations.Add(data.annotationData[i]);
}
}
Debug.LogFormat("{0} annotations above threshold (filtered out {0} annotations)",
filteredAnnotations.Count, data.annotationData.Length - filteredAnnotations.Count);
if (filteredAnnotations.Count > 0)
{
int boxCount = filteredAnnotations.Count;
// TBD: this needs to be pooled
BoxData boxData = new BoxData();
// TBD: this needs to be hidden in a method/constructor
boxData.frameNumber = frameNumber;
boxData.count = boxCount;
boxData.points = frameObjectData.points;
boxData.numPoints = frameObjectData.numPoints;
boxData.cam = frameObjectData.cam;
boxData.camPos = frameObjectData.camPos;
boxData.camRot = frameObjectData.camRot;
boxData.timestamp = frameObjectData.timestamp;
boxData.label = new string[boxCount];
boxData.xleft = new float[boxCount];
boxData.xright = new float[boxCount];
boxData.ytop = new float[boxCount];
boxData.ybottom = new float[boxCount];
boxData.prob = new float[boxCount];
for (int i = 0; i < boxCount; i++)
{
// safety checks
if (data.annotationData[i].ytop > 1)
{
data.annotationData[i].ytop = 1;
}
if (data.annotationData[i].ybottom < 0)
{
data.annotationData[i].ybottom = 0;
}
boxData.label[i] = filteredAnnotations[i].label;
// since we flipped the image, flip bbox coordinates again
// only y coordinates get flipped though, and it works.
// I think this has something to do with how Viewport coordinates
// are represented...
boxData.xleft[i] = filteredAnnotations[i].xleft;
boxData.xright[i] = filteredAnnotations[i].xright;
boxData.ytop[i] = 1 - filteredAnnotations[i].ytop;
boxData.ybottom[i] = 1 - filteredAnnotations[i].ybottom;
boxData.prob[i] = filteredAnnotations[i].prob;
Debug.LogFormat((ILogComponent)this, "will render {6}: {0} xleft {1} xright {2} ytop {3} ybottom {4} prob {5}",
boxData.label[i], boxData.xleft[i], boxData.xright[i],
boxData.ytop[i], boxData.ybottom[i], boxData.prob[i],
fetcherName);
}
//boxBufferToCalc.Enqueue(annoData);
boundingBoxBufferToCalc_.Enqueue(boxData);
}
}
else
{
// frame object was not in the pool, lifetime expired
Debug.LogFormat((ILogComponent)this, "received {0} annotations but frame expired",
fetcherName);
}
}
catch (System.Exception e)
{
Debug.LogExceptionFormat(e, "while parsing {0} annotation {1}...",
fetcherName, debugString.Substring(0, 100));
}
});
}
public void OnImageAvailable(TextureReaderApi.ImageFormatType format, int width, int height, IntPtr pixelBuffer, int bufferSize)
{
try{
System.DateTime current = System.DateTime.Now;
long elapsedTicks = current.Ticks - begin.Ticks;
System.TimeSpan elapsedSpan = new System.TimeSpan(elapsedTicks);
timestamp = elapsedSpan.TotalSeconds;
//Debug.Log("before call to ndnrtc");
int publishedFrameNo = NdnRtc.videoStream.processIncomingFrame(format, width, height, pixelBuffer, bufferSize);
Debug.Log("Published frame number: " + publishedFrameNo);
if (publishedFrameNo >= 0)
{
Debug.Log("create frame object frame number: " + publishedFrameNo);
Debug.Log("create frame object timestamp: " + timestamp);
Debug.Log("create frame object position: " + Frame.Pose.position);
Debug.Log("create frame object rotation: " + Frame.Pose.rotation);
Debug.Log("create frame object camera: " + camForCalcThread.ToString());
frameMgr.CreateFrameObject(publishedFrameNo, timestamp, Frame.Pose.position, Frame.Pose.rotation, camForCalcThread);
//frameMgr.CreateFrameObject (imgBuffer, publishedFrameNo, timestamp, Vector3.zero, Quaternion.identity, offset.m_uOffset, offset.m_vOffset, camForCalcThread);
//frameObjectBuffer.Enqueue (frameMgr.frameObjects [publishedFrameNo]);
frameBuffer.Enqueue(frameMgr.frameObjects);
Debug.Log("frame buffer enqueue: " + publishedFrameNo);
// spawn fetching task for annotations of this frame
// once successfully received, delegate callback will be called
aFetcher_.fetchAnnotation(publishedFrameNo, delegate(string jsonArrayString) {
int frameNumber = publishedFrameNo; // storing frame number locally
string debuglog = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("Received annotations JSON (frame " + frameNumber + "): " + debuglog);
//Debug.Log("annotations string length: " + jsonArrayString.Length);
string[] testDebug = jsonArrayString.Split(']');
string formatDebug = testDebug[0] + "]";
try{
Dictionary <int, FrameObjectData> frameObjects = frameBuffer.Dequeue();
FrameObjectData temp;
if (frameObjects.TryGetValue(frameNumber, out temp))
{
//AnnotationData[] data = JsonHelper.FromJson<AnnotationData>(jsonArrayString);
//try to print out how many characters the jsonArrayString has
string str = "{ \"annotationData\": " + formatDebug + "}";
AnnotationData data = JsonUtility.FromJson <AnnotationData>(str);
for (int i = 0; i < data.annotationData.Length; i++)
{
if (data.annotationData[i].prob >= 0.5f)
{
Debug.Log("test: " + data.annotationData.Length);
Debug.Log("test label: " + data.annotationData[i].label + " test xleft: " + data.annotationData[i].xleft
+ " test xright: " + data.annotationData[i].xright + " test ytop: " + (1 - data.annotationData[i].ytop) + " test ybottom: " + (1 - data.annotationData[i].ybottom));
// Debug.Log("test xleft: " + data.annotationData[i].xleft);
// Debug.Log("test xright: " + data.annotationData[i].xright);
// Debug.Log("test ytop: " + data.annotationData[i].ytop);
// Debug.Log("test ybottom: " + data.annotationData[i].ybottom);
// example how to fetch model from content-publisher
// Therese, please check this is the right place in code where models should be requested
// (prob. model doesn't need to be fetched every frame for same object)
//fetchModel(data.annotationData[i].label);
}
}
// FrameObjectData temp;
// bool success = frameObjectBuffer.TryDequeue(out temp);
// if(success)
// //FrameObjectData temp = frameBuffer.Dequeue();
// {
// Debug.Log("Frame info: " + frameNumber);
// Debug.Log ("Frame info camera position: " + temp.camPos);
// Debug.Log ("Frame info camera rotation: " + temp.camRot);
// Debug.Log ("Frame info points number: " + temp.numPoints);
// Debug.Log ("Frame info points: " + temp.points.ToString());
// }
Debug.Log("Frame number annotations: " + frameNumber);
Debug.Log("Frame info camera position: " + temp.camPos);
Debug.Log("Frame info camera rotation: " + temp.camRot);
//Debug.Log ("Frame info points number: " + temp.numPoints);
Debug.Log("Frame info points: " + temp.points.ToString());
Debug.Log("test time difference: " + (Mathf.Abs((float)(temp.timestamp - timestamp))) + " frame number: " + publishedFrameNo);
//int boxCount = Mathf.Min(data.annotationData.Length, 2);
int boxCount = data.annotationData.Length;
BoxData annoData = new BoxData();
Debug.Log("box created boxdata");
annoData.frameNumber = frameNumber;
annoData.count = boxCount;
annoData.points = temp.points;
annoData.numPoints = temp.numPoints;
annoData.cam = temp.cam;
annoData.camPos = temp.camPos;
annoData.camRot = temp.camRot;
annoData.timestamp = temp.timestamp;
annoData.label = new string[boxCount];
annoData.xleft = new float[boxCount];
annoData.xright = new float[boxCount];
annoData.ytop = new float[boxCount];
annoData.ybottom = new float[boxCount];
annoData.prob = new float[boxCount];
for (int i = 0; i < boxCount; i++)
{
annoData.label[i] = data.annotationData[i].label;
annoData.xleft[i] = 1 - data.annotationData[i].xright;
annoData.xright[i] = 1 - data.annotationData[i].xleft;
annoData.ytop[i] = data.annotationData[i].ybottom;
annoData.ybottom[i] = data.annotationData[i].ytop;
annoData.prob[i] = data.annotationData[i].prob;
}
Debug.Log("Received annotations box enqueue");
//boxBufferToCalc.Enqueue(annoData);
boundingBoxBufferToCalc.Enqueue(annoData);
}
else
{
//frame object was not in the pool, lifetime expired
Debug.Log("Received annotations but frame expired");
}
}
catch (System.Exception e)
{
Debug.Log("exception caught annotations: " + e);
string debug = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("exception caught string: " + debug);
string str = "{ \"annotationData\": " + debug + "}";
Debug.Log("exception caught string with format: " + str);
}
});
openFaceFetcher_.fetchAnnotation(publishedFrameNo, delegate(string jsonArrayString) {
int frameNumber = publishedFrameNo; // storing frame number locally
string debuglog = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("Received OpenFace annotations JSON (frame " + frameNumber + "): " + debuglog);
string[] testDebug = jsonArrayString.Split(']');
string formatDebug = testDebug[0] + "]";
try{
Dictionary <int, FrameObjectData> frameObjects = frameBuffer.Dequeue();
FrameObjectData temp;
if (frameObjects.TryGetValue(frameNumber, out temp))
{
string str = "{ \"annotationData\": " + formatDebug + "}";
AnnotationData data = JsonUtility.FromJson <AnnotationData>(str);
for (int i = 0; i < data.annotationData.Length; i++)
{
//if(data.annotationData[i].prob >= 0.7f)
{
Debug.Log("openface test: " + data.annotationData.Length);
Debug.Log("openface test label: " + data.annotationData[i].label + " test xleft: " + data.annotationData[i].xleft
+ " test xright: " + data.annotationData[i].xright + " test ytop: " + (data.annotationData[i].ytop) + " test ybottom: " + (data.annotationData[i].ybottom));
// Debug.Log("test xleft: " + data.annotationData[i].xleft);
// Debug.Log("test xright: " + data.annotationData[i].xright);
// Debug.Log("test ytop: " + data.annotationData[i].ytop);
// Debug.Log("test ybottom: " + data.annotationData[i].ybottom);
}
}
//int boxCount = Mathf.Min(data.annotationData.Length, 2);
int boxCount = data.annotationData.Length;
BoxData annoData = new BoxData();
Debug.Log("box created boxdata");
annoData.frameNumber = frameNumber;
annoData.count = boxCount;
annoData.points = temp.points;
annoData.numPoints = temp.numPoints;
annoData.cam = temp.cam;
annoData.camPos = temp.camPos;
annoData.camRot = temp.camRot;
annoData.timestamp = temp.timestamp;
annoData.label = new string[boxCount];
annoData.xleft = new float[boxCount];
annoData.xright = new float[boxCount];
annoData.ytop = new float[boxCount];
annoData.ybottom = new float[boxCount];
annoData.prob = new float[boxCount];
for (int i = 0; i < boxCount; i++)
{
if (data.annotationData[i].ytop > 1)
{
data.annotationData[i].ytop = 1;
}
if (data.annotationData[i].ybottom < 0)
{
data.annotationData[i].ybottom = 0;
}
annoData.label[i] = data.annotationData[i].label;
annoData.xleft[i] = data.annotationData[i].xleft;
annoData.xright[i] = data.annotationData[i].xright;
annoData.ytop[i] = data.annotationData[i].ytop;
annoData.ybottom[i] = data.annotationData[i].ybottom;
annoData.prob[i] = 1;
}
Debug.Log("Received openface annotations box enqueue");
//boxBufferToCalc.Enqueue(annoData);
boundingBoxBufferToCalc.Enqueue(annoData);
}
else
{
//frame object was not in the pool, lifetime expired
Debug.Log("Received openface annotations but frame expired");
}
}
catch (System.Exception e)
{
Debug.Log("exception caught openface annotations: " + e);
string debug = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("exception caught openface string: " + debug);
string str = "{ \"annotationData\": " + debug + "}";
Debug.Log("exception caught openface string with format: " + str);
}
});
}
else
{
// frame was dropped by the encoder and was not published
}
}
catch (System.Exception e)
{
Debug.Log("exception caught video" + e.ToString());
}
}
