public void OnImageAvailable(TextureReaderApi.ImageFormatType format, int width, int height, IntPtr pixelBuffer, int bufferSize)
{
try
{
System.DateTime current = System.DateTime.Now;
long elapsedTicks = current.Ticks - begin_.Ticks;
System.TimeSpan elapsedSpan = new System.TimeSpan(elapsedTicks);
timestamp_ = elapsedSpan.TotalSeconds;
Debug.LogFormat("pushing frame {0}x{1} to NDN-RTC...", width, height);
FrameInfo finfo = NdnRtc.videoStream.processIncomingFrame(format, width, height, pixelBuffer, bufferSize);
int publishedFrameNo = finfo.playbackNo_;
if (publishedFrameNo >= 0) // frame was not dropped by the encoder and was published
{
Debug.LogFormat("create frame object #{0}, ts {1}, pos {2}, rot {3}, cam {4}",
publishedFrameNo, timestamp_, Frame.Pose.position, Frame.Pose.rotation, camForCalcThread_.ToString());
frameMgr_.CreateFrameObject(publishedFrameNo, timestamp_, Frame.Pose.position, Frame.Pose.rotation, camForCalcThread_);
frameBuffer_.Enqueue(frameMgr_.frameObjects);
// spawn fetching task for annotations of this frame
foreach (var fetcher in annotationFetchers_)
{
spawnAnnotationFetchingTask(publishedFrameNo, fetcher, 0.6f, performSemanticDbQuery);
}
}
}
catch (System.Exception e)
{
Debug.LogExceptionFormat(e, "in OnImageAvailable call");
}
}
public void OnImageAvailable(TextureReaderApi.ImageFormatType format, int width, int height, IntPtr pixelBuffer, int bufferSize)
{
try{
System.DateTime current = System.DateTime.Now;
long elapsedTicks = current.Ticks - begin.Ticks;
System.TimeSpan elapsedSpan = new System.TimeSpan(elapsedTicks);
timestamp = elapsedSpan.TotalSeconds;
//Debug.Log("before call to ndnrtc");
int publishedFrameNo = NdnRtc.videoStream.processIncomingFrame(format, width, height, pixelBuffer, bufferSize);
Debug.Log("Published frame number: " + publishedFrameNo);
if (publishedFrameNo >= 0)
{
Debug.Log("create frame object frame number: " + publishedFrameNo);
Debug.Log("create frame object timestamp: " + timestamp);
Debug.Log("create frame object position: " + Frame.Pose.position);
Debug.Log("create frame object rotation: " + Frame.Pose.rotation);
Debug.Log("create frame object camera: " + camForCalcThread.ToString());
frameMgr.CreateFrameObject(publishedFrameNo, timestamp, Frame.Pose.position, Frame.Pose.rotation, camForCalcThread);
//frameMgr.CreateFrameObject (imgBuffer, publishedFrameNo, timestamp, Vector3.zero, Quaternion.identity, offset.m_uOffset, offset.m_vOffset, camForCalcThread);
//frameObjectBuffer.Enqueue (frameMgr.frameObjects [publishedFrameNo]);
frameBuffer.Enqueue(frameMgr.frameObjects);
Debug.Log("frame buffer enqueue: " + publishedFrameNo);
// spawn fetching task for annotations of this frame
// once successfully received, delegate callback will be called
aFetcher_.fetchAnnotation(publishedFrameNo, delegate(string jsonArrayString) {
int frameNumber = publishedFrameNo; // storing frame number locally
string debuglog = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("Received annotations JSON (frame " + frameNumber + "): " + debuglog);
//Debug.Log("annotations string length: " + jsonArrayString.Length);
string[] testDebug = jsonArrayString.Split(']');
string formatDebug = testDebug[0] + "]";
try{
Dictionary <int, FrameObjectData> frameObjects = frameBuffer.Dequeue();
FrameObjectData temp;
if (frameObjects.TryGetValue(frameNumber, out temp))
{
//AnnotationData[] data = JsonHelper.FromJson<AnnotationData>(jsonArrayString);
//try to print out how many characters the jsonArrayString has
string str = "{ \"annotationData\": " + formatDebug + "}";
AnnotationData data = JsonUtility.FromJson <AnnotationData>(str);
for (int i = 0; i < data.annotationData.Length; i++)
{
if (data.annotationData[i].prob >= 0.5f)
{
Debug.Log("test: " + data.annotationData.Length);
Debug.Log("test label: " + data.annotationData[i].label + " test xleft: " + data.annotationData[i].xleft
+ " test xright: " + data.annotationData[i].xright + " test ytop: " + (1 - data.annotationData[i].ytop) + " test ybottom: " + (1 - data.annotationData[i].ybottom));
// Debug.Log("test xleft: " + data.annotationData[i].xleft);
// Debug.Log("test xright: " + data.annotationData[i].xright);
// Debug.Log("test ytop: " + data.annotationData[i].ytop);
// Debug.Log("test ybottom: " + data.annotationData[i].ybottom);
// example how to fetch model from content-publisher
// Therese, please check this is the right place in code where models should be requested
// (prob. model doesn't need to be fetched every frame for same object)
//fetchModel(data.annotationData[i].label);
}
}
// FrameObjectData temp;
// bool success = frameObjectBuffer.TryDequeue(out temp);
// if(success)
// //FrameObjectData temp = frameBuffer.Dequeue();
// {
// Debug.Log("Frame info: " + frameNumber);
// Debug.Log ("Frame info camera position: " + temp.camPos);
// Debug.Log ("Frame info camera rotation: " + temp.camRot);
// Debug.Log ("Frame info points number: " + temp.numPoints);
// Debug.Log ("Frame info points: " + temp.points.ToString());
// }
Debug.Log("Frame number annotations: " + frameNumber);
Debug.Log("Frame info camera position: " + temp.camPos);
Debug.Log("Frame info camera rotation: " + temp.camRot);
//Debug.Log ("Frame info points number: " + temp.numPoints);
Debug.Log("Frame info points: " + temp.points.ToString());
Debug.Log("test time difference: " + (Mathf.Abs((float)(temp.timestamp - timestamp))) + " frame number: " + publishedFrameNo);
//int boxCount = Mathf.Min(data.annotationData.Length, 2);
int boxCount = data.annotationData.Length;
BoxData annoData = new BoxData();
Debug.Log("box created boxdata");
annoData.frameNumber = frameNumber;
annoData.count = boxCount;
annoData.points = temp.points;
annoData.numPoints = temp.numPoints;
annoData.cam = temp.cam;
annoData.camPos = temp.camPos;
annoData.camRot = temp.camRot;
annoData.timestamp = temp.timestamp;
annoData.label = new string[boxCount];
annoData.xleft = new float[boxCount];
annoData.xright = new float[boxCount];
annoData.ytop = new float[boxCount];
annoData.ybottom = new float[boxCount];
annoData.prob = new float[boxCount];
for (int i = 0; i < boxCount; i++)
{
annoData.label[i] = data.annotationData[i].label;
annoData.xleft[i] = 1 - data.annotationData[i].xright;
annoData.xright[i] = 1 - data.annotationData[i].xleft;
annoData.ytop[i] = data.annotationData[i].ybottom;
annoData.ybottom[i] = data.annotationData[i].ytop;
annoData.prob[i] = data.annotationData[i].prob;
}
Debug.Log("Received annotations box enqueue");
//boxBufferToCalc.Enqueue(annoData);
boundingBoxBufferToCalc.Enqueue(annoData);
}
else
{
//frame object was not in the pool, lifetime expired
Debug.Log("Received annotations but frame expired");
}
}
catch (System.Exception e)
{
Debug.Log("exception caught annotations: " + e);
string debug = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("exception caught string: " + debug);
string str = "{ \"annotationData\": " + debug + "}";
Debug.Log("exception caught string with format: " + str);
}
});
openFaceFetcher_.fetchAnnotation(publishedFrameNo, delegate(string jsonArrayString) {
int frameNumber = publishedFrameNo; // storing frame number locally
string debuglog = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("Received OpenFace annotations JSON (frame " + frameNumber + "): " + debuglog);
string[] testDebug = jsonArrayString.Split(']');
string formatDebug = testDebug[0] + "]";
try{
Dictionary <int, FrameObjectData> frameObjects = frameBuffer.Dequeue();
FrameObjectData temp;
if (frameObjects.TryGetValue(frameNumber, out temp))
{
string str = "{ \"annotationData\": " + formatDebug + "}";
AnnotationData data = JsonUtility.FromJson <AnnotationData>(str);
for (int i = 0; i < data.annotationData.Length; i++)
{
//if(data.annotationData[i].prob >= 0.7f)
{
Debug.Log("openface test: " + data.annotationData.Length);
Debug.Log("openface test label: " + data.annotationData[i].label + " test xleft: " + data.annotationData[i].xleft
+ " test xright: " + data.annotationData[i].xright + " test ytop: " + (data.annotationData[i].ytop) + " test ybottom: " + (data.annotationData[i].ybottom));
// Debug.Log("test xleft: " + data.annotationData[i].xleft);
// Debug.Log("test xright: " + data.annotationData[i].xright);
// Debug.Log("test ytop: " + data.annotationData[i].ytop);
// Debug.Log("test ybottom: " + data.annotationData[i].ybottom);
}
}
//int boxCount = Mathf.Min(data.annotationData.Length, 2);
int boxCount = data.annotationData.Length;
BoxData annoData = new BoxData();
Debug.Log("box created boxdata");
annoData.frameNumber = frameNumber;
annoData.count = boxCount;
annoData.points = temp.points;
annoData.numPoints = temp.numPoints;
annoData.cam = temp.cam;
annoData.camPos = temp.camPos;
annoData.camRot = temp.camRot;
annoData.timestamp = temp.timestamp;
annoData.label = new string[boxCount];
annoData.xleft = new float[boxCount];
annoData.xright = new float[boxCount];
annoData.ytop = new float[boxCount];
annoData.ybottom = new float[boxCount];
annoData.prob = new float[boxCount];
for (int i = 0; i < boxCount; i++)
{
if (data.annotationData[i].ytop > 1)
{
data.annotationData[i].ytop = 1;
}
if (data.annotationData[i].ybottom < 0)
{
data.annotationData[i].ybottom = 0;
}
annoData.label[i] = data.annotationData[i].label;
annoData.xleft[i] = data.annotationData[i].xleft;
annoData.xright[i] = data.annotationData[i].xright;
annoData.ytop[i] = data.annotationData[i].ytop;
annoData.ybottom[i] = data.annotationData[i].ybottom;
annoData.prob[i] = 1;
}
Debug.Log("Received openface annotations box enqueue");
//boxBufferToCalc.Enqueue(annoData);
boundingBoxBufferToCalc.Enqueue(annoData);
}
else
{
//frame object was not in the pool, lifetime expired
Debug.Log("Received openface annotations but frame expired");
}
}
catch (System.Exception e)
{
Debug.Log("exception caught openface annotations: " + e);
string debug = jsonArrayString.Replace(System.Environment.NewLine, " ");
Debug.Log("exception caught openface string: " + debug);
string str = "{ \"annotationData\": " + debug + "}";
Debug.Log("exception caught openface string with format: " + str);
}
});
}
else
{
// frame was dropped by the encoder and was not published
}
}
catch (System.Exception e)
{
Debug.Log("exception caught video" + e.ToString());
}
}