void Start()
{
TextureReaderComponent.OnImageAvailableCallback += OnImageAvailable;
//timestamp = gameObject.GetComponent<TangoARScreen> ().m_screenUpdateTime;
frameMgr = GameObject.FindObjectOfType <FramePoolManager>();
frameNumber = 0;
//frameObjects = new Dictionary<long, FrameObjectData> ();
boxMgr = GameObject.FindObjectOfType <BoundingBoxPoolManager>();
timestamp = 0;
frameBuffer = new ConcurrentQueue <Dictionary <int, FrameObjectData> > ();
boundingBoxBufferToCalc = new ConcurrentQueue <BoxData> ();
boundingBoxBufferToUpdate = new ConcurrentQueue <CreateBoxData> ();
boxData = new List <CreateBoxData> ();
// frameObjectBuffer = new RingBuffer<FrameObjectData> (100000);
// boxBufferToCalc = new RingBuffer<BoxData> (100000);
// boxBufferToUpdate = new RingBuffer<CreateBoxData> (100000);
camForCalcThread = GameObject.Find("Camera").GetComponent("Camera") as Camera;
calc = new Thread(calculationsForBoundingBox);
calc.Start();
labelColors = new Dictionary <string, Color> ();
kalman = new Dictionary <string, IKalmanWrapper> ();
colors = new List <Color> {
new Color(255f / 255, 109f / 255, 124f / 255),
new Color(119f / 255, 231f / 255, 255f / 255),
new Color(82f / 255, 255f / 255, 127f / 255),
new Color(252f / 255, 187f / 255, 255f / 255),
new Color(255f / 255, 193f / 255, 130f / 255)
};
// @Therese - these need to be moved somewhere to a higher-level entity as
// configuration parameters (may be changed frequently during testing)
string rootPrefix = "/icear/user";
string userId = "peter"; // "mobile-terminal0";
string serviceType = "object_recognizer";
string serviceInstance = "yolo"; // "yolo";
string serviceInstance2 = "openface"; // "yolo";
NdnRtc.Initialize(rootPrefix, userId);
faceProcessor_ = new FaceProcessor();
faceProcessor_.start();
assetFetcher_ = new AssetBundleFetcher(faceProcessor_);
string servicePrefix = rootPrefix + "/" + userId + "/" + serviceType;
// AnnotationsFetcher instance might also be a singleton class
// and initialized/created somewhere else. here just as an example
aFetcher_ = new AnnotationsFetcher(faceProcessor_, servicePrefix, serviceInstance);
openFaceFetcher_ = new AnnotationsFetcher(faceProcessor_, servicePrefix, serviceInstance2);
// setup CNL logging
ILOG.J2CsMapping.Util.Logging.Logger.getLogger("").setLevel(ILOG.J2CsMapping.Util.Logging.Level.FINE);
ILOG.J2CsMapping.Util.Logging.Logger.Write = delegate(string message) { Debug.Log(System.DateTime.Now + ": " + message); };
}
private void spawnAnnotationFetchingTask(int frameNo, AnnotationsFetcher fetcher,
float th,
OnAnnotationFetched onAnnotationFetched = null)
{
fetcher.fetchAnnotation(frameNo, delegate(string jsonArrayString)
{
var now = System.DateTime.Now;
string fetcherName = fetcher.getServiceName();
float threshold = th;
int frameNumber = frameNo;
string debugString = jsonArrayString.Replace(System.Environment.NewLine, "");
Debug.LogFormat((ILogComponent)this, "fetched {3} annotations JSON for {0}, length {1}: {2}",
frameNumber, jsonArrayString.Length, debugString, fetcherName);
try {
if (onAnnotationFetched != null)
{
onAnnotationFetched(now, jsonArrayString);
}
}
catch (System.Exception e) {
Debug.LogExceptionFormat(e, "caught exception while callback, annotation: {0}",
debugString.Substring(0, 100));
}
try
{
Dictionary <int, FrameObjectData> frameObjects = frameBuffer_.Dequeue();
FrameObjectData frameObjectData;
if (frameObjects.TryGetValue(frameNumber, out frameObjectData))
{
// some pre-processing for the received string
// TBD: this needs to be replaced with JSON deserialization, need to update edge too
string[] testDebug = jsonArrayString.Split(']');
string formatDebug = testDebug[0] + "]";
string str = "{ \"annotationData\": " + formatDebug + "}";
AnnotationData data = JsonUtility.FromJson <AnnotationData>(str);
#if DEVELOPMENT_BUILD
for (int i = 0; i < data.annotationData.Length; i++)
{
Debug.LogFormat((ILogComponent)this,
"{7} annotation {0}: label {1} prob {2} xleft {3} xright {4} ytop {5} ybottom {6}",
i, data.annotationData[i].label, data.annotationData[i].prob,
data.annotationData[i].xleft, data.annotationData[i].xright,
data.annotationData[i].ytop, data.annotationData[i].ybottom,
fetcherName);
}
#endif
Debug.LogFormat((ILogComponent)this,
"processing {5} annotation for frame #{0}, cam pos {1}, cam rot {2}, points {3}, lifetime {4} sec",
frameNumber, frameObjectData.camPos, frameObjectData.camRot, frameObjectData.points,
Mathf.Abs((float)(frameObjectData.timestamp - timestamp_)), fetcherName);
// filter out annotations with probability below threshold
List <AnnotationData.ArrayEntry> filteredAnnotations = new List <AnnotationData.ArrayEntry>();
for (int i = 0; i < data.annotationData.Length; ++i)
{
if (data.annotationData[i].prob >= threshold)
{
filteredAnnotations.Add(data.annotationData[i]);
}
}
Debug.LogFormat("{0} annotations above threshold (filtered out {0} annotations)",
filteredAnnotations.Count, data.annotationData.Length - filteredAnnotations.Count);
if (filteredAnnotations.Count > 0)
{
int boxCount = filteredAnnotations.Count;
// TBD: this needs to be pooled
BoxData boxData = new BoxData();
// TBD: this needs to be hidden in a method/constructor
boxData.frameNumber = frameNumber;
boxData.count = boxCount;
boxData.points = frameObjectData.points;
boxData.numPoints = frameObjectData.numPoints;
boxData.cam = frameObjectData.cam;
boxData.camPos = frameObjectData.camPos;
boxData.camRot = frameObjectData.camRot;
boxData.timestamp = frameObjectData.timestamp;
boxData.label = new string[boxCount];
boxData.xleft = new float[boxCount];
boxData.xright = new float[boxCount];
boxData.ytop = new float[boxCount];
boxData.ybottom = new float[boxCount];
boxData.prob = new float[boxCount];
for (int i = 0; i < boxCount; i++)
{
// safety checks
if (data.annotationData[i].ytop > 1)
{
data.annotationData[i].ytop = 1;
}
if (data.annotationData[i].ybottom < 0)
{
data.annotationData[i].ybottom = 0;
}
boxData.label[i] = filteredAnnotations[i].label;
// since we flipped the image, flip bbox coordinates again
// only y coordinates get flipped though, and it works.
// I think this has something to do with how Viewport coordinates
// are represented...
boxData.xleft[i] = filteredAnnotations[i].xleft;
boxData.xright[i] = filteredAnnotations[i].xright;
boxData.ytop[i] = 1 - filteredAnnotations[i].ytop;
boxData.ybottom[i] = 1 - filteredAnnotations[i].ybottom;
boxData.prob[i] = filteredAnnotations[i].prob;
Debug.LogFormat((ILogComponent)this, "will render {6}: {0} xleft {1} xright {2} ytop {3} ybottom {4} prob {5}",
boxData.label[i], boxData.xleft[i], boxData.xright[i],
boxData.ytop[i], boxData.ybottom[i], boxData.prob[i],
fetcherName);
}
//boxBufferToCalc.Enqueue(annoData);
boundingBoxBufferToCalc_.Enqueue(boxData);
}
}
else
{
// frame object was not in the pool, lifetime expired
Debug.LogFormat((ILogComponent)this, "received {0} annotations but frame expired",
fetcherName);
}
}
catch (System.Exception e)
{
Debug.LogExceptionFormat(e, "while parsing {0} annotation {1}...",
fetcherName, debugString.Substring(0, 100));
}
});
}
