// TBD: before perofrming a query, we shall wait a little to accumulate
// annotations from different engines for the same frame (i.e. YOLO and OpenFace, etc.).
// This, however, will require aggregating annotaitons based on frame number and
// creating a combined json string
private void performSemanticDbQuery(System.DateTime fetchTimestamp, string jsonArrayString)
{
// check if it's time to query Semantic DB...
if ((float)(fetchTimestamp - lastDbQuery_).TotalSeconds >= (1f / dbQueryRate_))
{
lastDbQuery_ = fetchTimestamp;
dbController_.runQuery(jsonArrayString,
delegate(DbReply reply, string errorMessage)
{
if (reply != null)
{
Debug.LogFormat(dbController_, "got reply from DB. entries {0} ", reply.entries.Length);
foreach (var entry in reply.entries)
{
NdnRtc.fetch(entry.frameName, NdnRtc.videoStream,
delegate(FrameInfo fi, int w, int h, byte[] argbBuffer)
{
Debug.LogFormat("[ff-task]: succesfully fetched frame {0}", fi.ndnName_);
imageController_.enqueueFrame(new FetchedUIFrame(argbBuffer, fi.timestamp_, entry.simLevel));
},
delegate(string frameName)
{
Debug.LogFormat("[ff-task]: failed to fetch {0}", frameName);
});
}
}
else
{
Debug.ErrorFormat(dbController_, "db request error {0} ", errorMessage);
}
});
} // if time for DB query
}
void Start()
{
TextureReaderComponent.OnImageAvailableCallback += OnImageAvailable;
//timestamp = gameObject.GetComponent<TangoARScreen> ().m_screenUpdateTime;
frameMgr = GameObject.FindObjectOfType <FramePoolManager>();
frameNumber = 0;
//frameObjects = new Dictionary<long, FrameObjectData> ();
boxMgr = GameObject.FindObjectOfType <BoundingBoxPoolManager>();
timestamp = 0;
frameBuffer = new ConcurrentQueue <Dictionary <int, FrameObjectData> > ();
boundingBoxBufferToCalc = new ConcurrentQueue <BoxData> ();
boundingBoxBufferToUpdate = new ConcurrentQueue <CreateBoxData> ();
boxData = new List <CreateBoxData> ();
// frameObjectBuffer = new RingBuffer<FrameObjectData> (100000);
// boxBufferToCalc = new RingBuffer<BoxData> (100000);
// boxBufferToUpdate = new RingBuffer<CreateBoxData> (100000);
camForCalcThread = GameObject.Find("Camera").GetComponent("Camera") as Camera;
calc = new Thread(calculationsForBoundingBox);
calc.Start();
labelColors = new Dictionary <string, Color> ();
kalman = new Dictionary <string, IKalmanWrapper> ();
colors = new List <Color> {
new Color(255f / 255, 109f / 255, 124f / 255),
new Color(119f / 255, 231f / 255, 255f / 255),
new Color(82f / 255, 255f / 255, 127f / 255),
new Color(252f / 255, 187f / 255, 255f / 255),
new Color(255f / 255, 193f / 255, 130f / 255)
};
// @Therese - these need to be moved somewhere to a higher-level entity as
// configuration parameters (may be changed frequently during testing)
string rootPrefix = "/icear/user";
string userId = "peter"; // "mobile-terminal0";
string serviceType = "object_recognizer";
string serviceInstance = "yolo"; // "yolo";
string serviceInstance2 = "openface"; // "yolo";
NdnRtc.Initialize(rootPrefix, userId);
faceProcessor_ = new FaceProcessor();
faceProcessor_.start();
assetFetcher_ = new AssetBundleFetcher(faceProcessor_);
string servicePrefix = rootPrefix + "/" + userId + "/" + serviceType;
// AnnotationsFetcher instance might also be a singleton class
// and initialized/created somewhere else. here just as an example
aFetcher_ = new AnnotationsFetcher(faceProcessor_, servicePrefix, serviceInstance);
openFaceFetcher_ = new AnnotationsFetcher(faceProcessor_, servicePrefix, serviceInstance2);
// setup CNL logging
ILOG.J2CsMapping.Util.Logging.Logger.getLogger("").setLevel(ILOG.J2CsMapping.Util.Logging.Level.FINE);
ILOG.J2CsMapping.Util.Logging.Logger.Write = delegate(string message) { Debug.Log(System.DateTime.Now + ": " + message); };
}
void Start()
{
Debug.Message("initializing OnCameraFrame...");
try
{
renderBoundingBoxes = true;
timestamp_ = 0;
Debug.Log("adding callback for capturing camera frames");
TextureReaderComponent.OnImageAvailableCallback += OnImageAvailable;
frameMgr_ = GameObject.FindObjectOfType <FramePoolManager>();
boxMgr_ = GameObject.FindObjectOfType <BoundingBoxPoolManager>();
Debug.Log("creating structures for frames and bounding boxes processing");
frameBuffer_ = new ConcurrentQueue <Dictionary <int, FrameObjectData> >();
boundingBoxBufferToCalc_ = new ConcurrentQueue <BoxData>();
boundingBoxBufferToUpdate_ = new ConcurrentQueue <CreateBoxData>();
boxData_ = new List <CreateBoxData>();
camForCalcThread_ = GameObject.Find("Camera").GetComponent("Camera") as Camera;
//calc_ = new Thread(calculationsForBoundingBox);
//calc_.Start();
labelColors_ = new Dictionary <string, Color>();
kalman_ = new Dictionary <string, IKalmanWrapper>();
colors_ = new List <Color> {
new Color(255f / 255, 109f / 255, 124f / 255),
new Color(119f / 255, 231f / 255, 255f / 255),
new Color(82f / 255, 255f / 255, 127f / 255),
new Color(252f / 255, 187f / 255, 255f / 255),
new Color(255f / 255, 193f / 255, 130f / 255)
};
Debug.Log("initializing semantic db controller");
lastDbQuery_ = System.DateTime.Now;
lastKeyFrame_ = System.DateTime.Now;
dbQueryRate_ = 0.5f; // once every 2 seconds
dbController_ = new SemanticDbController("http://131.179.142.7:8888/query");
Debug.Log("initializing NDN modules");
// @Therese - these need to be moved somewhere to a higher-level entity as
// configuration parameters (may be changed frequently during testing)
string rootPrefix = "/icear/user";
string userId = "peter"; // "mobile-terminal0";
string serviceType = "object_recognizer";
string [] edgeServices = { "yolo", "openface" }; // these must be unique!
NdnRtc.Initialize(rootPrefix, userId);
faceProcessor_ = new FaceProcessor();
faceProcessor_.start();
assetFetcher_ = new AssetBundleFetcher(faceProcessor_);
string servicePrefix = rootPrefix + "/" + userId + "/" + serviceType;
annotationFetchers_ = new List <AnnotationsFetcher>();
foreach (var service in edgeServices)
{
Debug.LogFormat("initializing annotations fetcher for {0}...", service);
annotationFetchers_.Add(new AnnotationsFetcher(faceProcessor_, servicePrefix, service));
}
// setup CNL logging
//ILOG.J2CsMapping.Util.Logging.Logger.getLogger("").setLevel(ILOG.J2CsMapping.Util.Logging.Level.FINE);
//ILOG.J2CsMapping.Util.Logging.Logger.Write = delegate (string message) { Debug.Log(System.DateTime.Now + ": " + message); };
}
catch (System.Exception e)
{
Debug.LogExceptionFormat(e, "while initializing");
}
}