public SemanticObject GetDeepCopy()
{
SemanticObject newSO = new SemanticObject(Scores, Corners);
newSO.SetId(Id);
return(newSO);
}
private VirtualObjectBox InstanceNewSemanticObject(SemanticObject _obj)
{
Transform obj_inst = Instantiate(prefDetectedObject, _obj.Position, _obj.Rotation).transform;
obj_inst.parent = tfFrameForObjects;
VirtualObjectBox result = obj_inst.GetComponentInChildren <VirtualObjectBox>();
result.InitializeSemanticObject(_obj);
return(result);
}
public void UpdateObject(SemanticObject oldObj, SemanticObject newObj)
{
var objectFact = new RDFOntologyFact(GetClassResource(oldObj.Id));
//Update pose
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("position")),
new RDFOntologyLiteral(new RDFPlainLiteral(oldObj.Position.ToString())));
ontology.Data.AddAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("position")),
new RDFOntologyLiteral(new RDFPlainLiteral(newObj.Position.ToString())));
//Update rotation
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("rotation")),
new RDFOntologyLiteral(new RDFPlainLiteral(oldObj.Rotation.eulerAngles.ToString())));
ontology.Data.AddAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("rotation")),
new RDFOntologyLiteral(new RDFPlainLiteral(newObj.Rotation.eulerAngles.ToString())));
//Update size
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("size")),
new RDFOntologyLiteral(new RDFPlainLiteral(oldObj.Size.ToString())));
ontology.Data.AddAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("size")),
new RDFOntologyLiteral(new RDFPlainLiteral(newObj.Size.ToString())));
//Update type
ontology.Data.RemoveClassTypeRelation(objectFact, new RDFOntologyClass(GetClassResource(oldObj.ObjectClass)));
ontology.Data.AddClassTypeRelation(objectFact, new RDFOntologyClass(GetClassResource(newObj.ObjectClass)));
//Update score
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("score")),
new RDFOntologyLiteral(new RDFPlainLiteral(oldObj.Score.ToString())));
ontology.Data.AddAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("score")),
new RDFOntologyLiteral(new RDFPlainLiteral(newObj.Score.ToString())));
//Update nDetections
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("nDetections")),
new RDFOntologyLiteral(new RDFPlainLiteral(oldObj.NDetections.ToString())));
ontology.Data.AddAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("nDetections")),
new RDFOntologyLiteral(new RDFPlainLiteral(newObj.NDetections.ToString())));
}
public void RemoveSemanticObject(SemanticObject obj)
{
var objectFact = ontology.Data.SelectFact(GetNameWithURI(obj.Id));
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("position")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Position.ToString())));
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("rotation")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Rotation.eulerAngles.ToString())));
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("score")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Score.ToString())));
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("nDetections")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.NDetections.ToString())));
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("size")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Size.ToString())));
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyObjectProperty(GetResource("recordedIn")), raidFact);
if (obj.Room != null)
{
ontology.Data.RemoveAssertionRelation(objectFact, new RDFOntologyDatatypeProperty(GetResource("isIn")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Room.ToString())));
}
ontology.Data.RemoveFact(objectFact);
}
public SemanticObject AddNewDetectedObject(SemanticObject obj)
{
if (obj.Id.Equals(""))
{
string id = GetNewTimeID() + "_" + obj.ObjectClass;
obj.SetId(id);
}
var newDetectedObject = new RDFOntologyFact(GetClassResource(obj.Id));
ontology.Data.AddFact(newDetectedObject);
ontology.Data.AddClassTypeRelation(newDetectedObject, new RDFOntologyClass(GetClassResource(obj.ObjectClass)));
ontology.Data.AddAssertionRelation(newDetectedObject, new RDFOntologyDatatypeProperty(GetResource("position")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Position.ToString())));
ontology.Data.AddAssertionRelation(newDetectedObject, new RDFOntologyDatatypeProperty(GetResource("rotation")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Rotation.eulerAngles.ToString())));
ontology.Data.AddAssertionRelation(newDetectedObject, new RDFOntologyDatatypeProperty(GetResource("score")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Score.ToString())));
ontology.Data.AddAssertionRelation(newDetectedObject, new RDFOntologyDatatypeProperty(GetResource("nDetections")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.NDetections.ToString())));
ontology.Data.AddAssertionRelation(newDetectedObject, new RDFOntologyDatatypeProperty(GetResource("size")), new RDFOntologyLiteral(new RDFPlainLiteral(obj.Size.ToString())));
ontology.Data.AddAssertionRelation(newDetectedObject, new RDFOntologyObjectProperty(GetResource("recordedIn")), raidFact);
return(obj);
}
private VirtualObjectBox Matching(SemanticObject obj1, ICollection <VirtualObjectBox> listToCompare)
{
VirtualObjectBox match = null;
float best_score = 999;
foreach (VirtualObjectBox vob in listToCompare)
{
List <SemanticObject.Corner> order = YNN(vob.semanticObject.Corners, obj1.Corners);
float score = CalculateMatchingScore(vob.semanticObject.Corners, order);
if (((score < thresholdMatchSameSype && obj1.ObjectClass.Equals(vob.semanticObject.ObjectClass)) ||
(score < thresholdMatchDiffType && !obj1.ObjectClass.Equals(vob.semanticObject.ObjectClass))) && score < best_score)
{
obj1.SetNewCorners(order);
match = vob;
best_score = score;
//Debug.Log("Union: " + obj1.Id+ " con: " + vob.semanticObject.Id + ", por distancia: " + score);
}//else { Debug.Log("NO Union: " + obj1.Id+ " con: " + vob.semanticObject.Id + ", por distancia: " + score); }
}
return(match);
}
public float GetProbabilityCategory(string categoryRoom, List <SemanticObject> detectedClasses, List <SemanticObject> previousClass)
{
if (detectedClasses.Count == 0)
{
return(0);
}
float probability = 0;
SemanticObject detection = detectedClasses[0];
detectedClasses.Remove(detection);
previousClass.Add(detection);
foreach (string classObject in objectClassInRooms)
{
var total = (objectClassInRooms.Count - 1) * 0.1f;
float P1 = 0.1f / total;
if (classObject.Equals(detection.ObjectClass))
{
P1 = (float)detection.Score / total;
}
float P2 = 0f;
if (detectedClasses.Count == 0)
{
P2 = GetProbabilityCategory(categoryRoom, previousClass);
}
else
{
P2 = GetProbabilityCategory(categoryRoom, detectedClasses, previousClass);
}
probability += P1 * P2;
}
return(probability);
}
private IEnumerator ProcessMsgs()
{
Rect rect = new Rect(0, 0, 640, 480);
Texture2D image = new Texture2D(640, 480, TextureFormat.RGB24, false);
RenderTexture renderTextureMask = new RenderTexture(640, 480, 24);
while (Application.isPlaying)
{
if (processingQueue.Count > 0)
{
DetectionArrayMsg _detections = processingQueue.Dequeue();
//Get view previous detections from bbCamera located in the origin
bbCamera.transform.position = _detections.origin.GetPositionUnity();
bbCamera.transform.rotation = _detections.origin.GetRotationUnity() * Quaternion.Euler(0f, 90f, 0f);
Dictionary <VirtualObjectBox, int> virtualObjectBoxInRange = new Dictionary <VirtualObjectBox, int>();
List <VirtualObjectBox> visibleVirtualObjectBox = new List <VirtualObjectBox>();
//int n = 0;
bool newIteration = true;
while (newIteration)
{
newIteration = false;
bbCamera.targetTexture = renderTextureMask;
bbCamera.Render();
RenderTexture.active = renderTextureMask;
image.ReadPixels(rect, 0, 0);
image.Apply();
var q = from x in image.GetPixels()
group x by x into g
let count = g.Count()
orderby count descending
select new { Value = g.Key, Count = count };
foreach (var xx in q)
{
if (boxColors.ContainsKey(xx.Value))
{
var vob = boxColors[xx.Value];
vob.gameObject.SetActive(false);
visibleVirtualObjectBox.Add(vob);
var distance = Vector3.Distance(vob.semanticObject.Position, bbCamera.transform.position);
if (deepRange.y >= distance && distance >= deepRange.x)
{
virtualObjectBoxInRange.Add(vob, xx.Count);
}
newIteration = true;
//Debug.Log("Value: " + xx.Value + " Count: " + xx.Count);
}
}
bbCamera.targetTexture = null;
RenderTexture.active = null; //Clean
//Destroy(renderTextureMask); //Free memory
if (q.Count() == 1)
{
break;
}
//n++;
}
foreach (VirtualObjectBox vob in visibleVirtualObjectBox)
{
vob.gameObject.SetActive(true);
}
List <VirtualObjectBox> detectedVirtualObjectBox = new List <VirtualObjectBox>();
foreach (DetectionMsg detection in _detections.detections)
{
SemanticObject virtualObject = new SemanticObject(detection.GetScores(),
detection.GetCorners(),
detection.occluded_corners);
//Check the type object is in the ontology
if (!OntologySystem.instance.CheckInteresObject(virtualObject.ObjectClass))
{
Log(virtualObject.ObjectClass + " - detected but it is not in the ontology", LogLevel.Error, true);
continue;
}
//Checks the distance to the object
var distance = Vector3.Distance(virtualObject.Position, bbCamera.transform.position);
if (distance < deepRange.x || distance > deepRange.y)
{
Log(virtualObject.ObjectClass + " - detected but it is not in deep range. Distance: " + distance, LogLevel.Normal, true);
continue;
}
//Insertion detection into the ontology
virtualObject = OntologySystem.instance.AddNewDetectedObject(virtualObject);
//Try to get a match
VirtualObjectBox match = Matching(virtualObject, virtualObjectBoxInRange.Keys);
//Match process
if (match != null)
{
match.NewDetection(virtualObject);
detectedVirtualObjectBox.Add(match);
}
else
{
Log("New object detected: " + virtualObject.ToString(), LogLevel.Developer);
m_objectDetected.Add(virtualObject);
VirtualObjectBox nvob = InstanceNewSemanticObject(virtualObject);
detectedVirtualObjectBox.Add(nvob);
}
nDetections++;
}
List <VirtualObjectBox> inRange = virtualObjectBoxInRange.Keys.ToList();
for (int i = 0; i < inRange.Count - 1; i++)
{
if (inRange[i] != null)
{
VirtualObjectBox match = null;
match = Matching(inRange[i].semanticObject, inRange.GetRange(i + 1, inRange.Count - i - 1));
if (match != null)
{
match.NewDetection(inRange[i].semanticObject);
inRange[i].RemoveVirtualBox();
}
}
}
detectedVirtualObjectBox.ForEach(dvob => virtualObjectBoxInRange.Remove(dvob));
foreach (KeyValuePair <VirtualObjectBox, int> o in virtualObjectBoxInRange)
{
if (o.Value > minPixelsMask)
{
o.Key.NewDetection(null);
}
}
}
yield return(null);
}
}
public void RemoveSemanticObjectUnion(SemanticObject father, SemanticObject child)
{
ontology.Data.RemoveAssertionRelation(new RDFOntologyFact(GetClassResource(child.Id)),
new RDFOntologyObjectProperty(GetResource("isPartOf")),
new RDFOntologyFact(GetClassResource(father.Id)));
}
public void JoinSemanticObject(SemanticObject father, SemanticObject child)
{
JoinSemanticObject(father.Id, child.Id);
}
public void InitializeSemanticObject(SemanticObject _semanticObject)
{
semanticObject = _semanticObject;
}
public void NewDetection(SemanticObject newDetection)
{
semanticObject.NewDetection(newDetection);
UpdateObject();
}
public void NewDetection(SemanticObject newDetection)
{
if (NDetections > 1)
{
OntologySystem.instance.RemoveSemanticObject(this);
}
if (newDetection != null)
{
if (Defined)
{
if (newDetection.Defined)
{
nOccludedDetection = 0;
float nPreviousDetections = Mathf.Min(NDetections, 20f);
for (int i = 0; i < Corners.Count; i++)
{
Corners[i] = new Corner((nPreviousDetections * Corners[i].position + newDetection.Corners[i].position) / (nPreviousDetections + 1f), false);
}
}
else
{
nOccludedDetection++;
if (nOccludedDetection > 20f)
{
nOccludedDetection = 0;
for (int i = 0; i < Corners.Count; i++)
{
Corners[i] = new Corner((Corners[i].position + newDetection.Corners[i].position) / 2, newDetection.Corners[i].occluded);
}
}
}
}
else
{
if (newDetection.Defined)
{
Corners = newDetection.Corners;
}
else
{
JointBB(newDetection.Corners);
}
}
// Update scores
foreach (KeyValuePair <string, float> s in newDetection.Scores)
{
Scores[CultureInfo.InvariantCulture.TextInfo.ToTitleCase(s.Key).Replace(" ", "_")] += s.Value;
}
OntologySystem.instance.JoinSemanticObject(this, newDetection);
NDetections += newDetection.NDetections;
UpdateProperties();
// Update ontology
if (NDetections == 2)
{
string oldId = Id;
Id = "";
Id = OntologySystem.instance.AddNewDetectedObject(this).Id;
OntologySystem.instance.JoinSemanticObject(Id, oldId);
OntologySystem.instance.JoinSemanticObject(Id, newDetection.Id);
}
else
{
OntologySystem.instance.AddNewDetectedObject(this);
OntologySystem.instance.JoinSemanticObject(Id, newDetection.Id);
}
}
else
{
Scores["Other"] += noDetectionProb;
float defaultValue = (1f - noDetectionProb) / (Scores.Count - 1);
foreach (string key in OntologySystem.instance.objectClassInOntology)
{
Scores[CultureInfo.InvariantCulture.TextInfo.ToTitleCase(key).Replace(" ", "_")] += defaultValue;
}
NDetections++;
UpdateProperties();
// Update ontology
OntologySystem.instance.AddNewDetectedObject(this);
}
}
