private async Task <Guid> CreatePerson(List <FaceSendInfo> facesInfo, Guid newPersonID, Face f)
{
AddPersistedFaceResult result = null;
try
{
//No candidate we are going to create new person and set candidate to be same as newly created person with confidence of 100%
var name = f.FaceAttributes.Gender + "-" + f.FaceAttributes.Age + "-" + newPersonID.ToString();
newPersonID = (await FaceServiceHelper.CreatePersonWithResultAsync(groupId, name)).PersonId;
var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault();
fi.canid = newPersonID.ToString();
fi.canname = name;
fi.canconf = 1;
await FaceServiceHelper.AddPersonFaceAsync(groupId, newPersonID, await this.GetImageStreamCallback(), "", f.FaceRectangle);
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Problem adding face to group");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem adding face to group");
}
}
return(newPersonID);
}
public static async Task ResetFaceLists()
{
faceLists = new Dictionary <string, FaceListInfo>();
try
{
//IEnumerable<FaceListMetadata> metadata = await FaceServiceHelper.GetFaceListsAsync(FaceListsUserDataFilter);
IEnumerable <FaceListMetadata> metadata = await FaceServiceHelper.GetFaceListsAsync();
foreach (var item in metadata)
{
await FaceServiceHelper.DeleteFaceListAsync(item.FaceListId);
}
//We want to delete also local persisted DB
using (var db = new KioskDBContext())
{
db.SimilarFaces.RemoveRange(db.SimilarFaces);
db.SaveChanges();
}
}
catch (Exception e)
{
ErrorTrackingHelper.TrackException(e, "Error resetting face lists");
}
}
public static async Task DeleteFaceFromFaceList(string faceIdToDelete)
{
if (faceLists == null)
{
await Initialize();
}
var faceListId = faceLists.FirstOrDefault().Key;
await FaceServiceHelper.DeleteFaceFromFaceListAsync(faceListId, new Guid(faceIdToDelete));
}
public static async Task ResetFaceLists()
{
faceLists = new Dictionary <string, FaceListInfo>();
try
{
IEnumerable <FaceList> metadata = await FaceServiceHelper.GetFaceListsAsync(FaceListsUserDataFilter);
foreach (var item in metadata)
{
await FaceServiceHelper.DeleteFaceListAsync(item.FaceListId);
}
}
catch (Exception e)
{
ErrorTrackingHelper.TrackException(e, "Error resetting face lists");
}
}
private async Task <IList <IdentifyResult> > GetFaceIdentificationResultsOrDefaultAsync(string personGroupId, Guid[] detectedFaceIds)
{
try
{
return(await FaceServiceHelper.IdentifyAsync(personGroupId, detectedFaceIds));
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Face API IdentifyAsync error");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure identifying faces");
}
}
return(new List <IdentifyResult>());
}
public async Task DetectFacesAsync(bool detectFaceAttributes = false, bool detectFaceLandmarks = false)
{
try
{
if (this.ImageUrl != null)
{
this.DetectedFaces = await FaceServiceHelper.DetectAsync(
this.ImageUrl,
returnFaceId : true,
returnFaceLandmarks : detectFaceLandmarks,
returnFaceAttributes : detectFaceAttributes?DefaultFaceAttributeTypes : null);
}
else if (this.GetImageStreamCallback != null)
{
this.DetectedFaces = await FaceServiceHelper.DetectAsync(
this.GetImageStreamCallback,
returnFaceId : true,
returnFaceLandmarks : detectFaceLandmarks,
returnFaceAttributes : detectFaceAttributes?DefaultFaceAttributeTypes : null);
}
if (this.FilterOutSmallFaces)
{
this.DetectedFaces = this.DetectedFaces.Where(f => CoreUtil.IsFaceBigEnoughForDetection(f.FaceRectangle.Height, this.DecodedImageHeight));
}
}
catch (Exception e)
{
ErrorTrackingHelper.TrackException(e, "Face API DetectAsync error");
this.DetectedFaces = Enumerable.Empty <Face>();
if (this.ShowDialogOnFaceApiErrors)
{
//await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Face detection failed.");
ErrorMessage = e.Message + " - Face Detection Failed";
}
}
finally
{
this.OnFaceDetectionCompleted();
}
}
public static async Task Initialize()
{
faceLists = new Dictionary <string, FaceListInfo>();
try
{
IEnumerable <FaceList> metadata = await FaceServiceHelper.GetFaceListsAsync(FaceListsUserDataFilter);
foreach (var item in metadata)
{
faceLists.Add(item.FaceListId, new FaceListInfo {
FaceListId = item.FaceListId, LastMatchTimestamp = DateTime.Now
});
}
}
catch (Exception e)
{
ErrorTrackingHelper.TrackException(e, "Face API GetFaceListsAsync error");
}
}
private async Task <IEnumerable <PersonGroup> > GetPersonGroupsAsync()
{
IEnumerable <PersonGroup> personGroups = Enumerable.Empty <PersonGroup>();
try
{
personGroups = (await FaceServiceHelper.ListPersonGroupsAsync(PeopleGroupsUserDataFilter))
.Where(x => x.RecognitionModel.Equals(FaceServiceHelper.LatestRecognitionModelName, StringComparison.OrdinalIgnoreCase));
}
catch (Exception e)
{
ErrorTrackingHelper.TrackException(e, "Face API GetPersonGroupsAsync error");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure getting PersonGroups");
}
}
return(personGroups);
}
public async Task <IdentifyResult[]> IdentifyFacesAsync(string groupId)
{
this.IdentifiedPersons = Enumerable.Empty <IdentifiedPerson>();
IdentifyResult[] groupResults = null;
Guid[] detectedFaceIds = this.DetectedFaces?.Select(f => f.FaceId).ToArray();
if (detectedFaceIds != null && detectedFaceIds.Any())
{
List <IdentifiedPerson> result = new List <IdentifiedPerson>();
try {
groupResults = await FaceServiceHelper.IdentifyAsync(groupId, detectedFaceIds);
foreach (var match in groupResults)
{
if (!match.Candidates.Any())
{
continue;
}
Person person = await FaceServiceHelper.GetPersonAsync(groupId, match.Candidates[0].PersonId);
result.Add(new IdentifiedPerson {
Person = person, Confidence = match.Candidates[0].Confidence, FaceId = match.FaceId
});
}
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Face API IdentifyAsync error");
//if (this.ShowDialogOnFaceApiErrors)
//{
// await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure identifying faces");
//}
}
this.IdentifiedPersons = result;
}
return(groupResults);
}
private async Task AddFaceToPerson(Face f, Person p, Guid personId)
{
//Maximum faces that we are able to persist
if (p.PersistedFaceIds.Length == 248)
{
Guid persistedFaceId = p.PersistedFaceIds.OrderBy(x => Guid.NewGuid()).FirstOrDefault();
await FaceServiceHelper.DeletePersonFaceAsync(groupId, personId, persistedFaceId);
}
try
{
await FaceServiceHelper.AddPersonFaceAsync(groupId, personId, await this.GetImageStreamCallback(), "", f.FaceRectangle);
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Problem adding face to group");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem adding face to group");
}
}
}
public async Task IdentifyFacesAsync()
{
this.IdentifiedPersons = Enumerable.Empty <IdentifiedPerson>();
Guid[] detectedFaceIds = this.DetectedFaces?.Where(f => f.FaceId.HasValue).Select(f => f.FaceId.GetValueOrDefault()).ToArray();
if (detectedFaceIds != null && detectedFaceIds.Any())
{
List <IdentifiedPerson> result = new List <IdentifiedPerson>();
var personGroupResultTasks = new List <Tuple <string, Task <IList <IdentifyResult> > > >();
IEnumerable <PersonGroup> personGroups = await GetPersonGroupsAsync();
foreach (PersonGroup group in personGroups)
{
personGroupResultTasks.Add(new Tuple <string, Task <IList <IdentifyResult> > >(group.PersonGroupId, GetFaceIdentificationResultsOrDefaultAsync(group.PersonGroupId, detectedFaceIds)));
}
// identify detected faces using person groups
await Task.WhenAll(personGroupResultTasks.Select(x => x.Item2));
// check matches
foreach (var group in personGroupResultTasks)
{
foreach (IdentifyResult match in group.Item2.Result)
{
if (!match.Candidates.Any())
{
continue;
}
Person person = null;
try
{
person = await FaceServiceHelper.GetPersonAsync(group.Item1, match.Candidates[0].PersonId);
}
catch { /* Ignore errors */ }
if (person == null)
{
continue;
}
IdentifiedPerson alreadyIdentifiedPerson = result.FirstOrDefault(p => p.Person.PersonId == match.Candidates[0].PersonId);
if (alreadyIdentifiedPerson != null)
{
// We already tagged this person in another group. Replace the existing one if this new one if the confidence is higher.
if (alreadyIdentifiedPerson.Confidence < match.Candidates[0].Confidence)
{
alreadyIdentifiedPerson.Person = person;
alreadyIdentifiedPerson.Confidence = match.Candidates[0].Confidence;
alreadyIdentifiedPerson.FaceId = match.FaceId;
}
}
else
{
result.Add(new IdentifiedPerson {
Person = person, Confidence = match.Candidates[0].Confidence, FaceId = match.FaceId
});
}
}
}
this.IdentifiedPersons = result;
}
}
/// <summary>
///
/// </summary>
/// <param name="imageStream"></param>
/// <param name="faceId"></param>
/// <param name="faceRectangle"></param>
/// <param name="complexScenario"> - Whether we are identifying face hard way - means whether we need to use db or not</param>
/// <returns></returns>
public static async Task <Tuple <SimilarPersistedFace, string> > FindSimilarPersistedFaceAsync(Stream imageStream, Guid faceId, FaceRectangle faceRectangle)
{
if (faceLists == null)
{
await Initialize();
}
Tuple <SimilarPersistedFace, string> bestMatch = null;
var faceListId = faceLists.FirstOrDefault().Key;
//Delete for testing purposes
//await FaceServiceHelper.DeleteFaceListAsync(faceListId);
try
{
SimilarPersistedFace similarFace = null;
if (faceListId != null)
{
try
{
similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault();
}
catch (Exception e)
{
if ((e as FaceAPIException).ErrorCode == "FaceListNotReady")
{
// do nothing, list is empty but we continue
}
}
}
if (similarFace != null)
{
if (bestMatch != null)
{
// We already found a match for this face in another list. Replace the previous one if the new confidence is higher.
if (bestMatch.Item1.Confidence < similarFace.Confidence)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
else
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
else
{
// If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We
// might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we
// last matched anything on it) so that we can add the new one.
if (!faceLists.Any())
{
// We don't have any FaceLists yet. Create one
newFaceListId = Guid.NewGuid().ToString();
// await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
//We are not using filters
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList");
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
}
AddPersistedFaceResult addResult = null;
var faceList = faceLists.First();
if (faceList.Value.IsFull)
{
//It is here only for complex scenario where we use groups and lists mappings
try
{
DBSimilarFace faceToDelete = null;
using (var db = new KioskDBContext())
{
faceToDelete = db.SimilarFaces.OrderByDescending(sf => sf.PersonId).First();
db.SimilarFaces.Remove(faceToDelete);
await db.SaveChangesAsync();
}
await FaceServiceHelper.DeleteFaceFromFaceListAsync(faceList.Key, new Guid(faceToDelete.FaceId));
}
catch (Exception e)
{
Debug.WriteLine("No face to be deleted" + e.Message);
}
}
addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStream, faceRectangle);
if (addResult != null)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace {
Confidence = 1, PersistedFaceId = addResult.PersistedFaceId
}, null);
}
}
}
catch (Exception e)
{
// Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error");
}
return(bestMatch);
}
public async Task <List <FaceSendInfo> > IdentifyOrAddPersonWithEmotionsAsync(string groupName, ObservableCollection <IdentifiedFaces> identifiedPersonsIdCollection)
{
var facesInfo = new List <FaceSendInfo>();
//Loop thru all detected faces from previous steps and fill the facesInfo array
//For ease of processing in Azure Stream Analytics we create single level object
foreach (var f in this.DetectedFaces)
{
var fsi = new FaceSendInfo();
//Add emotions
var e = CoreUtil.FindEmotionForFace(f, this.DetectedEmotion);
FeedFaceInfo(f, fsi, e);
//We sen also info how many faces in total were recognized on the picture with current face
fsi.facesNo = this.DetectedFaces.Count();
facesInfo.Add(fsi);
}
//Now we proceed to face recognition/identification
//First we create group if it does not exist
try
{
var g = await FaceServiceHelper.CreatePersonGroupIfNoGroupExists(groupName);
groupId = g.PersonGroupId;
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Problem creating group");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem creating group");
}
}
//We need to find candidate for every face
try
{
IdentifyResult[] groupResults = await this.IdentifyFacesAsync(groupId);
//We loop thri all faces again in order to find candidate
foreach (var f in this.DetectedFaces)
{
bool needToRetrain = true;
var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault();
var newPersonID = Guid.NewGuid();
if (groupResults != null && groupResults.Where(gr => gr.FaceId == f.FaceId).Any() && groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.Any())
{
var candidates = groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.OrderByDescending(ca => ca.Confidence);
Person p = new Person();
var can = candidates.FirstOrDefault();
//If we have sufficient confidence, we add Face for person
if (can.Confidence >= SettingsHelper.Instance.Confidence)
{
fi.canid = can.PersonId.ToString();
fi.canconf = can.Confidence;
//In order to get also name we need to obtain Person
p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId);
fi.canname = p.Name;
var identifiedPersonFromList = identifiedPersonsIdCollection.Where(ip => ip.Id == can.PersonId.ToString()).FirstOrDefault();
//Check whether we did not added too much photos lately, it is not neccesary to add photo for face every time
if (identifiedPersonFromList == null)
{
await AddFaceToPerson(f, p, can.PersonId);
}
else if (identifiedPersonFromList.NumOfAddedPhotosInLastPeriod < SettingsHelper.Instance.NumberOfPhotoAddsInPeriod)
{
await AddFaceToPerson(f, p, can.PersonId);
identifiedPersonFromList.NumOfAddedPhotosInLastPeriod++;
}
else if ((DateTime.Now - identifiedPersonFromList.FirstPhotoAddedInLastPeriod).Hours > SettingsHelper.Instance.PhotoAddPeriodSize)
{
identifiedPersonFromList.NumOfAddedPhotosInLastPeriod = 1;
identifiedPersonFromList.FirstPhotoAddedInLastPeriod = DateTime.Now;
await AddFaceToPerson(f, p, can.PersonId);
}
else
{
needToRetrain = false;
}
}
else
{
//if not sufficient confidence we also need to check whether there is similar face/ if not create new person
await CreatePrsonIfNoSimilarFaceExistsAsync(facesInfo, newPersonID, f);
}
}
else
{
//if no candidate we also need to check whether there is similar fac,e if not create new person
await CreatePrsonIfNoSimilarFaceExistsAsync(facesInfo, newPersonID, f);
}
try
{
//We need to train after operation on top of group (addition of photo, person etc.)
if (needToRetrain)
{
await FaceServiceHelper.TrainPersonGroupAsync(groupId);
}
}
catch (Exception e)
{
// Catch error with training of group
ErrorTrackingHelper.TrackException(e, "Problem training group");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem training group");
}
}
//Handle the identified persons collection to which we locally save every identified person
if (!identifiedPersonsIdCollection.Where(ip => ip.Id == fi.canid).Any())
{
identifiedPersonsIdCollection.Add(new IdentifiedFaces()
{
Id = fi.canid
});
}
//Increase counter of identifications
else if (identifiedPersonsIdCollection.Where(ip => ip.Id == fi.canid).Any())
{
identifiedPersonsIdCollection.Where(ip => ip.Id == fi.canid).FirstOrDefault().NumberOfIdentifications++;
}
//Find faces which were wrongly learned (small number of identifications)
var tbd = new List <IdentifiedFaces>();
foreach (var ip in identifiedPersonsIdCollection)
{
if (ip.NumberOfIdentifications <= SettingsHelper.Instance.NeededFaceIdentNum && (ip.CreatedAt.AddSeconds(SettingsHelper.Instance.DeleteWindow) < DateTime.Now))
{
var g = (await FaceServiceHelper.GetPersonGroupsAsync()).Where(gr => gr.Name == groupName).FirstOrDefault();
Person pers = await FaceServiceHelper.GetPersonAsync(g.PersonGroupId, new Guid(ip.Id));
//if we saved insufficient number of faces than delete
if (pers.PersistedFaceIds.Length <= SettingsHelper.Instance.NeededFaceIdentNum)
{
await FaceServiceHelper.DeletePersonAsync(g.PersonGroupId, pers.PersonId);
string similarFaceId = "";
using (var db = new KioskDBContext())
{
var sfToDelete = db.SimilarFaces.Where(sf => sf.PersonId == pers.PersonId.ToString()).FirstOrDefault();
similarFaceId = sfToDelete.FaceId.ToString();
db.SimilarFaces.Remove(sfToDelete);
}
await FaceListManager.DeleteFaceFromFaceList(similarFaceId);
await FaceServiceHelper.TrainPersonGroupAsync(g.PersonGroupId);
tbd.Add(ip);
}
}
}
foreach (var iptodelete in tbd)
{
identifiedPersonsIdCollection.Remove(iptodelete);
}
}
}
catch (Exception e)
{
// Catch error with training of group
ErrorTrackingHelper.TrackException(e, "Problem with cognitive services");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem with cognitive services");
}
}
return(facesInfo);
}
private async Task CreatePrsonIfNoSimilarFaceExistsAsync(List <FaceSendInfo> facesInfo, Guid newPersonID, Face f)
{
//TODO return person result so we can change candidate if we are not going with the one selected
//We try to find person also thru similar face api
SimilarFaceMatch result = await GetSimilarFace(f);
using (var db = new KioskDBContext())
{
//If we create new similar face, we create also new person
if (result.isNew)
{
//We create db where we map SimilarPersonID to PersonID, because those are different in cognitive services
var perResult = await CreatePerson(facesInfo, newPersonID, f);
db.SimilarFaces.Add(new DBSimilarFace()
{
CreatedAt = DateTime.Now, FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString(), PersonId = perResult.ToString()
});
await db.SaveChangesAsync();
}
else
{
string personId = "";
try
{
personId = db.SimilarFaces.Where(sf => sf.FaceId == result.SimilarPersistedFace.PersistedFaceId.ToString()).FirstOrDefault().PersonId;
var person = await FaceServiceHelper.GetPersonAsync(groupId, new Guid(personId));
//Fill new info
var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault();
fi.canid = personId;
fi.canname = person.Name;
fi.canconf = result.SimilarPersistedFace.Confidence;
//We did not identified person thru person group and we needed to use silimar face api, so we add picture
await AddFaceToPerson(f, person, new Guid(personId));
}
catch (Exception)
{
//Person was not found due to old entry in local DB (Exception thrown by FaceApi GetPersonAsync) or face was created only in list when not using complex identification
//If person is only in list, we need to create it also in person list
if (personId == "")
{
var perResult2 = await CreatePerson(facesInfo, newPersonID, f);
db.SimilarFaces.Add(new DBSimilarFace()
{
CreatedAt = DateTime.Now, FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString(), PersonId = perResult2.ToString()
});
await db.SaveChangesAsync();
}
//We clean the old entry from DB and create new Person
var oldDbEntry = db.SimilarFaces.Where(sf => sf.FaceId == result.SimilarPersistedFace.PersistedFaceId.ToString()).FirstOrDefault();
var perResult = await CreatePerson(facesInfo, newPersonID, f);
if (oldDbEntry != null)
{
db.SimilarFaces.Update(oldDbEntry);
oldDbEntry.FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString();
oldDbEntry.PersonId = perResult.ToString();
}
else
{
db.SimilarFaces.Add(new DBSimilarFace()
{
CreatedAt = DateTime.Now, FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString(), PersonId = perResult.ToString()
});
}
await db.SaveChangesAsync();
}
}
}
}
private static async Task <SimilarFace> FindSimilarOrInsertAsync(string imageUrl, Func <Task <Stream> > imageStreamCallback, Guid faceId, DetectedFace face)
{
if (faceLists == null)
{
await Initialize();
}
Tuple <SimilarFace, string> bestMatch = null;
bool foundMatch = false;
foreach (var faceListId in faceLists.Keys)
{
try
{
SimilarFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault();
if (similarFace == null)
{
continue;
}
foundMatch = true;
if (bestMatch != null)
{
// We already found a match for this face in another list. Replace the previous one if the new confidence is higher.
if (bestMatch.Item1.Confidence < similarFace.Confidence)
{
bestMatch = new Tuple <SimilarFace, string>(similarFace, faceListId);
}
}
else
{
bestMatch = new Tuple <SimilarFace, string>(similarFace, faceListId);
}
}
catch (Exception e)
{
// Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error");
}
}
if (!foundMatch)
{
// If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We
// might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we
// last match anything on) so that we can add the new one.
double maxAngle = 30;
if (face.FaceAttributes.HeadPose != null &&
(Math.Abs(face.FaceAttributes.HeadPose.Yaw) > maxAngle ||
Math.Abs(face.FaceAttributes.HeadPose.Pitch) > maxAngle ||
Math.Abs(face.FaceAttributes.HeadPose.Roll) > maxAngle))
{
// This isn't a good frontal shot, so let's not use it as the primary example face for this person
return(null);
}
if (!faceLists.Any())
{
// We don't have any FaceLists yet. Create one
string newFaceListId = Guid.NewGuid().ToString();
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
}
PersistedFace addResult = null;
bool failedToAddToNonFullList = false;
foreach (var faceList in faceLists)
{
if (faceList.Value.IsFull)
{
continue;
}
try
{
if (imageUrl != null)
{
addResult = await FaceServiceHelper.AddFaceToFaceListFromUrlAsync(faceList.Key, imageUrl, face.FaceRectangle);
}
else
{
addResult = await FaceServiceHelper.AddFaceToFaceListFromStreamAsync(faceList.Key, imageStreamCallback, face.FaceRectangle);
}
break;
}
catch (Exception ex)
{
if (ex is APIErrorException && ((APIErrorException)ex).Response.StatusCode == (System.Net.HttpStatusCode) 403)
{
// FaceList is full. Continue so we can try again with the next FaceList
faceList.Value.IsFull = true;
continue;
}
else
{
failedToAddToNonFullList = true;
break;
}
}
}
if (addResult == null && !failedToAddToNonFullList)
{
// We were not able to add the face to an existing list because they were all full.
// If possible, let's create a new list now and add the new face to it. If we can't (e.g. we already maxed out on list count),
// let's delete an old list, create a new one and add the new face to it.
if (faceLists.Count == MaxFaceListCount)
{
// delete oldest face list
var oldestFaceList = faceLists.OrderBy(fl => fl.Value.LastMatchTimestamp).FirstOrDefault();
faceLists.Remove(oldestFaceList.Key);
await FaceServiceHelper.DeleteFaceListAsync(oldestFaceList.Key);
}
// create new list
string newFaceListId = Guid.NewGuid().ToString();
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
// Add face to new list
if (imageUrl != null)
{
addResult = await FaceServiceHelper.AddFaceToFaceListFromUrlAsync(newFaceListId, imageUrl, face.FaceRectangle);
}
else
{
addResult = await FaceServiceHelper.AddFaceToFaceListFromStreamAsync(newFaceListId, imageStreamCallback, face.FaceRectangle);
}
}
if (addResult != null)
{
bestMatch = new Tuple <SimilarFace, string>(new SimilarFace {
Confidence = 1, PersistedFaceId = addResult.PersistedFaceId
}, null);
}
}
return(bestMatch?.Item1);
}
public static async Task <SimilarPersistedFace> FindSimilarPersistedFaceAsync(Stream imageStream, Guid faceId, FaceRectangle faceRectangle)
{
if (faceLists == null)
{
await Initialize();
}
Tuple <SimilarPersistedFace, string> bestMatch = null;
bool foundMatch = false;
foreach (var faceListId in faceLists.Keys)
{
try
{
SimilarPersistedFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault();
if (similarFace == null)
{
continue;
}
foundMatch = true;
if (bestMatch != null)
{
// We already found a match for this face in another list. Replace the previous one if the new confidence is higher.
if (bestMatch.Item1.Confidence < similarFace.Confidence)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
else
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
catch (Exception e)
{
// Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error");
}
}
if (!foundMatch)
{
// If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We
// might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we
// last matched anything on it) so that we can add the new one.
if (!faceLists.Any())
{
// We don't have any FaceLists yet. Create one
string newFaceListId = Guid.NewGuid().ToString();
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
}
AddPersistedFaceResult addResult = null;
bool failedToAddToNonFullList = false;
foreach (var faceList in faceLists)
{
if (faceList.Value.IsFull)
{
continue;
}
try
{
addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStream, faceRectangle);
break;
}
catch (Exception ex)
{
if (ex is FaceAPIException && ((FaceAPIException)ex).ErrorCode == "403")
{
// FaceList is full. Continue so we can try again with the next FaceList
faceList.Value.IsFull = true;
continue;
}
else
{
failedToAddToNonFullList = true;
break;
}
}
}
if (addResult == null && !failedToAddToNonFullList)
{
// We were not able to add the face to an existing list because they were all full.
// If possible, let's create a new list now and add the new face to it. If we can't (e.g. we already maxed out on list count),
// let's delete an old list, create a new one and add the new face to it.
if (faceLists.Count == 64)
{
// delete oldest face list
var oldestFaceList = faceLists.OrderBy(fl => fl.Value.LastMatchTimestamp).FirstOrDefault();
faceLists.Remove(oldestFaceList.Key);
await FaceServiceHelper.DeleteFaceListAsync(oldestFaceList.Key);
}
// create new list
string newFaceListId = Guid.NewGuid().ToString();
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
// Add face to new list
addResult = await FaceServiceHelper.AddFaceToFaceListAsync(newFaceListId, imageStream, faceRectangle);
}
if (addResult != null)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace {
Confidence = 1, PersistedFaceId = addResult.PersistedFaceId
}, null);
}
}
return(bestMatch?.Item1);
}
public static async Task <SimilarPersistedFace> FindSimilarPersistedFaceAsync(Stream imageStream, Guid faceId, FaceRectangle faceRectangle, string faceIdToDelete = "")
{
if (faceLists == null)
{
await Initialize();
}
Tuple <SimilarPersistedFace, string> bestMatch = null;
var faceListId = faceLists.FirstOrDefault().Key;
try
{
SimilarPersistedFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault();
if (similarFace != null)
{
if (bestMatch != null)
{
// We already found a match for this face in another list. Replace the previous one if the new confidence is higher.
if (bestMatch.Item1.Confidence < similarFace.Confidence)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
else
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
else
{
// If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We
// might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we
// last matched anything on it) so that we can add the new one.
if (!faceLists.Any())
{
// We don't have any FaceLists yet. Create one
string newFaceListId = Guid.NewGuid().ToString();
// await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
//We are not using filters
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList");
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
}
AddPersistedFaceResult addResult = null;
var faceList = faceLists.First();
if (faceList.Value.IsFull)
{
await FaceServiceHelper.DeleteFaceFromFaceListAsync(faceList.Key, new Guid(faceIdToDelete));
addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStream, faceRectangle);
}
if (addResult != null)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace {
Confidence = 1, PersistedFaceId = addResult.PersistedFaceId
}, null);
}
}
}
catch (Exception e)
{
// Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error");
}
return(bestMatch?.Item1);
}
public static async Task <SimilarPersistedFace> FindSimilarPersistedFaceAsync(Func <Task <Stream> > imageStreamCallback, Guid faceId, Face face)
{
if (faceLists == null)
{
await Initialize();
}
Tuple <SimilarPersistedFace, string> bestMatch = null;
bool foundMatch = false;
foreach (var faceListId in faceLists.Keys)
{
try
{
SimilarPersistedFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault();
if (similarFace == null)
{
continue;
}
foundMatch = true;
if (bestMatch != null)
{
if (bestMatch.Item1.Confidence < similarFace.Confidence)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
else
{
bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId);
}
}
catch (Exception e)
{
ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error");
}
}
if (!foundMatch)
{
double maxAngle = 30;
if (face.FaceAttributes.HeadPose != null &&
(Math.Abs(face.FaceAttributes.HeadPose.Yaw) > maxAngle ||
Math.Abs(face.FaceAttributes.HeadPose.Pitch) > maxAngle ||
Math.Abs(face.FaceAttributes.HeadPose.Roll) > maxAngle))
{
return(null);
}
if (!faceLists.Any())
{
string newFaceListId = Guid.NewGuid().ToString();
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
}
AddPersistedFaceResult addResult = null;
bool failedToAddToNonFullList = false;
foreach (var faceList in faceLists)
{
if (faceList.Value.IsFull)
{
continue;
}
try
{
addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStreamCallback, face.FaceRectangle);
break;
}
catch (Exception ex)
{
if (ex is FaceAPIException && ((FaceAPIException)ex).ErrorCode == "403")
{
faceList.Value.IsFull = true;
continue;
}
else
{
failedToAddToNonFullList = true;
break;
}
}
}
if (addResult == null && !failedToAddToNonFullList)
{
if (faceLists.Count == 64)
{
var oldestFaceList = faceLists.OrderBy(fl => fl.Value.LastMatchTimestamp).FirstOrDefault();
faceLists.Remove(oldestFaceList.Key);
await FaceServiceHelper.DeleteFaceListAsync(oldestFaceList.Key);
}
string newFaceListId = Guid.NewGuid().ToString();
await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter);
faceLists.Add(newFaceListId, new FaceListInfo {
FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now
});
addResult = await FaceServiceHelper.AddFaceToFaceListAsync(newFaceListId, imageStreamCallback, face.FaceRectangle);
}
if (addResult != null)
{
bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace {
Confidence = 1, PersistedFaceId = addResult.PersistedFaceId
}, null);
}
}
return(bestMatch?.Item1);
}
public async Task IdentifyFacesAsync()
{
this.IdentifiedPersons = Enumerable.Empty <IdentifiedPerson>();
Guid[] detectedFaceIds = this.DetectedFaces?.Select(f => f.FaceId).ToArray();
if (detectedFaceIds != null && detectedFaceIds.Any())
{
List <IdentifiedPerson> result = new List <IdentifiedPerson>();
IEnumerable <PersonGroup> personGroups = Enumerable.Empty <PersonGroup>();
try
{
personGroups = await FaceServiceHelper.GetPersonGroupsAsync(PeopleGroupsUserDataFilter);
}
catch (Exception e)
{
ErrorTrackingHelper.TrackException(e, "Face API GetPersonGroupsAsync error");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure getting PersonGroups");
}
}
foreach (var group in personGroups)
{
try
{
IdentifyResult[] groupResults = await FaceServiceHelper.IdentifyAsync(group.PersonGroupId, detectedFaceIds);
foreach (var match in groupResults)
{
if (!match.Candidates.Any())
{
continue;
}
Person person = await FaceServiceHelper.GetPersonAsync(group.PersonGroupId, match.Candidates[0].PersonId);
IdentifiedPerson alreadyIdentifiedPerson = result.FirstOrDefault(p => p.Person.PersonId == match.Candidates[0].PersonId);
if (alreadyIdentifiedPerson != null)
{
// We already tagged this person in another group. Replace the existing one if this new one if the confidence is higher.
if (alreadyIdentifiedPerson.Confidence < match.Candidates[0].Confidence)
{
alreadyIdentifiedPerson.Person = person;
alreadyIdentifiedPerson.Confidence = match.Candidates[0].Confidence;
alreadyIdentifiedPerson.FaceId = match.FaceId;
}
}
else
{
result.Add(new IdentifiedPerson {
Person = person, Confidence = match.Candidates[0].Confidence, FaceId = match.FaceId
});
}
}
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Face API IdentifyAsync error");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure identifying faces");
}
}
}
this.IdentifiedPersons = result;
}
this.OnFaceRecognitionCompleted();
}
public async Task <List <FaceSendInfo> > IdentifyOrAddPersonWithEmotionsAsync(string groupName, double confidence)
{
var time = DateTime.Now;
//We also add emotions
var facesInfo = new List <FaceSendInfo>();
foreach (var f in this.DetectedFaces)
{
var fsi = new FaceSendInfo();
var e = CoreUtil.FindEmotionForFace(f, this.DetectedEmotion);
fsi.faceId = f.FaceId.ToString();
fsi.age = f.FaceAttributes.Age;
fsi.faceRecHeight = f.FaceRectangle.Height;
fsi.faceRecLeft = f.FaceRectangle.Left;
fsi.faceRecTop = f.FaceRectangle.Top;
fsi.faceRecWidth = f.FaceRectangle.Width;
fsi.gender = f.FaceAttributes.Gender;
fsi.smile = f.FaceAttributes.Smile;
fsi.beard = f.FaceAttributes.FacialHair.Beard;
fsi.moustache = f.FaceAttributes.FacialHair.Moustache;
fsi.sideburns = f.FaceAttributes.FacialHair.Sideburns;
fsi.glasses = f.FaceAttributes.Glasses.ToString();
fsi.headYaw = f.FaceAttributes.HeadPose.Yaw;
fsi.headRoll = f.FaceAttributes.HeadPose.Roll;
fsi.headPitch = f.FaceAttributes.HeadPose.Pitch;
fsi.anger = e.Scores.Anger;
fsi.contempt = e.Scores.Contempt;
fsi.disgust = e.Scores.Disgust;
fsi.fear = e.Scores.Fear;
fsi.happiness = e.Scores.Happiness;
fsi.neutral = e.Scores.Neutral;
fsi.sadness = e.Scores.Sadness;
fsi.surprise = e.Scores.Surprise;
fsi.timeStamp = time;
fsi.facesNo = this.DetectedFaces.Count();
facesInfo.Add(fsi);
}
var newPersonID = Guid.NewGuid();
//Move to initialization
try
{
var g = await FaceServiceHelper.CreatePersonGroupIfNoGroupExists(groupName);
groupId = g.PersonGroupId;
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Problem creating group");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem creating group");
}
}
List <IdentifiedPerson> ipresult = new List <IdentifiedPerson>();
// Compute Face Identification and Unique Face Ids
//We need to map detected faceID with actual personID (Identified face)
try
{
IdentifyResult[] groupResults = await this.IdentifyFacesAsync(groupId);
foreach (var f in this.DetectedFaces)
{
if (groupResults != null && groupResults.Where(gr => gr.FaceId == f.FaceId).Any() && groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.Any())
{
var candidates = groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.OrderByDescending(ca => ca.Confidence);
var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault();
int i = 0;
Person p = new Person();
foreach (var can in candidates)
{
switch (i)
{
case 0:
fi.can1id = can.PersonId.ToString();
fi.can1conf = can.Confidence;
p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId);
fi.can1name = p.Name;
if (can.Confidence >= confidence)
{
ipresult.Add(new IdentifiedPerson()
{
Person = p, Confidence = can.Confidence, FaceId = f.FaceId
});
if (p.PersistedFaceIds.Length == 248)
{
Guid persistedFaceId = p.PersistedFaceIds.OrderBy(x => Guid.NewGuid()).FirstOrDefault();
await FaceServiceHelper.DeletePersonFaceAsync(groupId, can.PersonId, persistedFaceId);
}
try
{
await FaceServiceHelper.AddPersonFaceAsync(groupId, can.PersonId, await this.GetImageStreamCallback(), "", f.FaceRectangle);
}
catch (Exception e)
{
// Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from
// another one.
ErrorTrackingHelper.TrackException(e, "Problem adding face to group");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem adding face to group");
}
}
}
else
{
//create new Guy if confidence not sufficient
SimilarFaceMatch result = await GetSimilarFace(f);
//using (var db = new KioskDbContext())
//{
// Blogs.ItemsSource = db.Blogs.ToList();
//}
await CreatePerson(facesInfo, newPersonID, f);
}
break;
case 1:
fi.can2id = can.PersonId.ToString();
fi.can2conf = can.Confidence;
p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId);
fi.can2name = p.Name;
break;
case 2:
fi.can3id = can.PersonId.ToString();
fi.can3conf = can.Confidence;
p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId);
fi.can3name = p.Name;
break;
case 3:
fi.can4id = can.PersonId.ToString();
fi.can4conf = can.Confidence;
p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId);
fi.can4name = p.Name;
break;
}
i++;
}
}
else
{
//if no candidate we also need to create new person
await CreatePerson(facesInfo, newPersonID, f);
}
try
{
await FaceServiceHelper.TrainPersonGroupAsync(groupId);
}
catch (Exception e)
{
// Catch error with training of group
ErrorTrackingHelper.TrackException(e, "Problem training group");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem training group");
}
}
}
}
catch (Exception e)
{
// Catch error with training of group
ErrorTrackingHelper.TrackException(e, "Problem with cognitive services");
if (this.ShowDialogOnFaceApiErrors)
{
await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem with cognitive services");
}
}
return(facesInfo);
}
