private async Task <Guid> CreatePerson(List <FaceSendInfo> facesInfo, Guid newPersonID, Face f) { AddPersistedFaceResult result = null; try { //No candidate we are going to create new person and set candidate to be same as newly created person with confidence of 100% var name = f.FaceAttributes.Gender + "-" + f.FaceAttributes.Age + "-" + newPersonID.ToString(); newPersonID = (await FaceServiceHelper.CreatePersonWithResultAsync(groupId, name)).PersonId; var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault(); fi.canid = newPersonID.ToString(); fi.canname = name; fi.canconf = 1; await FaceServiceHelper.AddPersonFaceAsync(groupId, newPersonID, await this.GetImageStreamCallback(), "", f.FaceRectangle); } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Problem adding face to group"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem adding face to group"); } } return(newPersonID); }
public static async Task ResetFaceLists() { faceLists = new Dictionary <string, FaceListInfo>(); try { //IEnumerable<FaceListMetadata> metadata = await FaceServiceHelper.GetFaceListsAsync(FaceListsUserDataFilter); IEnumerable <FaceListMetadata> metadata = await FaceServiceHelper.GetFaceListsAsync(); foreach (var item in metadata) { await FaceServiceHelper.DeleteFaceListAsync(item.FaceListId); } //We want to delete also local persisted DB using (var db = new KioskDBContext()) { db.SimilarFaces.RemoveRange(db.SimilarFaces); db.SaveChanges(); } } catch (Exception e) { ErrorTrackingHelper.TrackException(e, "Error resetting face lists"); } }
public static async Task DeleteFaceFromFaceList(string faceIdToDelete) { if (faceLists == null) { await Initialize(); } var faceListId = faceLists.FirstOrDefault().Key; await FaceServiceHelper.DeleteFaceFromFaceListAsync(faceListId, new Guid(faceIdToDelete)); }
public static async Task ResetFaceLists() { faceLists = new Dictionary <string, FaceListInfo>(); try { IEnumerable <FaceList> metadata = await FaceServiceHelper.GetFaceListsAsync(FaceListsUserDataFilter); foreach (var item in metadata) { await FaceServiceHelper.DeleteFaceListAsync(item.FaceListId); } } catch (Exception e) { ErrorTrackingHelper.TrackException(e, "Error resetting face lists"); } }
private async Task <IList <IdentifyResult> > GetFaceIdentificationResultsOrDefaultAsync(string personGroupId, Guid[] detectedFaceIds) { try { return(await FaceServiceHelper.IdentifyAsync(personGroupId, detectedFaceIds)); } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Face API IdentifyAsync error"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure identifying faces"); } } return(new List <IdentifyResult>()); }
public async Task DetectFacesAsync(bool detectFaceAttributes = false, bool detectFaceLandmarks = false) { try { if (this.ImageUrl != null) { this.DetectedFaces = await FaceServiceHelper.DetectAsync( this.ImageUrl, returnFaceId : true, returnFaceLandmarks : detectFaceLandmarks, returnFaceAttributes : detectFaceAttributes?DefaultFaceAttributeTypes : null); } else if (this.GetImageStreamCallback != null) { this.DetectedFaces = await FaceServiceHelper.DetectAsync( this.GetImageStreamCallback, returnFaceId : true, returnFaceLandmarks : detectFaceLandmarks, returnFaceAttributes : detectFaceAttributes?DefaultFaceAttributeTypes : null); } if (this.FilterOutSmallFaces) { this.DetectedFaces = this.DetectedFaces.Where(f => CoreUtil.IsFaceBigEnoughForDetection(f.FaceRectangle.Height, this.DecodedImageHeight)); } } catch (Exception e) { ErrorTrackingHelper.TrackException(e, "Face API DetectAsync error"); this.DetectedFaces = Enumerable.Empty <Face>(); if (this.ShowDialogOnFaceApiErrors) { //await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Face detection failed."); ErrorMessage = e.Message + " - Face Detection Failed"; } } finally { this.OnFaceDetectionCompleted(); } }
public static async Task Initialize() { faceLists = new Dictionary <string, FaceListInfo>(); try { IEnumerable <FaceList> metadata = await FaceServiceHelper.GetFaceListsAsync(FaceListsUserDataFilter); foreach (var item in metadata) { faceLists.Add(item.FaceListId, new FaceListInfo { FaceListId = item.FaceListId, LastMatchTimestamp = DateTime.Now }); } } catch (Exception e) { ErrorTrackingHelper.TrackException(e, "Face API GetFaceListsAsync error"); } }
private async Task <IEnumerable <PersonGroup> > GetPersonGroupsAsync() { IEnumerable <PersonGroup> personGroups = Enumerable.Empty <PersonGroup>(); try { personGroups = (await FaceServiceHelper.ListPersonGroupsAsync(PeopleGroupsUserDataFilter)) .Where(x => x.RecognitionModel.Equals(FaceServiceHelper.LatestRecognitionModelName, StringComparison.OrdinalIgnoreCase)); } catch (Exception e) { ErrorTrackingHelper.TrackException(e, "Face API GetPersonGroupsAsync error"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure getting PersonGroups"); } } return(personGroups); }
public async Task <IdentifyResult[]> IdentifyFacesAsync(string groupId) { this.IdentifiedPersons = Enumerable.Empty <IdentifiedPerson>(); IdentifyResult[] groupResults = null; Guid[] detectedFaceIds = this.DetectedFaces?.Select(f => f.FaceId).ToArray(); if (detectedFaceIds != null && detectedFaceIds.Any()) { List <IdentifiedPerson> result = new List <IdentifiedPerson>(); try { groupResults = await FaceServiceHelper.IdentifyAsync(groupId, detectedFaceIds); foreach (var match in groupResults) { if (!match.Candidates.Any()) { continue; } Person person = await FaceServiceHelper.GetPersonAsync(groupId, match.Candidates[0].PersonId); result.Add(new IdentifiedPerson { Person = person, Confidence = match.Candidates[0].Confidence, FaceId = match.FaceId }); } } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Face API IdentifyAsync error"); //if (this.ShowDialogOnFaceApiErrors) //{ // await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure identifying faces"); //} } this.IdentifiedPersons = result; } return(groupResults); }
private async Task AddFaceToPerson(Face f, Person p, Guid personId) { //Maximum faces that we are able to persist if (p.PersistedFaceIds.Length == 248) { Guid persistedFaceId = p.PersistedFaceIds.OrderBy(x => Guid.NewGuid()).FirstOrDefault(); await FaceServiceHelper.DeletePersonFaceAsync(groupId, personId, persistedFaceId); } try { await FaceServiceHelper.AddPersonFaceAsync(groupId, personId, await this.GetImageStreamCallback(), "", f.FaceRectangle); } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Problem adding face to group"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem adding face to group"); } } }
public async Task IdentifyFacesAsync() { this.IdentifiedPersons = Enumerable.Empty <IdentifiedPerson>(); Guid[] detectedFaceIds = this.DetectedFaces?.Where(f => f.FaceId.HasValue).Select(f => f.FaceId.GetValueOrDefault()).ToArray(); if (detectedFaceIds != null && detectedFaceIds.Any()) { List <IdentifiedPerson> result = new List <IdentifiedPerson>(); var personGroupResultTasks = new List <Tuple <string, Task <IList <IdentifyResult> > > >(); IEnumerable <PersonGroup> personGroups = await GetPersonGroupsAsync(); foreach (PersonGroup group in personGroups) { personGroupResultTasks.Add(new Tuple <string, Task <IList <IdentifyResult> > >(group.PersonGroupId, GetFaceIdentificationResultsOrDefaultAsync(group.PersonGroupId, detectedFaceIds))); } // identify detected faces using person groups await Task.WhenAll(personGroupResultTasks.Select(x => x.Item2)); // check matches foreach (var group in personGroupResultTasks) { foreach (IdentifyResult match in group.Item2.Result) { if (!match.Candidates.Any()) { continue; } Person person = null; try { person = await FaceServiceHelper.GetPersonAsync(group.Item1, match.Candidates[0].PersonId); } catch { /* Ignore errors */ } if (person == null) { continue; } IdentifiedPerson alreadyIdentifiedPerson = result.FirstOrDefault(p => p.Person.PersonId == match.Candidates[0].PersonId); if (alreadyIdentifiedPerson != null) { // We already tagged this person in another group. Replace the existing one if this new one if the confidence is higher. if (alreadyIdentifiedPerson.Confidence < match.Candidates[0].Confidence) { alreadyIdentifiedPerson.Person = person; alreadyIdentifiedPerson.Confidence = match.Candidates[0].Confidence; alreadyIdentifiedPerson.FaceId = match.FaceId; } } else { result.Add(new IdentifiedPerson { Person = person, Confidence = match.Candidates[0].Confidence, FaceId = match.FaceId }); } } } this.IdentifiedPersons = result; } }
/// <summary> /// /// </summary> /// <param name="imageStream"></param> /// <param name="faceId"></param> /// <param name="faceRectangle"></param> /// <param name="complexScenario"> - Whether we are identifying face hard way - means whether we need to use db or not</param> /// <returns></returns> public static async Task <Tuple <SimilarPersistedFace, string> > FindSimilarPersistedFaceAsync(Stream imageStream, Guid faceId, FaceRectangle faceRectangle) { if (faceLists == null) { await Initialize(); } Tuple <SimilarPersistedFace, string> bestMatch = null; var faceListId = faceLists.FirstOrDefault().Key; //Delete for testing purposes //await FaceServiceHelper.DeleteFaceListAsync(faceListId); try { SimilarPersistedFace similarFace = null; if (faceListId != null) { try { similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault(); } catch (Exception e) { if ((e as FaceAPIException).ErrorCode == "FaceListNotReady") { // do nothing, list is empty but we continue } } } if (similarFace != null) { if (bestMatch != null) { // We already found a match for this face in another list. Replace the previous one if the new confidence is higher. if (bestMatch.Item1.Confidence < similarFace.Confidence) { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } else { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } else { // If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We // might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we // last matched anything on it) so that we can add the new one. if (!faceLists.Any()) { // We don't have any FaceLists yet. Create one newFaceListId = Guid.NewGuid().ToString(); // await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); //We are not using filters await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList"); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); } AddPersistedFaceResult addResult = null; var faceList = faceLists.First(); if (faceList.Value.IsFull) { //It is here only for complex scenario where we use groups and lists mappings try { DBSimilarFace faceToDelete = null; using (var db = new KioskDBContext()) { faceToDelete = db.SimilarFaces.OrderByDescending(sf => sf.PersonId).First(); db.SimilarFaces.Remove(faceToDelete); await db.SaveChangesAsync(); } await FaceServiceHelper.DeleteFaceFromFaceListAsync(faceList.Key, new Guid(faceToDelete.FaceId)); } catch (Exception e) { Debug.WriteLine("No face to be deleted" + e.Message); } } addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStream, faceRectangle); if (addResult != null) { bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace { Confidence = 1, PersistedFaceId = addResult.PersistedFaceId }, null); } } } catch (Exception e) { // Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error"); } return(bestMatch); }
public async Task <List <FaceSendInfo> > IdentifyOrAddPersonWithEmotionsAsync(string groupName, ObservableCollection <IdentifiedFaces> identifiedPersonsIdCollection) { var facesInfo = new List <FaceSendInfo>(); //Loop thru all detected faces from previous steps and fill the facesInfo array //For ease of processing in Azure Stream Analytics we create single level object foreach (var f in this.DetectedFaces) { var fsi = new FaceSendInfo(); //Add emotions var e = CoreUtil.FindEmotionForFace(f, this.DetectedEmotion); FeedFaceInfo(f, fsi, e); //We sen also info how many faces in total were recognized on the picture with current face fsi.facesNo = this.DetectedFaces.Count(); facesInfo.Add(fsi); } //Now we proceed to face recognition/identification //First we create group if it does not exist try { var g = await FaceServiceHelper.CreatePersonGroupIfNoGroupExists(groupName); groupId = g.PersonGroupId; } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Problem creating group"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem creating group"); } } //We need to find candidate for every face try { IdentifyResult[] groupResults = await this.IdentifyFacesAsync(groupId); //We loop thri all faces again in order to find candidate foreach (var f in this.DetectedFaces) { bool needToRetrain = true; var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault(); var newPersonID = Guid.NewGuid(); if (groupResults != null && groupResults.Where(gr => gr.FaceId == f.FaceId).Any() && groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.Any()) { var candidates = groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.OrderByDescending(ca => ca.Confidence); Person p = new Person(); var can = candidates.FirstOrDefault(); //If we have sufficient confidence, we add Face for person if (can.Confidence >= SettingsHelper.Instance.Confidence) { fi.canid = can.PersonId.ToString(); fi.canconf = can.Confidence; //In order to get also name we need to obtain Person p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId); fi.canname = p.Name; var identifiedPersonFromList = identifiedPersonsIdCollection.Where(ip => ip.Id == can.PersonId.ToString()).FirstOrDefault(); //Check whether we did not added too much photos lately, it is not neccesary to add photo for face every time if (identifiedPersonFromList == null) { await AddFaceToPerson(f, p, can.PersonId); } else if (identifiedPersonFromList.NumOfAddedPhotosInLastPeriod < SettingsHelper.Instance.NumberOfPhotoAddsInPeriod) { await AddFaceToPerson(f, p, can.PersonId); identifiedPersonFromList.NumOfAddedPhotosInLastPeriod++; } else if ((DateTime.Now - identifiedPersonFromList.FirstPhotoAddedInLastPeriod).Hours > SettingsHelper.Instance.PhotoAddPeriodSize) { identifiedPersonFromList.NumOfAddedPhotosInLastPeriod = 1; identifiedPersonFromList.FirstPhotoAddedInLastPeriod = DateTime.Now; await AddFaceToPerson(f, p, can.PersonId); } else { needToRetrain = false; } } else { //if not sufficient confidence we also need to check whether there is similar face/ if not create new person await CreatePrsonIfNoSimilarFaceExistsAsync(facesInfo, newPersonID, f); } } else { //if no candidate we also need to check whether there is similar fac,e if not create new person await CreatePrsonIfNoSimilarFaceExistsAsync(facesInfo, newPersonID, f); } try { //We need to train after operation on top of group (addition of photo, person etc.) if (needToRetrain) { await FaceServiceHelper.TrainPersonGroupAsync(groupId); } } catch (Exception e) { // Catch error with training of group ErrorTrackingHelper.TrackException(e, "Problem training group"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem training group"); } } //Handle the identified persons collection to which we locally save every identified person if (!identifiedPersonsIdCollection.Where(ip => ip.Id == fi.canid).Any()) { identifiedPersonsIdCollection.Add(new IdentifiedFaces() { Id = fi.canid }); } //Increase counter of identifications else if (identifiedPersonsIdCollection.Where(ip => ip.Id == fi.canid).Any()) { identifiedPersonsIdCollection.Where(ip => ip.Id == fi.canid).FirstOrDefault().NumberOfIdentifications++; } //Find faces which were wrongly learned (small number of identifications) var tbd = new List <IdentifiedFaces>(); foreach (var ip in identifiedPersonsIdCollection) { if (ip.NumberOfIdentifications <= SettingsHelper.Instance.NeededFaceIdentNum && (ip.CreatedAt.AddSeconds(SettingsHelper.Instance.DeleteWindow) < DateTime.Now)) { var g = (await FaceServiceHelper.GetPersonGroupsAsync()).Where(gr => gr.Name == groupName).FirstOrDefault(); Person pers = await FaceServiceHelper.GetPersonAsync(g.PersonGroupId, new Guid(ip.Id)); //if we saved insufficient number of faces than delete if (pers.PersistedFaceIds.Length <= SettingsHelper.Instance.NeededFaceIdentNum) { await FaceServiceHelper.DeletePersonAsync(g.PersonGroupId, pers.PersonId); string similarFaceId = ""; using (var db = new KioskDBContext()) { var sfToDelete = db.SimilarFaces.Where(sf => sf.PersonId == pers.PersonId.ToString()).FirstOrDefault(); similarFaceId = sfToDelete.FaceId.ToString(); db.SimilarFaces.Remove(sfToDelete); } await FaceListManager.DeleteFaceFromFaceList(similarFaceId); await FaceServiceHelper.TrainPersonGroupAsync(g.PersonGroupId); tbd.Add(ip); } } } foreach (var iptodelete in tbd) { identifiedPersonsIdCollection.Remove(iptodelete); } } } catch (Exception e) { // Catch error with training of group ErrorTrackingHelper.TrackException(e, "Problem with cognitive services"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem with cognitive services"); } } return(facesInfo); }
private async Task CreatePrsonIfNoSimilarFaceExistsAsync(List <FaceSendInfo> facesInfo, Guid newPersonID, Face f) { //TODO return person result so we can change candidate if we are not going with the one selected //We try to find person also thru similar face api SimilarFaceMatch result = await GetSimilarFace(f); using (var db = new KioskDBContext()) { //If we create new similar face, we create also new person if (result.isNew) { //We create db where we map SimilarPersonID to PersonID, because those are different in cognitive services var perResult = await CreatePerson(facesInfo, newPersonID, f); db.SimilarFaces.Add(new DBSimilarFace() { CreatedAt = DateTime.Now, FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString(), PersonId = perResult.ToString() }); await db.SaveChangesAsync(); } else { string personId = ""; try { personId = db.SimilarFaces.Where(sf => sf.FaceId == result.SimilarPersistedFace.PersistedFaceId.ToString()).FirstOrDefault().PersonId; var person = await FaceServiceHelper.GetPersonAsync(groupId, new Guid(personId)); //Fill new info var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault(); fi.canid = personId; fi.canname = person.Name; fi.canconf = result.SimilarPersistedFace.Confidence; //We did not identified person thru person group and we needed to use silimar face api, so we add picture await AddFaceToPerson(f, person, new Guid(personId)); } catch (Exception) { //Person was not found due to old entry in local DB (Exception thrown by FaceApi GetPersonAsync) or face was created only in list when not using complex identification //If person is only in list, we need to create it also in person list if (personId == "") { var perResult2 = await CreatePerson(facesInfo, newPersonID, f); db.SimilarFaces.Add(new DBSimilarFace() { CreatedAt = DateTime.Now, FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString(), PersonId = perResult2.ToString() }); await db.SaveChangesAsync(); } //We clean the old entry from DB and create new Person var oldDbEntry = db.SimilarFaces.Where(sf => sf.FaceId == result.SimilarPersistedFace.PersistedFaceId.ToString()).FirstOrDefault(); var perResult = await CreatePerson(facesInfo, newPersonID, f); if (oldDbEntry != null) { db.SimilarFaces.Update(oldDbEntry); oldDbEntry.FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString(); oldDbEntry.PersonId = perResult.ToString(); } else { db.SimilarFaces.Add(new DBSimilarFace() { CreatedAt = DateTime.Now, FaceId = result.SimilarPersistedFace.PersistedFaceId.ToString(), PersonId = perResult.ToString() }); } await db.SaveChangesAsync(); } } } }
private static async Task <SimilarFace> FindSimilarOrInsertAsync(string imageUrl, Func <Task <Stream> > imageStreamCallback, Guid faceId, DetectedFace face) { if (faceLists == null) { await Initialize(); } Tuple <SimilarFace, string> bestMatch = null; bool foundMatch = false; foreach (var faceListId in faceLists.Keys) { try { SimilarFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault(); if (similarFace == null) { continue; } foundMatch = true; if (bestMatch != null) { // We already found a match for this face in another list. Replace the previous one if the new confidence is higher. if (bestMatch.Item1.Confidence < similarFace.Confidence) { bestMatch = new Tuple <SimilarFace, string>(similarFace, faceListId); } } else { bestMatch = new Tuple <SimilarFace, string>(similarFace, faceListId); } } catch (Exception e) { // Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error"); } } if (!foundMatch) { // If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We // might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we // last match anything on) so that we can add the new one. double maxAngle = 30; if (face.FaceAttributes.HeadPose != null && (Math.Abs(face.FaceAttributes.HeadPose.Yaw) > maxAngle || Math.Abs(face.FaceAttributes.HeadPose.Pitch) > maxAngle || Math.Abs(face.FaceAttributes.HeadPose.Roll) > maxAngle)) { // This isn't a good frontal shot, so let's not use it as the primary example face for this person return(null); } if (!faceLists.Any()) { // We don't have any FaceLists yet. Create one string newFaceListId = Guid.NewGuid().ToString(); await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); } PersistedFace addResult = null; bool failedToAddToNonFullList = false; foreach (var faceList in faceLists) { if (faceList.Value.IsFull) { continue; } try { if (imageUrl != null) { addResult = await FaceServiceHelper.AddFaceToFaceListFromUrlAsync(faceList.Key, imageUrl, face.FaceRectangle); } else { addResult = await FaceServiceHelper.AddFaceToFaceListFromStreamAsync(faceList.Key, imageStreamCallback, face.FaceRectangle); } break; } catch (Exception ex) { if (ex is APIErrorException && ((APIErrorException)ex).Response.StatusCode == (System.Net.HttpStatusCode) 403) { // FaceList is full. Continue so we can try again with the next FaceList faceList.Value.IsFull = true; continue; } else { failedToAddToNonFullList = true; break; } } } if (addResult == null && !failedToAddToNonFullList) { // We were not able to add the face to an existing list because they were all full. // If possible, let's create a new list now and add the new face to it. If we can't (e.g. we already maxed out on list count), // let's delete an old list, create a new one and add the new face to it. if (faceLists.Count == MaxFaceListCount) { // delete oldest face list var oldestFaceList = faceLists.OrderBy(fl => fl.Value.LastMatchTimestamp).FirstOrDefault(); faceLists.Remove(oldestFaceList.Key); await FaceServiceHelper.DeleteFaceListAsync(oldestFaceList.Key); } // create new list string newFaceListId = Guid.NewGuid().ToString(); await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); // Add face to new list if (imageUrl != null) { addResult = await FaceServiceHelper.AddFaceToFaceListFromUrlAsync(newFaceListId, imageUrl, face.FaceRectangle); } else { addResult = await FaceServiceHelper.AddFaceToFaceListFromStreamAsync(newFaceListId, imageStreamCallback, face.FaceRectangle); } } if (addResult != null) { bestMatch = new Tuple <SimilarFace, string>(new SimilarFace { Confidence = 1, PersistedFaceId = addResult.PersistedFaceId }, null); } } return(bestMatch?.Item1); }
public static async Task <SimilarPersistedFace> FindSimilarPersistedFaceAsync(Stream imageStream, Guid faceId, FaceRectangle faceRectangle) { if (faceLists == null) { await Initialize(); } Tuple <SimilarPersistedFace, string> bestMatch = null; bool foundMatch = false; foreach (var faceListId in faceLists.Keys) { try { SimilarPersistedFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault(); if (similarFace == null) { continue; } foundMatch = true; if (bestMatch != null) { // We already found a match for this face in another list. Replace the previous one if the new confidence is higher. if (bestMatch.Item1.Confidence < similarFace.Confidence) { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } else { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } catch (Exception e) { // Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error"); } } if (!foundMatch) { // If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We // might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we // last matched anything on it) so that we can add the new one. if (!faceLists.Any()) { // We don't have any FaceLists yet. Create one string newFaceListId = Guid.NewGuid().ToString(); await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); } AddPersistedFaceResult addResult = null; bool failedToAddToNonFullList = false; foreach (var faceList in faceLists) { if (faceList.Value.IsFull) { continue; } try { addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStream, faceRectangle); break; } catch (Exception ex) { if (ex is FaceAPIException && ((FaceAPIException)ex).ErrorCode == "403") { // FaceList is full. Continue so we can try again with the next FaceList faceList.Value.IsFull = true; continue; } else { failedToAddToNonFullList = true; break; } } } if (addResult == null && !failedToAddToNonFullList) { // We were not able to add the face to an existing list because they were all full. // If possible, let's create a new list now and add the new face to it. If we can't (e.g. we already maxed out on list count), // let's delete an old list, create a new one and add the new face to it. if (faceLists.Count == 64) { // delete oldest face list var oldestFaceList = faceLists.OrderBy(fl => fl.Value.LastMatchTimestamp).FirstOrDefault(); faceLists.Remove(oldestFaceList.Key); await FaceServiceHelper.DeleteFaceListAsync(oldestFaceList.Key); } // create new list string newFaceListId = Guid.NewGuid().ToString(); await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); // Add face to new list addResult = await FaceServiceHelper.AddFaceToFaceListAsync(newFaceListId, imageStream, faceRectangle); } if (addResult != null) { bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace { Confidence = 1, PersistedFaceId = addResult.PersistedFaceId }, null); } } return(bestMatch?.Item1); }
public static async Task <SimilarPersistedFace> FindSimilarPersistedFaceAsync(Stream imageStream, Guid faceId, FaceRectangle faceRectangle, string faceIdToDelete = "") { if (faceLists == null) { await Initialize(); } Tuple <SimilarPersistedFace, string> bestMatch = null; var faceListId = faceLists.FirstOrDefault().Key; try { SimilarPersistedFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault(); if (similarFace != null) { if (bestMatch != null) { // We already found a match for this face in another list. Replace the previous one if the new confidence is higher. if (bestMatch.Item1.Confidence < similarFace.Confidence) { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } else { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } else { // If we are here we didnt' find a match, so let's add the face to the first FaceList that we can add it to. We // might create a new list if none exist, and if all lists are full we will delete the oldest face list (based on when we // last matched anything on it) so that we can add the new one. if (!faceLists.Any()) { // We don't have any FaceLists yet. Create one string newFaceListId = Guid.NewGuid().ToString(); // await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); //We are not using filters await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList"); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); } AddPersistedFaceResult addResult = null; var faceList = faceLists.First(); if (faceList.Value.IsFull) { await FaceServiceHelper.DeleteFaceFromFaceListAsync(faceList.Key, new Guid(faceIdToDelete)); addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStream, faceRectangle); } if (addResult != null) { bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace { Confidence = 1, PersistedFaceId = addResult.PersistedFaceId }, null); } } } catch (Exception e) { // Catch errors with individual face lists so we can continue looping through all lists. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error"); } return(bestMatch?.Item1); }
public static async Task <SimilarPersistedFace> FindSimilarPersistedFaceAsync(Func <Task <Stream> > imageStreamCallback, Guid faceId, Face face) { if (faceLists == null) { await Initialize(); } Tuple <SimilarPersistedFace, string> bestMatch = null; bool foundMatch = false; foreach (var faceListId in faceLists.Keys) { try { SimilarPersistedFace similarFace = (await FaceServiceHelper.FindSimilarAsync(faceId, faceListId))?.FirstOrDefault(); if (similarFace == null) { continue; } foundMatch = true; if (bestMatch != null) { if (bestMatch.Item1.Confidence < similarFace.Confidence) { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } else { bestMatch = new Tuple <SimilarPersistedFace, string>(similarFace, faceListId); } } catch (Exception e) { ErrorTrackingHelper.TrackException(e, "Face API FindSimilarAsync error"); } } if (!foundMatch) { double maxAngle = 30; if (face.FaceAttributes.HeadPose != null && (Math.Abs(face.FaceAttributes.HeadPose.Yaw) > maxAngle || Math.Abs(face.FaceAttributes.HeadPose.Pitch) > maxAngle || Math.Abs(face.FaceAttributes.HeadPose.Roll) > maxAngle)) { return(null); } if (!faceLists.Any()) { string newFaceListId = Guid.NewGuid().ToString(); await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); } AddPersistedFaceResult addResult = null; bool failedToAddToNonFullList = false; foreach (var faceList in faceLists) { if (faceList.Value.IsFull) { continue; } try { addResult = await FaceServiceHelper.AddFaceToFaceListAsync(faceList.Key, imageStreamCallback, face.FaceRectangle); break; } catch (Exception ex) { if (ex is FaceAPIException && ((FaceAPIException)ex).ErrorCode == "403") { faceList.Value.IsFull = true; continue; } else { failedToAddToNonFullList = true; break; } } } if (addResult == null && !failedToAddToNonFullList) { if (faceLists.Count == 64) { var oldestFaceList = faceLists.OrderBy(fl => fl.Value.LastMatchTimestamp).FirstOrDefault(); faceLists.Remove(oldestFaceList.Key); await FaceServiceHelper.DeleteFaceListAsync(oldestFaceList.Key); } string newFaceListId = Guid.NewGuid().ToString(); await FaceServiceHelper.CreateFaceListAsync(newFaceListId, "ManagedFaceList", FaceListsUserDataFilter); faceLists.Add(newFaceListId, new FaceListInfo { FaceListId = newFaceListId, LastMatchTimestamp = DateTime.Now }); addResult = await FaceServiceHelper.AddFaceToFaceListAsync(newFaceListId, imageStreamCallback, face.FaceRectangle); } if (addResult != null) { bestMatch = new Tuple <SimilarPersistedFace, string>(new SimilarPersistedFace { Confidence = 1, PersistedFaceId = addResult.PersistedFaceId }, null); } } return(bestMatch?.Item1); }
public async Task IdentifyFacesAsync() { this.IdentifiedPersons = Enumerable.Empty <IdentifiedPerson>(); Guid[] detectedFaceIds = this.DetectedFaces?.Select(f => f.FaceId).ToArray(); if (detectedFaceIds != null && detectedFaceIds.Any()) { List <IdentifiedPerson> result = new List <IdentifiedPerson>(); IEnumerable <PersonGroup> personGroups = Enumerable.Empty <PersonGroup>(); try { personGroups = await FaceServiceHelper.GetPersonGroupsAsync(PeopleGroupsUserDataFilter); } catch (Exception e) { ErrorTrackingHelper.TrackException(e, "Face API GetPersonGroupsAsync error"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure getting PersonGroups"); } } foreach (var group in personGroups) { try { IdentifyResult[] groupResults = await FaceServiceHelper.IdentifyAsync(group.PersonGroupId, detectedFaceIds); foreach (var match in groupResults) { if (!match.Candidates.Any()) { continue; } Person person = await FaceServiceHelper.GetPersonAsync(group.PersonGroupId, match.Candidates[0].PersonId); IdentifiedPerson alreadyIdentifiedPerson = result.FirstOrDefault(p => p.Person.PersonId == match.Candidates[0].PersonId); if (alreadyIdentifiedPerson != null) { // We already tagged this person in another group. Replace the existing one if this new one if the confidence is higher. if (alreadyIdentifiedPerson.Confidence < match.Candidates[0].Confidence) { alreadyIdentifiedPerson.Person = person; alreadyIdentifiedPerson.Confidence = match.Candidates[0].Confidence; alreadyIdentifiedPerson.FaceId = match.FaceId; } } else { result.Add(new IdentifiedPerson { Person = person, Confidence = match.Candidates[0].Confidence, FaceId = match.FaceId }); } } } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Face API IdentifyAsync error"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Failure identifying faces"); } } } this.IdentifiedPersons = result; } this.OnFaceRecognitionCompleted(); }
public async Task <List <FaceSendInfo> > IdentifyOrAddPersonWithEmotionsAsync(string groupName, double confidence) { var time = DateTime.Now; //We also add emotions var facesInfo = new List <FaceSendInfo>(); foreach (var f in this.DetectedFaces) { var fsi = new FaceSendInfo(); var e = CoreUtil.FindEmotionForFace(f, this.DetectedEmotion); fsi.faceId = f.FaceId.ToString(); fsi.age = f.FaceAttributes.Age; fsi.faceRecHeight = f.FaceRectangle.Height; fsi.faceRecLeft = f.FaceRectangle.Left; fsi.faceRecTop = f.FaceRectangle.Top; fsi.faceRecWidth = f.FaceRectangle.Width; fsi.gender = f.FaceAttributes.Gender; fsi.smile = f.FaceAttributes.Smile; fsi.beard = f.FaceAttributes.FacialHair.Beard; fsi.moustache = f.FaceAttributes.FacialHair.Moustache; fsi.sideburns = f.FaceAttributes.FacialHair.Sideburns; fsi.glasses = f.FaceAttributes.Glasses.ToString(); fsi.headYaw = f.FaceAttributes.HeadPose.Yaw; fsi.headRoll = f.FaceAttributes.HeadPose.Roll; fsi.headPitch = f.FaceAttributes.HeadPose.Pitch; fsi.anger = e.Scores.Anger; fsi.contempt = e.Scores.Contempt; fsi.disgust = e.Scores.Disgust; fsi.fear = e.Scores.Fear; fsi.happiness = e.Scores.Happiness; fsi.neutral = e.Scores.Neutral; fsi.sadness = e.Scores.Sadness; fsi.surprise = e.Scores.Surprise; fsi.timeStamp = time; fsi.facesNo = this.DetectedFaces.Count(); facesInfo.Add(fsi); } var newPersonID = Guid.NewGuid(); //Move to initialization try { var g = await FaceServiceHelper.CreatePersonGroupIfNoGroupExists(groupName); groupId = g.PersonGroupId; } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Problem creating group"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem creating group"); } } List <IdentifiedPerson> ipresult = new List <IdentifiedPerson>(); // Compute Face Identification and Unique Face Ids //We need to map detected faceID with actual personID (Identified face) try { IdentifyResult[] groupResults = await this.IdentifyFacesAsync(groupId); foreach (var f in this.DetectedFaces) { if (groupResults != null && groupResults.Where(gr => gr.FaceId == f.FaceId).Any() && groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.Any()) { var candidates = groupResults.Where(gr => gr.FaceId == f.FaceId).FirstOrDefault().Candidates.OrderByDescending(ca => ca.Confidence); var fi = facesInfo.Where(fin => fin.faceId == f.FaceId.ToString()).FirstOrDefault(); int i = 0; Person p = new Person(); foreach (var can in candidates) { switch (i) { case 0: fi.can1id = can.PersonId.ToString(); fi.can1conf = can.Confidence; p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId); fi.can1name = p.Name; if (can.Confidence >= confidence) { ipresult.Add(new IdentifiedPerson() { Person = p, Confidence = can.Confidence, FaceId = f.FaceId }); if (p.PersistedFaceIds.Length == 248) { Guid persistedFaceId = p.PersistedFaceIds.OrderBy(x => Guid.NewGuid()).FirstOrDefault(); await FaceServiceHelper.DeletePersonFaceAsync(groupId, can.PersonId, persistedFaceId); } try { await FaceServiceHelper.AddPersonFaceAsync(groupId, can.PersonId, await this.GetImageStreamCallback(), "", f.FaceRectangle); } catch (Exception e) { // Catch errors with individual groups so we can continue looping through all groups. Maybe an answer will come from // another one. ErrorTrackingHelper.TrackException(e, "Problem adding face to group"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem adding face to group"); } } } else { //create new Guy if confidence not sufficient SimilarFaceMatch result = await GetSimilarFace(f); //using (var db = new KioskDbContext()) //{ // Blogs.ItemsSource = db.Blogs.ToList(); //} await CreatePerson(facesInfo, newPersonID, f); } break; case 1: fi.can2id = can.PersonId.ToString(); fi.can2conf = can.Confidence; p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId); fi.can2name = p.Name; break; case 2: fi.can3id = can.PersonId.ToString(); fi.can3conf = can.Confidence; p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId); fi.can3name = p.Name; break; case 3: fi.can4id = can.PersonId.ToString(); fi.can4conf = can.Confidence; p = await FaceServiceHelper.GetPersonAsync(groupId, can.PersonId); fi.can4name = p.Name; break; } i++; } } else { //if no candidate we also need to create new person await CreatePerson(facesInfo, newPersonID, f); } try { await FaceServiceHelper.TrainPersonGroupAsync(groupId); } catch (Exception e) { // Catch error with training of group ErrorTrackingHelper.TrackException(e, "Problem training group"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem training group"); } } } } catch (Exception e) { // Catch error with training of group ErrorTrackingHelper.TrackException(e, "Problem with cognitive services"); if (this.ShowDialogOnFaceApiErrors) { await ErrorTrackingHelper.GenericApiCallExceptionHandler(e, "Problem with cognitive services"); } } return(facesInfo); }