public static void UpdateFaceRectangle(this azure.FaceRectangle sourceRectangle, ref FaceRectangle targetRectangle)
{
targetRectangle.Width = sourceRectangle.Width;
targetRectangle.Height = sourceRectangle.Height;
targetRectangle.Left = sourceRectangle.Left;
targetRectangle.Top = sourceRectangle.Top;
}
/// <summary>
/// Initializes a new instance of the DetectedFace class.
/// </summary>
public DetectedFace(FaceRectangle faceRectangle, System.Guid?faceId = default(System.Guid?), FaceLandmarks faceLandmarks = default(FaceLandmarks), FaceAttributes faceAttributes = default(FaceAttributes))
{
FaceId = faceId;
FaceRectangle = faceRectangle;
FaceLandmarks = faceLandmarks;
FaceAttributes = faceAttributes;
CustomInit();
}
/// <summary>
/// Initializes a new instance of the DetectedFace class.
/// </summary>
/// <param name="recognitionModel">Possible values include:
/// 'recognition_01', 'recognition_02', 'recognition_03',
/// 'recognition_04'</param>
public DetectedFace(FaceRectangle faceRectangle, System.Guid?faceId = default(System.Guid?), string recognitionModel = default(string), FaceLandmarks faceLandmarks = default(FaceLandmarks), FaceAttributes faceAttributes = default(FaceAttributes))
{
FaceId = faceId;
RecognitionModel = recognitionModel;
FaceRectangle = faceRectangle;
FaceLandmarks = faceLandmarks;
FaceAttributes = faceAttributes;
CustomInit();
}
public static FaceRectangle CreateFaceRectangle(this azure.FaceRectangle faceRectangle)
{
return(new FaceRectangle
{
Width = faceRectangle.Width,
Height = faceRectangle.Height,
Left = faceRectangle.Left,
Top = faceRectangle.Top
});
}
/// <summary>
/// Validate the object.
/// </summary>
/// <exception cref="ValidationException">
/// Thrown if validation fails
/// </exception>
public virtual void Validate()
{
if (FaceRectangle == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "FaceRectangle");
}
if (FaceRectangle != null)
{
FaceRectangle.Validate();
}
if (FaceLandmarks != null)
{
FaceLandmarks.Validate();
}
}
private void AddFacialLandmark(Face.FaceRectangle faceRect, Face.Coordinate coordinate, double scaleX, double scaleY, Rectangle rect)
{
double dotSize = 3;
rect.Fill = new SolidColorBrush(Colors.White);
rect.Width = dotSize;
rect.Height = dotSize;
rect.HorizontalAlignment = HorizontalAlignment.Left;
rect.VerticalAlignment = VerticalAlignment.Top;
rect.RenderTransform = new ScaleTransform {
ScaleX = 1, ScaleY = 1
};
rect.Margin = new Thickness(
((coordinate.X - faceRect.Left) * scaleX) - dotSize / 2,
((coordinate.Y - faceRect.Top) * scaleY) - dotSize / 2,
0, 0);
}
public static void DisplayFaceLandmarks(Grid grid, Face.FaceRectangle faceRect, Face.FaceLandmarks landmarks,
double scaleX, double scaleY, SolidColorBrush color = null)
{
// Mouth (6)
SolidColorBrush colorBrush = color ?? new SolidColorBrush(Colors.White);
AddFacialLandmark(grid, faceRect, landmarks.MouthLeft, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.MouthRight, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.UpperLipBottom, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.UpperLipTop, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.UnderLipBottom, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.UnderLipTop, scaleX, scaleY, colorBrush);
// Eyes (10)
colorBrush = color ?? new SolidColorBrush(Colors.Red);
AddFacialLandmark(grid, faceRect, landmarks.EyeLeftBottom, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyeLeftTop, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyeLeftInner, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyeLeftOuter, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyeRightBottom, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyeRightTop, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyeRightInner, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyeRightOuter, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.PupilLeft, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.PupilRight, scaleX, scaleY, colorBrush);
// nose (7)
colorBrush = color ?? new SolidColorBrush(Colors.LimeGreen);
AddFacialLandmark(grid, faceRect, landmarks.NoseLeftAlarOutTip, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.NoseLeftAlarTop, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.NoseRightAlarOutTip, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.NoseRightAlarTop, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.NoseRootLeft, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.NoseRootRight, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.NoseTip, scaleX, scaleY, colorBrush);
// eyebrows (4)
colorBrush = color ?? new SolidColorBrush(Colors.Yellow);
AddFacialLandmark(grid, faceRect, landmarks.EyebrowLeftInner, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyebrowLeftOuter, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyebrowRightInner, scaleX, scaleY, colorBrush);
AddFacialLandmark(grid, faceRect, landmarks.EyebrowRightOuter, scaleX, scaleY, colorBrush);
}
public void DisplayFaceLandmarks(Face.FaceRectangle faceRect, Face.FaceLandmarks landmarks, double scaleX, double scaleY)
{
// Mouth (6)
AddFacialLandmark(faceRect, landmarks.MouthLeft, scaleX, scaleY, this.mouthleft);
AddFacialLandmark(faceRect, landmarks.MouthRight, scaleX, scaleY, this.mouthright);
AddFacialLandmark(faceRect, landmarks.UpperLipBottom, scaleX, scaleY, this.upperlipbottom);
AddFacialLandmark(faceRect, landmarks.UpperLipTop, scaleX, scaleY, this.upperliptop);
AddFacialLandmark(faceRect, landmarks.UnderLipBottom, scaleX, scaleY, this.underlipbottom);
AddFacialLandmark(faceRect, landmarks.UnderLipTop, scaleX, scaleY, this.underliptop);
// Eyes (10)
AddFacialLandmark(faceRect, landmarks.EyeLeftBottom, scaleX, scaleY, eyeleftbottom);
AddFacialLandmark(faceRect, landmarks.EyeLeftTop, scaleX, scaleY, eyelefttop);
AddFacialLandmark(faceRect, landmarks.EyeLeftInner, scaleX, scaleY, eyeleftinner);
AddFacialLandmark(faceRect, landmarks.EyeLeftOuter, scaleX, scaleY, eyeleftouter);
AddFacialLandmark(faceRect, landmarks.EyeRightBottom, scaleX, scaleY, eyerightbottom);
AddFacialLandmark(faceRect, landmarks.EyeRightTop, scaleX, scaleY, eyerighttop);
AddFacialLandmark(faceRect, landmarks.EyeRightInner, scaleX, scaleY, eyerightinner);
AddFacialLandmark(faceRect, landmarks.EyeRightOuter, scaleX, scaleY, eyerightouter);
AddFacialLandmark(faceRect, landmarks.PupilLeft, scaleX, scaleY, pupilleft);
AddFacialLandmark(faceRect, landmarks.PupilRight, scaleX, scaleY, pupilright);
// nose (7)
AddFacialLandmark(faceRect, landmarks.NoseLeftAlarOutTip, scaleX, scaleY, noseleftalarouttip);
AddFacialLandmark(faceRect, landmarks.NoseLeftAlarTop, scaleX, scaleY, noseleftalartop);
AddFacialLandmark(faceRect, landmarks.NoseRightAlarOutTip, scaleX, scaleY, noserightalarouttip);
AddFacialLandmark(faceRect, landmarks.NoseRightAlarTop, scaleX, scaleY, noserightalartop);
AddFacialLandmark(faceRect, landmarks.NoseRootLeft, scaleX, scaleY, noserootleft);
AddFacialLandmark(faceRect, landmarks.NoseRootRight, scaleX, scaleY, noserootright);
AddFacialLandmark(faceRect, landmarks.NoseTip, scaleX, scaleY, nosetip);
// eyebrows (4)
AddFacialLandmark(faceRect, landmarks.EyebrowLeftInner, scaleX, scaleY, eyebrowleftinner);
AddFacialLandmark(faceRect, landmarks.EyebrowLeftOuter, scaleX, scaleY, eyebrowleftouter);
AddFacialLandmark(faceRect, landmarks.EyebrowRightInner, scaleX, scaleY, eyebrowrightinner);
AddFacialLandmark(faceRect, landmarks.EyebrowRightOuter, scaleX, scaleY, eyebrowrightouter);
this.mainGrid.Visibility = Visibility.Visible;
this.FaceLandmarksStoryboard.Begin();
}
private static void AddFacialLandmark(Grid grid, Face.FaceRectangle rect, Face.Coordinate coordinate,
double scaleX, double scaleY, SolidColorBrush color)
{
double dotSize = 3;
Rectangle b = new Rectangle
{
Fill = color,
Width = dotSize,
Height = dotSize,
HorizontalAlignment = HorizontalAlignment.Left, VerticalAlignment = VerticalAlignment.Top
};
b.RenderTransform = new ScaleTransform {
ScaleX = 1, ScaleY = 1
};
b.Margin = new Thickness(
((coordinate.X - rect.Left) * scaleX) - dotSize / 2,
((coordinate.Y - rect.Top) * scaleY) - dotSize / 2,
0, 0);
grid.Children.Add(b);
}
private static string GetDisplayTextForPersonAsync(ImageAnalyzer analyzer, SimilarFaceMatch item)
{
// See if we identified this person against a trained model
IdentifiedPerson identifiedPerson = analyzer.IdentifiedPersons.FirstOrDefault(p => p.FaceId == item.Face.FaceId);
if (identifiedPerson != null)
{
return(identifiedPerson.Person.Name);
}
if (identifiedPerson == null)
{
// Let's see if this is a celebrity
if (analyzer.AnalysisResult?.Categories != null)
{
foreach (var category in analyzer.AnalysisResult.Categories.Where(c => c.Detail != null))
{
foreach (var celebrity in category.Detail.Celebrities)
{
var celebrityFaceRectangle = new Microsoft.Azure.CognitiveServices.Vision.Face.Models.FaceRectangle(
celebrity.FaceRectangle.Width,
celebrity.FaceRectangle.Height,
celebrity.FaceRectangle.Left,
celebrity.FaceRectangle.Top);
if (CoreUtil.AreFacesPotentiallyTheSame(celebrityFaceRectangle, item.Face.FaceRectangle))
{
return(celebrity.Name.ToString());
}
}
}
}
}
return(string.Empty);
}
private async Task ProcessPeopleInsightsAsync(ImageAnalyzer analyzer, int frameNumber)
{
foreach (var item in analyzer.SimilarFaceMatches)
{
bool demographicsChanged = false;
Visitor personInVideo;
Guid persistedFaceId = item.SimilarPersistedFace.PersistedFaceId.GetValueOrDefault();
if (this.peopleInVideo.TryGetValue(persistedFaceId, out personInVideo))
{
personInVideo.Count++;
if (this.pendingIdentificationAttemptCount.ContainsKey(persistedFaceId))
{
// This is a face we haven't identified yet. See how many times we have tried it, if we need to do it again or stop trying
if (this.pendingIdentificationAttemptCount[persistedFaceId] <= 5)
{
string personName = GetDisplayTextForPersonAsync(analyzer, item);
if (string.IsNullOrEmpty(personName))
{
// Increment the times we have tried and failed to identify this person
this.pendingIdentificationAttemptCount[persistedFaceId]++;
}
else
{
// Bingo! Let's remove it from the list of pending identifications
this.pendingIdentificationAttemptCount.Remove(persistedFaceId);
VideoTrack existingTrack = (VideoTrack)this.peopleListView.Children.FirstOrDefault(f => (Guid)((FrameworkElement)f).Tag == persistedFaceId);
if (existingTrack != null)
{
existingTrack.DisplayText = ShowAgeAndGender ?
string.Format("{0}, {1}", personName, Math.Floor(item.Face.FaceAttributes.Age.GetValueOrDefault())) :
personName;
}
}
}
else
{
// Give up
this.pendingIdentificationAttemptCount.Remove(persistedFaceId);
}
}
}
else
{
// New person... let's catalog it.
// Crop the face, enlarging the rectangle so we frame it better
double heightScaleFactor = 1.8;
double widthScaleFactor = 1.8;
var biggerRectangle = new Microsoft.Azure.CognitiveServices.Vision.Face.Models.FaceRectangle
{
Height = Math.Min((int)(item.Face.FaceRectangle.Height * heightScaleFactor), FrameRelayVideoEffect.LatestSoftwareBitmap.PixelHeight),
Width = Math.Min((int)(item.Face.FaceRectangle.Width * widthScaleFactor), FrameRelayVideoEffect.LatestSoftwareBitmap.PixelWidth)
};
biggerRectangle.Left = Math.Max(0, item.Face.FaceRectangle.Left - (int)(item.Face.FaceRectangle.Width * ((widthScaleFactor - 1) / 2)));
biggerRectangle.Top = Math.Max(0, item.Face.FaceRectangle.Top - (int)(item.Face.FaceRectangle.Height * ((heightScaleFactor - 1) / 1.4)));
var croppedImage = await Util.GetCroppedBitmapAsync(analyzer.GetImageStreamCallback, biggerRectangle.ToRect());
if (croppedImage == null || biggerRectangle.Height == 0 && biggerRectangle.Width == 0)
{
// Couldn't get a shot of this person
continue;
}
demographicsChanged = true;
string personName = GetDisplayTextForPersonAsync(analyzer, item);
if (string.IsNullOrEmpty(personName))
{
if (ShowAgeAndGender)
{
personName = item.Face.FaceAttributes.Gender?.ToString();
}
// Add the person to the list of pending identifications so we can try again on some future frames
this.pendingIdentificationAttemptCount.Add(persistedFaceId, 1);
}
personInVideo = new Visitor {
UniqueId = persistedFaceId
};
this.peopleInVideo.Add(persistedFaceId, personInVideo);
this.demographics.Visitors.Add(personInVideo);
// Update the demographics stats.
this.UpdateDemographics(item);
VideoTrack videoTrack = new VideoTrack
{
Tag = persistedFaceId,
CroppedFace = croppedImage,
DisplayText = ShowAgeAndGender ? string.Format("{0}, {1}", personName, Math.Floor(item.Face.FaceAttributes.Age.GetValueOrDefault())) : personName,
Duration = (int)this.videoPlayer.NaturalDuration.TimeSpan.TotalSeconds,
};
videoTrack.Tapped += this.TimelineTapped;
this.peopleListView.Children.Insert(0, videoTrack);
}
// Update the timeline for this person
VideoTrack track = (VideoTrack)this.peopleListView.Children.FirstOrDefault(f => (Guid)((FrameworkElement)f).Tag == persistedFaceId);
if (track != null)
{
track.SetVideoFrameState(frameNumber, item.Face.FaceAttributes.Emotion);
uint childIndex = (uint)this.peopleListView.Children.IndexOf(track);
if (childIndex > 5)
{
// Bring to towards the top so it becomes visible
this.peopleListView.Children.Move(childIndex, 5);
}
}
if (demographicsChanged)
{
this.ageGenderDistributionControl.UpdateData(this.demographics);
}
this.overallStatsControl.UpdateData(this.demographics);
}
}
public static Rect ToRect(this Face.FaceRectangle rect)
{
return(new Rect(rect.Left, rect.Top, rect.Width, rect.Height));
}
public static bool AreFacesPotentiallyTheSame(Face.FaceRectangle face1, Face.FaceRectangle face2)
{
return(AreFacesPotentiallyTheSame(face1.Left, face1.Top, face1.Width, face1.Height, face2.Left, face2.Top, face2.Width, face2.Height));
}
internal static bool AreFacesPotentiallyTheSame(BitmapBounds face1, Face.FaceRectangle face2)
{
return(CoreUtil.AreFacesPotentiallyTheSame((int)face1.X, (int)face1.Y, (int)face1.Width, (int)face1.Height, face2.Left, face2.Top, face2.Width, face2.Height));
}
private void UpdateFaceRectangle(azure.FaceRectangle faceRectangle)
{
faceRectangle.UpdateFaceRectangle(ref _faceRectangle);
RaisePropertyChanged(() => FaceRectangle);
}