ShapeModel smodel; //shape model #endregion Fields #region Constructors public FaceTracker(string filepath) { points = new List<Point[]> (); string jsonText = null; using (System.IO.StreamReader reader = new System.IO.StreamReader(filepath)) { jsonText = reader.ReadToEnd (); reader.Close (); } // TextAsset textAsset = Resources.Load (filename) as TextAsset; // string jsonText = textAsset.text; IDictionary json = (IDictionary)Json.Deserialize (jsonText); IDictionary ft = (IDictionary)json ["ft object"]; detector = new FaceDetector (); detector.read (ft ["detector"]); smodel = new ShapeModel (); smodel.read (ft ["smodel"]); pmodel = new PatchModels (); pmodel.read (ft ["pmodel"]); }
PatchModels pmodel; //feature detectors public FaceTracker (string filepath) { points = new List<Point[]> (); string jsonText = null; #if UNITY_WSA var data = File.ReadAllBytes(filepath); jsonText = Encoding.UTF8.GetString(data, 0, data.Length); #else jsonText = File.ReadAllText(filepath); #endif // TextAsset textAsset = Resources.Load (filename) as TextAsset; // string jsonText = textAsset.text; IDictionary json = (IDictionary)Json.Deserialize (jsonText); IDictionary ft = (IDictionary)json ["ft object"]; detector = new FaceDetector (); detector.read (ft ["detector"]); smodel = new ShapeModel (); smodel.read (ft ["smodel"]); pmodel = new PatchModels (); pmodel.read (ft ["pmodel"]); }
public void ControlLoad(object sender, EventArgs e) { var environmentService = new EnvironmentService(); var haarEyeFile = new FileInfo(environmentService.GetAbsolutePathFromAssemblyRelative("haarcascades/haarcascade_eye.xml")); var haarFaceFile = new FileInfo(environmentService.GetAbsolutePathFromAssemblyRelative("haarcascades/haarcascade_frontalface_default.xml")); _faceDetector = new FaceDetector(haarFaceFile.FullName, haarEyeFile.FullName); _sunglassOverlay2 = new AccessoryOverlay(environmentService.GetAbsolutePathFromAssemblyRelative("Resources/Images/sunglasses2.png")); _hatOverlay1 = new AccessoryOverlay(environmentService.GetAbsolutePathFromAssemblyRelative("Resources/Images/partyhat.png")); _classiferParams = new ClassifierParameters(); classifierConfigControl.ConfigChanged += classifierConfigControl_ConfigChanged; }
async Task OnProcessFrameAsync(SoftwareBitmap bitmap) { if (this.faceDetector == null) { this.faceDetector = await FaceDetector.CreateAsync(); } var faces = await this.faceDetector.DetectFacesAsync(bitmap); this.cameraDisplay.ShowCamera(faces.Count > 0); if (faces.Count > 0) { foreach (var face in faces) { this.cameraDisplay.HighlightFace(face.FaceBox); } } }
private static void FindManyFaces(FaceDetector faceDetector, string imagesFilePath, string outputDir, bool debug) { if (!Directory.Exists(outputDir)) { Directory.CreateDirectory(outputDir); } var images = Directory.EnumerateFiles(imagesFilePath, "*.jpg"); int cnt = 0; foreach (var image in images) { var outputFile = Path.Combine(outputDir, $"output_{cnt}.png"); FindFaces(faceDetector, image, outputFile, debug); cnt++; } }
public int InitializeWithEyeList(BackgroundCanvas mainCanvas, string filename, List <Point> leftEyeList, List <Point> rightEyeList) { if (null == _detector) { _detector = new FaceDetector( mainCanvas.OptionDialog.FaceDetectorDataPath, true, mainCanvas.OptionDialog.FaceDetectorThreshold); } // VIOLA: Removed face detection... since we have the eye are are relying on them List <Rect> faceRects = new List <Rect>(); for (int i = 0; i < leftEyeList.Count; ++i) { Rect rect = new Rect(); Vector leye = new Vector(leftEyeList[i].X, leftEyeList[i].Y); Vector reye = new Vector(rightEyeList[i].X, rightEyeList[i].Y); Vector delta = reye - leye; double eyeWidth = delta.Length; Vector center = leye + 0.5 * delta; rect.X = center.X - 1.5 * eyeWidth; rect.Y = center.Y - 1.0 * eyeWidth; rect.Width = 3.0 * eyeWidth; rect.Height = 3.0 * eyeWidth; faceRects.Add(rect); } _targetRect = new Rect(); _faceDisplayWidth = mainCanvas.OptionDialog.FaceDisplayWidth; _defaultDPI = mainCanvas.OptionDialog.DefaultDPI; _leftEyeList = leftEyeList; _rightEyeList = rightEyeList; return(InitializeInternal(mainCanvas, filename, faceRects, mainCanvas.OptionDialog.BorderWidth, null)); }
void AllocateObjects(ResourceSet resources) { _resources = resources; _faceDetector = new FaceDetector(_resources.blazeFace); _faceMesh = new FaceLandmarkDetector(_resources.faceMesh); _irisMeshL = new EyeLandmarkDetector(_resources.iris); _irisMeshR = new EyeLandmarkDetector(_resources.iris); _cropMaterial = new Material(_resources.cropShader); _faceCrop = new RenderTexture(192, 192, 0); _irisCropL = new RenderTexture(64, 64, 0); _irisCropR = new RenderTexture(64, 64, 0); _refineBuffer = new ComputeBuffer(FaceLandmarkDetector.VertexCount, sizeof(float) * 4); _eyeToFace = IndexTable.CreateEyeToFaceLandmarkBuffer(); }
public async void ProcessCurrentVideoFrame(ThreadPoolTimer timer) { if (!frameProcessingSemaphore.Wait(0)) { return; } IList <DetectedFace> detectedFaces = null; try { const BitmapPixelFormat faceDetectionPixelFormat = BitmapPixelFormat.Nv12; using (VideoFrame previewFrame = new VideoFrame(faceDetectionPixelFormat, (int)this.videoProperties.Width, (int)this.videoProperties.Height)) { await this.mediaCapture.GetPreviewFrameAsync(previewFrame); if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat)) { detectedFaces = await this.faceTracker.ProcessNextFrameAsync(previewFrame); } else { frameProcessingSemaphore.Release(); return; } var previewFrameSize = new Size(previewFrame.SoftwareBitmap.PixelWidth, previewFrame.SoftwareBitmap.PixelHeight); var ignored = this.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { this.SetupVisualization(previewFrameSize, detectedFaces); }); } } catch (Exception e) { // Face tracking failed } finally { frameProcessingSemaphore.Release(); } }
/// <summary> /// /// </summary> /// <param name="timer"></param> private async void CurrentVideoFrame(ThreadPoolTimer timer) { if (!semaphore.Wait(0)) { return; } try { IList <DetectedFace> faces = null; const BitmapPixelFormat inputPixelFormat = BitmapPixelFormat.Nv12; using (VideoFrame previewFrame = new VideoFrame(inputPixelFormat, 320, 240)) { await capture.GetPreviewFrameAsync(previewFrame); //顔検出実行 if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat)) { faces = await faceTracker.ProcessNextFrameAsync(previewFrame); } else { throw new System.NotSupportedException("PixelFormat '" + inputPixelFormat.ToString() + "' is not supported by FaceDetector"); } //顔が検出されたら録画スタート if (faces.Count != 0) { Debug.WriteLine("Found Face"); await startRecoding(); } } } catch (Exception ex) { Debug.WriteLine(ex.Message); } finally { semaphore.Release(); } }
public static IEnumerable <FaceModel> FaceAnalysis(FaceMaskDetector faceMaskDetector, byte[] imageData, IEnumerable <FaceRectangle> faceRectangles, bool mask) { if (mask) { IEnumerable <Rectangle> rectangles = faceRectangles.Select(faceRectangle => new Rectangle(faceRectangle.Top, faceRectangle.Left, faceRectangle.Width, faceRectangle.Height)); IEnumerable <Bitmap> faces = rectangles.Select(rectangle => FaceDetector.GetFaceImage(imageData, rectangle).ToBitmap()); IEnumerable <bool> isMasks = faceMaskDetector.Detect(faces); return(faceRectangles.Zip(isMasks, (faceRectangle, isMask) => new FaceModel { FaceRectangle = faceRectangle, FaceAttributes = new FaceAttributes { IsMask = isMask } })); } return(faceRectangles.Select(faceRectangle => new FaceModel { FaceRectangle = faceRectangle, FaceAttributes = new FaceAttributes() })); }
/// <summary> /// Initializes a new MediaCapture instance and starts the Preview streaming to the CamPreview UI element. /// </summary> /// <returns>Async Task object returning true if initialization and streaming were successful and false if an exception occurred.</returns> private async Task <bool> StartWebcamStreamingAsync() { bool successful = false; this.faceDetector = await FaceDetector.CreateAsync(); try { this.mediaCapture = new MediaCapture(); // For this scenario, we only need Video (not microphone) so specify this in the initializer. // NOTE: the appxmanifest only declares "webcam" under capabilities and if this is changed to include // microphone (default constructor) you must add "microphone" to the manifest or initialization will fail. MediaCaptureInitializationSettings settings = new MediaCaptureInitializationSettings(); settings.StreamingCaptureMode = StreamingCaptureMode.Video; await this.mediaCapture.InitializeAsync(settings); this.mediaCapture.CameraStreamStateChanged += this.MediaCapture_CameraStreamStateChanged; // Cache the media properties as we'll need them later. var deviceController = this.mediaCapture.VideoDeviceController; this.videoProperties = deviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; // Immediately start streaming to our CaptureElement UI. // NOTE: CaptureElement's Source must be set before streaming is started. this.CamPreview.Source = this.mediaCapture; await this.mediaCapture.StartPreviewAsync(); successful = true; } catch (System.UnauthorizedAccessException) { // If the user has disabled their webcam this exception is thrown; provide a descriptive message to inform the user of this fact. this.rootPage.NotifyUser("Webcam is disabled or access to the webcam is disabled for this app.\nEnsure Privacy Settings allow webcam usage.", NotifyType.ErrorMessage); } catch (Exception ex) { this.rootPage.NotifyUser(ex.ToString(), NotifyType.ErrorMessage); } return(successful); }
/// <summary> /// This method is invoked by a ThreadPoolTimer to execute the FaceTracker and Visualization logic at approximately 15 frames per second. /// </summary> /// <remarks> /// Keep in mind this method is called from a Timer and not synchronized with the camera stream. Also, the processing time of FaceTracker /// will vary depending on the size of each frame and the number of faces being tracked. That is, a large image with several tracked faces may /// take longer to process. /// </remarks> /// <param name="timer">Timer object invoking this call</param> private async void ProcessCurrentVideoFrame(ThreadPoolTimer timer) { // If a lock is being held it means we're still waiting for processing work on the previous frame to complete. // In this situation, don't wait on the semaphore but exit immediately. if (!frameProcessingSemaphore.Wait(0)) { return; } try { IList <DetectedFace> faces = null; // Create a VideoFrame object specifying the pixel format we want our capture image to be (NV12 bitmap in this case). // GetPreviewFrame will convert the native webcam frame into this format. const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Nv12; using (VideoFrame previewFrame = new VideoFrame(InputPixelFormat, (int)this.videoProperties.Width, (int)this.videoProperties.Height)) { await this.camera.MediaCapture.GetPreviewFrameAsync(previewFrame); // The returned VideoFrame should be in the supported NV12 format but we need to verify this. if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat)) { faces = await this.faceTracker.ProcessNextFrameAsync(previewFrame); } else { throw new System.NotSupportedException("PixelFormat '" + InputPixelFormat.ToString() + "' is not supported by FaceDetector"); } // Create our visualization using the frame dimensions and face results but run it on the UI thread. var previewFrameSize = new Windows.Foundation.Size(previewFrame.SoftwareBitmap.PixelWidth, previewFrame.SoftwareBitmap.PixelHeight); var ignored = this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () => { this.SetupVisualization(previewFrameSize, faces); }); } } catch (Exception ex) { Debug.WriteLine(ex); } finally { frameProcessingSemaphore.Release(); } }
private void Initialize(object sender, PipelineRunEventArgs e) { string rootDirectory = AppDomain.CurrentDomain.BaseDirectory; faceModelParameters = new FaceModelParameters(rootDirectory, true, false, false); faceModelParameters.optimiseForVideo(); faceDetector = new FaceDetector(faceModelParameters.GetHaarLocation(), faceModelParameters.GetMTCNNLocation()); if (!faceDetector.IsMTCNNLoaded()) { faceModelParameters.SetFaceDetector(false, true, false); } landmarkDetector = new CLNF(faceModelParameters); faceAnalyser = new FaceAnalyser(rootDirectory, dynamic: true, output_width: 112, mask_aligned: true); gazeAnalyser = new GazeAnalyser(); landmarkDetector.Reset(); faceAnalyser.Reset(); }
protected ExtractorBase(FaceModelParameters faceModelParameters) { if (_initialized) { return; } ModelParams = faceModelParameters; GazeAnalyzer = new GazeAnalyserManaged(); var face_detector = new FaceDetector(ModelParams.GetHaarLocation(), ModelParams.GetMTCNNLocation()); if (!face_detector.IsMTCNNLoaded()) // If MTCNN model not available, use HOG { ModelParams.SetFaceDetector(false, true, false); } FaceModel = new CLNF(ModelParams); _initialized = true; }
public void StartDetection() { if (CoreXT.IsDevice) { if (_faceDetector == null) { // create detector with low accuracy and don't track faces _faceDetector = new FaceDetector(false, false); // shrink image to 12.5% first for faster detection _faceDetector.preprocessImageScale = 0.125f; } _isDetecting = true; } else { Debug.Log("Not on device."); } }
IEnumerator Start() { Input.backButtonLeavesApp = true; Screen.sleepTimeout = SleepTimeout.NeverSleep; _detector = GetComponent <FaceDetector>(); _apiManager = GetComponent <FaceApiManager>(); _toMatHelperMgr = GetComponent <WebCamTextureToMatHelperManager>(); _toMatHelper = GetComponent <WebCamTextureToMatHelper>(); //カメラ等の初期化完了後,画像サイズを取得する _toMatHelper.Initialize(); yield return(WaitInitialization()); var imgSize = new Size(_toMatHelper.GetWidth(), _toMatHelper.GetHeight()); _zeroMat = new ZeroMat(imgSize); _volume.profile.TryGetSettings(out _distortion); }
void Start() { if (CoreXT.IsDevice) webCam = new WebCamTexture("Front Camera"); else webCam = new WebCamTexture(); RotatableGUIItem guiItem = ((RotatableGUIItem)(GameObject.Find ("CameraVideo").GetComponent("RotatableGUIItem"))); guiItem.texture = webCam; guiItem.size = new Vector2(Screen.width / 8, Screen.height / 8); guiItem.relativePosition = new Vector2(-Screen.width / 16, Screen.height / 16 + 75); if (CoreXT.IsDevice) { faceDetector = new FaceDetector(false, false); faceDetector.IsMirrored = true; } StartDetect(); }
public async Task <Rect?> ProcessCameraFrameAsync(SoftwareBitmap bitmap) { if (this.faceDetector == null) { this.faceDetector = await FaceDetector.CreateAsync(); } var result = await this.faceDetector.DetectFacesAsync(bitmap); this.photoControl.Switch(result?.Count > 0); Rect?returnValue = null; if (result?.Count > 0) { returnValue = new Rect( (double)result[0].FaceBox.X / bitmap.PixelWidth, (double)result[0].FaceBox.Y / bitmap.PixelHeight, (double)result[0].FaceBox.Width / bitmap.PixelWidth, (double)result[0].FaceBox.Height / bitmap.PixelHeight); } return(returnValue); }
private async void _timer_Tick(object sender, object e) { try { this.FaceCanvas.Children.Clear(); IEnumerable <DetectedFace> faces = null; // Create a VideoFrame object specifying the pixel format we want our capture image to be (NV12 bitmap in this case). // GetPreviewFrame will convert the native webcam frame into this format. const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Nv12; using (VideoFrame previewFrame = new VideoFrame(InputPixelFormat, 1280, 720)) { await this._mediaCapture.GetPreviewFrameAsync(previewFrame); // The returned VideoFrame should be in the supported NV12 format but we need to verify this. if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat)) { faces = await this.faceTracker.ProcessNextFrameAsync(previewFrame); } } if (faces != null) { foreach (DetectedFace face in faces) { Face.Margin = new Thickness(face.FaceBox.X, face.FaceBox.Y, 0, 0); //faceBorder.ShowFaceRectangle(0, 0, (uint)(face.FaceBox.Width), (uint)(face.FaceBox.Height )); FaceText.Text = face.FaceBox.X.ToString() + face.FaceBox.Y.ToString(); } } PicBtn.Content = DateTime.Now.ToString(); await Task.Delay(50); } catch (Exception) { } }
/// <summary> /// Creates and initializes a FaceSentimentAnalyzerSkill instance /// </summary> /// <param name="descriptor"></param> /// <param name="device"></param> /// <returns></returns> internal static IAsyncOperation <FaceSentimentAnalyzerSkill> CreateAsync( ISkillDescriptor descriptor, ISkillExecutionDevice device) { return(AsyncInfo.Run(async(token) => { // Create instance var skillInstance = new FaceSentimentAnalyzerSkill(descriptor, device); // Instantiate the FaceDetector skillInstance.m_faceDetector = await FaceDetector.CreateAsync(); // Load WinML model var modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Contoso.FaceSentimentAnalyzer/{FaceSentimentAnalyzerConst.WINML_MODEL_FILENAME}")); var winmlModel = LearningModel.LoadFromFilePath(modelFile.Path); // Create WinML session skillInstance.m_winmlSession = new LearningModelSession(winmlModel, GetWinMLDevice(device)); return skillInstance; })); }
public async Task <int> DetectFacesAsync(byte[] photoByteArray) { BitmapDecoder decoder = await BitmapDecoder.CreateAsync(photoByteArray.ToRandomAccessMemory()); BitmapTransform transform = new BitmapTransform(); const float sourceImageHeightLimit = 1280; if (decoder.PixelHeight > sourceImageHeightLimit) { float scalingFactor = (float)sourceImageHeightLimit / (float)decoder.PixelHeight; transform.ScaledWidth = (uint)Math.Floor(decoder.PixelWidth * scalingFactor); transform.ScaledHeight = (uint)Math.Floor(decoder.PixelHeight * scalingFactor); } SoftwareBitmap sourceBitmap = await decoder.GetSoftwareBitmapAsync(decoder.BitmapPixelFormat, BitmapAlphaMode.Premultiplied, transform, ExifOrientationMode.IgnoreExifOrientation, ColorManagementMode.DoNotColorManage); SoftwareBitmap convertedBitmap = sourceBitmap; if (sourceBitmap.BitmapPixelFormat != faceDetectionPixelFormat) { convertedBitmap = SoftwareBitmap.Convert(sourceBitmap, faceDetectionPixelFormat); } FaceDetector detector = await FaceDetector.CreateAsync(); IList <DetectedFace> faces = null; faces = await detector.DetectFacesAsync(convertedBitmap); /* ICollection<System.Drawing.Rectangle> rectangles = new List<System.Drawing.Rectangle>(); * * foreach(DetectedFace face in faces) * rectangles.Add(new System.Drawing.Rectangle(Convert.ToInt32(face.FaceBox.X), Convert.ToInt32(face.FaceBox.Y), Convert.ToInt32(face.FaceBox.Width), Convert.ToInt32(face.FaceBox.Height))); */ sourceBitmap.Dispose(); convertedBitmap.Dispose(); return(faces.Count); }
/// <summary> /// Initialize a photo - run faceDetection /// </summary> /// <param name="mainCanvas">Main canvas reference</param> /// <param name="filename">Full path name to image file</param> public int InitializeWithFaceDetection(BackgroundCanvas mainCanvas, string filename) { if (null == _detector) { _detector = new FaceDetector( mainCanvas.OptionDialog.FaceDetectorDataPath, true, mainCanvas.OptionDialog.FaceDetectorThreshold); } _detector.SetTargetDimension(mainCanvas.OptionDialog.FaceDetectTargetWidth, mainCanvas.OptionDialog.FaceDetectTargetHeight); DetectionResult detectionResult = _detector.DetectObject(filename); List <ScoredRect> scoredResultList = detectionResult.GetMergedRectList(0.0F); if (scoredResultList.Count < 0) { return(0); } List <Rect> faceRects = new List <Rect>(); foreach (ScoredRect scoredRect in scoredResultList) { Rect rect = new Rect(); rect.X = scoredRect.X; rect.Y = scoredRect.Y; rect.Width = scoredRect.Width; rect.Height = scoredRect.Height; faceRects.Add(rect); } _targetRect = new Rect(); _faceDisplayWidth = mainCanvas.OptionDialog.FaceDisplayWidth; _defaultDPI = mainCanvas.OptionDialog.DefaultDPI; return(InitializeInternal(mainCanvas, filename, faceRects, mainCanvas.OptionDialog.BorderWidth, null)); }
private async void ProcessCurrentVideoFrame(ThreadPoolTimer timer) { if (m_mediaCapture == null) { return; } if (m_mediaCapture.CameraStreamState != CameraStreamState.Streaming) { return; } if (!m_faceProcessingSemaphore.Wait(0)) { return; } IList <DetectedFace> faces = null; const BitmapPixelFormat inputPixelFormat = BitmapPixelFormat.Nv12; using (VideoFrame previewFrame = new VideoFrame(inputPixelFormat, (int)m_videoProperties.Width, (int)m_videoProperties.Height)) { await m_mediaCapture.GetPreviewFrameAsync(previewFrame); if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat)) { faces = await m_faceTracker.ProcessNextFrameAsync(previewFrame); } }; foreach (DetectedFace face in faces) { Debug.Log(string.Format("x={0}, y={1}, w={2}, h={3}", face.FaceBox.X, face.FaceBox.Y, face.FaceBox.Width, face.FaceBox.Height)); } m_faceProcessingSemaphore.Release(); }
private void StartFaceDetection(MediaCapture capture) { if (capture == null) { throw new ArgumentNullException(); } if (_faceDetectionCancellationTokenSource != null) { return; } _faceDetectionCancellationTokenSource = new CancellationTokenSource(); var token = _faceDetectionCancellationTokenSource.Token; _faceDetectionTask = Task.Factory.StartNew(async() => { try { FaceDetector detector = new FaceDetector(); detector.Initialize(new FaceDetectorInitializationData() { FaceData = Package.Current.InstalledLocation.Path + "\\" + FacePredictorFileName }); while (!token.IsCancellationRequested) { await FaceDetectAsync(detector, capture, token); } } catch (OperationCanceledException) { } catch (Exception e) { Debug.WriteLine("Face detection failed."); Debug.WriteLine(e.Message); } }, token); }
private async Task <string> DetectEmotionWithWinML() { var videoFrame = lastFrame; if (faceDetector == null) { faceDetector = await FaceDetector.CreateAsync(); } var detectedFaces = await faceDetector.DetectFacesAsync(videoFrame.SoftwareBitmap); if (detectedFaces != null && detectedFaces.Any()) { var face = detectedFaces.OrderByDescending(s => s.FaceBox.Height * s.FaceBox.Width).First(); var randomAccessStream = new InMemoryRandomAccessStream(); var decoder = await BitmapDecoder.CreateAsync(randomAccessStream); var croppedImage = await decoder.GetSoftwareBitmapAsync(decoder.BitmapPixelFormat, BitmapAlphaMode.Ignore, new BitmapTransform() { Bounds = new BitmapBounds() { X = face.FaceBox.X, Y = face.FaceBox.Y, Width = face.FaceBox.Width, Height = face.FaceBox.Height } }, ExifOrientationMode.IgnoreExifOrientation, ColorManagementMode.DoNotColorManage); videoFrame = VideoFrame.CreateWithSoftwareBitmap(croppedImage); } var emotion = await model.EvaluateAsync(new CNTKGraphModelInput() { Input338 = videoFrame }); var index = emotion.Plus692_Output_0.IndexOf(emotion.Plus692_Output_0.Max()); string label = labels[index]; return(label); }
private async Task InitializeCamera() { _requestStopCancellationToken = new CancellationTokenSource(); _captureElement = new CaptureElement(); var videoCaptureDevices = await DeviceInformation.FindAllAsync(DeviceClass.VideoCapture); var camera = videoCaptureDevices.FirstOrDefault(); MediaCaptureInitializationSettings initialisationSettings = new MediaCaptureInitializationSettings() { StreamingCaptureMode = StreamingCaptureMode.Video, VideoDeviceId = camera.Id }; _mediaCapture = new MediaCapture(); await _mediaCapture.InitializeAsync(initialisationSettings); _captureElement.Source = _mediaCapture; await _mediaCapture.StartPreviewAsync(); var videoProperties = (_mediaCapture.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties); var videoSize = new Rect(0, 0, videoProperties.Width, videoProperties.Height); var detector = await FaceDetector.CreateAsync(); var bitmap = FaceDetector.GetSupportedBitmapPixelFormats().First(); try { await Task.Run(async() => { VideoFrame frame = new VideoFrame(bitmap, (int)videoSize.Width, (int)videoSize.Height); TimeSpan?lastFrameTime = null; while (true) { if (!_requestStopCancellationToken.Token.IsCancellationRequested) { await _mediaCapture.GetPreviewFrameAsync(frame); if ((!lastFrameTime.HasValue) || (lastFrameTime != frame.RelativeTime)) { var detectedFaces = await detector.DetectFacesAsync(frame.SoftwareBitmap); if (detectedFaces.Count == 1) { var convertedRgba16Bitmap = SoftwareBitmap.Convert(frame.SoftwareBitmap, BitmapPixelFormat.Rgba16); InMemoryRandomAccessStream stream = new InMemoryRandomAccessStream(); BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, stream); encoder.SetSoftwareBitmap(convertedRgba16Bitmap); await encoder.FlushAsync(); var detectedPerson = await _faceService.DetectFace(stream.AsStream()); if (detectedPerson != null && detectedPerson.PersonId.HasValue) { _userService.PersonId = detectedPerson.PersonId.Value; var user = await _userService.GetModelAsync(); if (user == null) { user = new UserProfileModel().RandomData(detectedPerson.Gender); user.PersonId = detectedPerson.PersonId.Value; user.FaceIds.Add(detectedPerson.FaceId.Value); user = await _userService.AddUserAsync(user); } await UserViewModel.SetValuesAsync(User, user); } else { // bug: when a person was not detected, the stream gets disposed //stream.Seek(0); stream = new InMemoryRandomAccessStream(); encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, stream); encoder.SetSoftwareBitmap(convertedRgba16Bitmap); await encoder.FlushAsync(); // TODO: ask new user for initial profile data var user = new UserProfileModel().RandomData(detectedPerson.Gender); user.PersonId = await _faceService.CreatePersonAsync(user.FullName); var faceIds = new List <Guid>(); faceIds.Add(await _faceService.AddFaceAsync(user.PersonId, stream.AsStream())); user.FaceIds.AddRange(faceIds); user = await _userService.AddUserAsync(user); await UserViewModel.SetValuesAsync(User, user); } await Task.Delay(CHECK_INTERVAL * 1000, _requestStopCancellationToken.Token); } } lastFrameTime = frame.RelativeTime; } } }, _requestStopCancellationToken.Token); } catch (Microsoft.ProjectOxford.Face.FaceAPIException fex) { Debug.WriteLine(fex.ErrorMessage); } catch (Exception ex) { Debug.WriteLine(ex.Message); } if (_requestStopCancellationToken.IsCancellationRequested) { await _mediaCapture.StopPreviewAsync(); _captureElement.Source = null; _requestStopCancellationToken.Dispose(); } }
void Start() { if (CoreXT.IsDevice) { // subscribes to events GUIXT.MediaPicked += OnMediaPicked; _faceDetector = new FaceDetector(true, false); _imageFilter = new ImageFilter(); } }
private void ProcessIndividualImages(ImageReader reader) { // Make sure the GUI is setup appropriately SetupFeatureExtractionMode(); // Indicate we will start running the thread thread_running = true; // Reload the face landmark detector if needed ReloadLandmarkDetector(); if (!landmark_detector.isLoaded()) { DetectorNotFoundWarning(); EndMode(); thread_running = false; return; } // Setup the parameters optimized for working on individual images rather than sequences face_model_params.optimiseForImages(); // Setup the visualization Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance, false); // Initialize the face detector if it has not been initialized yet if (face_detector == null) { face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation()); } // Initialize the face analyser face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, false, image_output_size, MaskAligned); // Loading an image file var frame = new RawImage(reader.GetNextImage()); var gray_frame = new RawImage(reader.GetCurrentFrameGray()); // For FPS tracking DateTime?startTime = CurrentTime; var lastFrameTime = CurrentTime; // This will be false when the image is not available while (reader.isOpened()) { if (!thread_running) { break; } // Setup recording RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, RecordGaze, RecordHOG, RecordTracked, RecordAligned, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0); RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); // Detect faces here and return bounding boxes List <Rect> face_detections = new List <Rect>(); List <float> confidences = new List <float>(); if (DetectorHOG) { face_detector.DetectFacesHOG(face_detections, gray_frame, confidences); } else if (DetectorCNN) { face_detector.DetectFacesMTCNN(face_detections, frame, confidences); } else if (DetectorHaar) { face_detector.DetectFacesHaar(face_detections, gray_frame, confidences); } // For visualization double progress = reader.GetProgress(); for (int i = 0; i < face_detections.Count; ++i) { bool detection_succeeding = landmark_detector.DetectFaceLandmarksInImage(frame, face_detections[i], face_model_params, gray_frame); var landmarks = landmark_detector.CalculateAllLandmarks(); // Predict action units var au_preds = face_analyser.PredictStaticAUsAndComputeFeatures(frame, landmarks); // Predic eye gaze gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details VisualizeFeatures(frame, visualizer_of, landmarks, landmark_detector.GetVisibilities(), detection_succeeding, i == 0, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation RecordObservation(recorder, visualizer_of.GetVisImage(), i, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0, 0); } frame = new RawImage(reader.GetNextImage()); gray_frame = new RawImage(reader.GetCurrentFrameGray()); // Write out the tracked image if (RecordTracked) { recorder.WriteObservationTracked(); } // Do not cary state accross images landmark_detector.Reset(); face_analyser.Reset(); recorder.Close(); lastFrameTime = CurrentTime; processing_fps.AddFrame(); // TODO how to report errors from the reader here? exceptions? logging? Problem for future versions? } EndMode(); }
private async void OpenImg_Click(object sender, RoutedEventArgs e) { IList <DetectedFace> faces = null; SoftwareBitmap detectorInput = null; WriteableBitmap displaySource = null; try { FileOpenPicker photoPicker = new FileOpenPicker(); photoPicker.ViewMode = PickerViewMode.Thumbnail; photoPicker.SuggestedStartLocation = PickerLocationId.PicturesLibrary; photoPicker.FileTypeFilter.Add(".jpg"); photoPicker.FileTypeFilter.Add(".jpeg"); photoPicker.FileTypeFilter.Add(".png"); photoPicker.FileTypeFilter.Add(".bmp"); photoFile = await photoPicker.PickSingleFileAsync(); if (photoFile == null) { return; } using (IRandomAccessStream fileStream = await photoFile.OpenAsync(Windows.Storage.FileAccessMode.Read)) { BitmapImage bitmapImage = new BitmapImage(); bitmapImage.SetSource(fileStream); sourceImg.Source = bitmapImage; BitmapDecoder decoder = await BitmapDecoder.CreateAsync(fileStream); BitmapTransform transform = this.ComputeScalingTransformForSourceImage(decoder); using (SoftwareBitmap originalBitmap = await decoder.GetSoftwareBitmapAsync(decoder.BitmapPixelFormat, BitmapAlphaMode.Ignore, transform, ExifOrientationMode.IgnoreExifOrientation, ColorManagementMode.DoNotColorManage)) { // face can detect Gray8 file const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Gray8; if (FaceDetector.IsBitmapPixelFormatSupported(InputPixelFormat)) { using (detectorInput = SoftwareBitmap.Convert(originalBitmap, InputPixelFormat)) { // Create a WritableBitmap for our visualization display; copy the original bitmap pixels to wb's buffer. displaySource = new WriteableBitmap(originalBitmap.PixelWidth, originalBitmap.PixelHeight); originalBitmap.CopyToBuffer(displaySource.PixelBuffer); FaceDetector detector = await FaceDetector.CreateAsync(); // should reuse the detect obj faces = await detector.DetectFacesAsync(detectorInput); // Create our display using the available image and face results. this.SetupVisualization(displaySource, faces); } } else { } } } } catch (Exception ex) { this.ClearVisualization(); } }
/// <summary> /// Responds when we navigate to this page. /// </summary> /// <param name="e">Event data</param> protected override async void OnNavigatedTo(NavigationEventArgs e) { this.rootPage = MainPage.Current; // The 'await' operation can only be used from within an async method but class constructors // cannot be labeled as async, and so we'll initialize FaceDetector here. if (this.faceDetector == null) { this.faceDetector = await FaceDetector.CreateAsync(); } }
public FaceDetectionBenchmark() { faceDetector = new FaceDetector(); }
public LocalFaceDetector(FaceDetector faceDetector, IImagePersiter imagePersiter) { _faceDetector = faceDetector; _imagePersiter = imagePersiter; }
public async Task <ImageAnalyzer> CaptureFrameAsync(CurrentFrameModel currentFrame) { try { using (Stream stream = currentFrame.DataCurrent.AsBuffer().AsStream()) { stream.Position = 0; var decoder = await BitmapDecoder.CreateAsync(stream.AsRandomAccessStream()); var softwareBitmap = await decoder.GetSoftwareBitmapAsync(); var detector = await FaceDetector.CreateAsync(); using (SoftwareBitmap convertedBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Gray8)) { faces = await detector.DetectFacesAsync(convertedBitmap, SearchArea); convertedBitmap.Dispose(); } this.NumFacesOnLastFrame = faces.Count(); var previewFrameSize = new Windows.Foundation.Size(softwareBitmap.PixelWidth, softwareBitmap.PixelHeight); this.ShowFaceTrackingVisualization(previewFrameSize, faces); softwareBitmap.Dispose(); stream.Dispose(); } //Không có face thì không phân tích if (this.NumFacesOnLastFrame == 0) { faces = null; CoreUtil.FreeMemory(); return(null); } //Hai khung hình có số lượng khung mật giống nhau quá nửa thì không phân tích nữa if (this.AreFacesStill(this.detectedFacesFromPreviousFrame, faces)) { faces = null; CoreUtil.FreeMemory(); return(null); } this.detectedFacesFromPreviousFrame = faces; imageWithFace = new ImageAnalyzer(currentFrame.DataCurrent); imageWithFace.CameraIPAdres = CameraIPAdres; imageWithFace.imageWidth = InfoSettingFix.FixImageWidth; imageWithFace.imageHeight = InfoSettingFix.FixImageHeight; imageWithFace.CaptureTime = currentFrame.CaptureTime; imageWithFace.ListDetectedFaceJson = JsonConvert.SerializeObject(faces.Select(r => r.FaceBox).ToList()); faces = null; CoreUtil.FreeMemory(); return(imageWithFace); } catch (Exception ex) { CoreUtil.FreeMemory(); return(null); } }
/// <summary> /// detect faces on a picture and draw a square in each face /// </summary> private void detectFaces() { //first check if picture has been taken if (null != cameraBitmap) { //get width of a picture int width = cameraBitmap.Width; //get height of a picture int height = cameraBitmap.Height; //Initialize a facedetector with the picture dimensions and the max number of faces to check FaceDetector detector = new FaceDetector(width, height, MainActivity.MAX_FACES); //Create an array of faces with the number of max faces to check Android.Media.FaceDetector.Face[] faces = new Android.Media.FaceDetector.Face[MainActivity.MAX_FACES]; //create a main bitmap Bitmap bitmap565 = Bitmap.CreateBitmap(width, height, Bitmap.Config.Rgb565); //create a dither paint Paint ditherPaint = new Paint(); //create a draw paint Paint drawPaint = new Paint(); //set true dither to dither paint variable ditherPaint.Dither = true; //set color red for the square drawPaint.Color = Color.Red; //set stroke to style drawPaint.SetStyle(Paint.Style.Stroke); //set stroke width drawPaint.StrokeWidth = 2; //Create a canvas Canvas canvas = new Canvas(); //set bitmap to canvas canvas.SetBitmap(bitmap565); //draw bitmap to canvas canvas.DrawBitmap(cameraBitmap, 0, 0, ditherPaint); //get a number of faces detected int facesFound = detector.FindFaces(bitmap565, faces); //mid face point PointF midPoint = new PointF(); //eye distance variable float eyeDistance = 0.0f; //confidence variable float confidence = 0.0f; //print numbre of faces found System.Console.WriteLine("Number of faces found: " + facesFound); //check if found at least one face if (facesFound > 0) { //for each face draw a red squeare for (int index = 0; index < facesFound; ++index) { // get midpoint of a face faces[index].GetMidPoint(midPoint); //get eye distance eyeDistance = faces[index].EyesDistance(); //get confidence confidence = faces [index].Confidence(); //print all parameters System.Console.WriteLine("Confidence: " + confidence + ", Eye distance: " + eyeDistance + ", Mid Point: (" + midPoint.X + ", " + midPoint.Y + ")"); //draw a square in the picture canvas.DrawRect((int)midPoint.X - eyeDistance, (int)midPoint.Y - eyeDistance, (int)midPoint.X + eyeDistance, (int)midPoint.Y + eyeDistance, drawPaint); } } //get imageview from layout ImageView imageView = (ImageView)FindViewById(Resource.Id.image_view); //set image with the red squares to imageview imageView.SetImageBitmap(bitmap565); } }
public FaceDetectorController(FaceDetector faceDetector, FaceMaskDetector faceMaskDetector) { this.faceDetector = faceDetector; this.faceMaskDetector = faceMaskDetector; }
//Start the process private async void button_Click(object sender, RoutedEventArgs e) { FolderPicker folderPicker = new FolderPicker(); folderPicker.FileTypeFilter.Add(".jpg"); folderPicker.FileTypeFilter.Add(".jpeg"); folderPicker.FileTypeFilter.Add(".png"); folderPicker.FileTypeFilter.Add(".bmp"); folderPicker.ViewMode = PickerViewMode.Thumbnail; StorageFolder photoFolder = await folderPicker.PickSingleFolderAsync(); if (photoFolder == null) { return; } var files = await photoFolder.GetFilesAsync(); List <Scores> E = new List <Scores>(); int[] num = new int[files.Count]; for (int i = 0; i < files.Count; i++) { IRandomAccessStream fileStream = await files[i].OpenAsync(FileAccessMode.Read); BitmapDecoder decoder = await BitmapDecoder.CreateAsync(fileStream); BitmapTransform transform = new BitmapTransform(); const float sourceImageHeightLimit = 1280; if (decoder.PixelHeight > sourceImageHeightLimit) { float scalingFactor = (float)sourceImageHeightLimit / (float)decoder.PixelHeight; transform.ScaledWidth = (uint)Math.Floor(decoder.PixelWidth * scalingFactor); transform.ScaledHeight = (uint)Math.Floor(decoder.PixelHeight * scalingFactor); } SoftwareBitmap sourceBitmap = await decoder.GetSoftwareBitmapAsync(decoder.BitmapPixelFormat, BitmapAlphaMode.Premultiplied, transform, ExifOrientationMode.IgnoreExifOrientation, ColorManagementMode.DoNotColorManage); const BitmapPixelFormat faceDetectionPixelFormat = BitmapPixelFormat.Gray8; SoftwareBitmap convertedBitmap; if (sourceBitmap.BitmapPixelFormat != faceDetectionPixelFormat) { convertedBitmap = SoftwareBitmap.Convert(sourceBitmap, faceDetectionPixelFormat); } else { convertedBitmap = sourceBitmap; } if (faceDetector == null) { faceDetector = await FaceDetector.CreateAsync(); } detectedFaces = await faceDetector.DetectFacesAsync(convertedBitmap); Scores sc = null; if (detectedFaces.Count > 0) { num[i] = detectedFaces.Count; FaceRectangle rectID = new FaceRectangle(); rectID = await UploadAndDetectFaces(files[i].Path); if (rectID != null) { sc = await EstimateEmotions(files[i].Path, rectID); } } E.Add(sc); if (sc != null) { Items.Add(new DataItem(i.ToString(), (int)(sc.Happiness * 100))); } sourceBitmap.Dispose(); fileStream.Dispose(); convertedBitmap.Dispose(); } }