public OpencvSource(string cam_or_url) { MAssert.Check(cam_or_url != string.Empty); // check if cam_or_url is number bool stream = false; for (int i = 0; i < cam_or_url.Length; ++i) { stream = stream || (cam_or_url[i] < '0') || (cam_or_url[i] > '9'); } if (stream) { // open stream Console.WriteLine("opening stream '{0}'", cam_or_url); capturer = new OpenCvSharp.VideoCapture(cam_or_url); } else { // convert to integer int cam_id = Convert.ToInt32(cam_or_url, 10); MAssert.Check(cam_id >= 0, "wrong webcam id"); // open vebcam Console.WriteLine("opening webcam {0}", cam_id); capturer = new OpenCvSharp.VideoCapture(cam_id); MAssert.Check(capturer.IsOpened(), "webcam not opened"); // set resolution capturer.Set(OpenCvSharp.CaptureProperty.FrameWidth, 1280); capturer.Set(OpenCvSharp.CaptureProperty.FrameHeight, 720); MAssert.Check(capturer.IsOpened(), "webcam not opened"); } // sometimes first few frames can be empty even if camera is good // so skip few frames OpenCvSharp.Mat frame; for (int i = 0; i < 10; ++i) { frame = capturer.RetrieveMat(); } // check first two frames OpenCvSharp.Mat image1 = new OpenCvSharp.Mat(), image2 = new OpenCvSharp.Mat(); capturer.Read(image1); capturer.Read(image2); Console.WriteLine("image1 size: {0}", image1.Size()); Console.WriteLine("image1 size: {0}", image2.Size()); MAssert.Check( !image1.Empty() && !image2.Empty() && image1.Size() == image2.Size() && image1.Type() == OpenCvSharp.MatType.CV_8UC3 && image2.Type() == OpenCvSharp.MatType.CV_8UC3, "error opening webcam or stream"); }
public DepthMapRaw depth_opts; // all but depth_data and depth_data_stride_in_bytes must be set public DepthMapRaw make_dmr() { MAssert.Check(!depth.Empty()); DepthMapRaw r = depth_opts; r.depth_data_ptr = depth.Data; r.depth_data_stride_in_bytes = (int)depth.Step(); return(r); }
void TrackingLostCallback( VideoWorker.TrackingLostCallbackData data, Object userdata) { MAssert.Check(data != null); MAssert.Check(userdata != null); int stream_id = (int)data.stream_id; int track_id = (int)data.track_id; int first_frame_id = (int)data.first_frame_id; int last_frame_id = (int)data.last_frame_id; float best_quality = (float)data.best_quality; int best_quality_frame_id = (int)data.best_quality_frame_id; RawSample best_quality_sample = data.best_quality_sample; Template best_quality_templ = data.best_quality_templ; // userdata is supposed to be pointer to this Worker // so cast the pointer Worker worker = (Worker)(userdata); // we care only about the worker._stream_id source // so just ignore any others if (stream_id != worker._stream_id) { return; } // here we just remember the time when the face was lost // but the real purpose of this callback if to store faces // for futher matching or processing { worker._drawing_data_mutex.WaitOne(); FaceData face = worker._drawing_data.faces[track_id]; MAssert.Check(!face.lost); face.lost = true; face.lost_time = new TimeSpan(DateTime.Now.Ticks); if (best_quality_sample != null) { face.sample = best_quality_sample; } worker._drawing_data_mutex.ReleaseMutex(); } }
public Worker( Database database, VideoWorker video_worker, ImageAndDepthSource source, int stream_id, Mutex draw_image_mutex, OpenCvSharp.Mat draw_image, float frame_fps_limit) { _draw_image_mutex = draw_image_mutex; _draw_image = draw_image; _video_worker = video_worker; _source = source; _frame_min_delay_ms = 1000f / frame_fps_limit; _stream_id = stream_id; _database = database; // check arguments MAssert.Check(video_worker != null); MAssert.Check(source != null); MAssert.Check(stream_id >= 0 && stream_id < video_worker.getStreamsCount()); // set callbacks arguments _tracking_callback_id = video_worker.addTrackingCallbackU( TrackingCallback, this); _tracking_lost_callback_id = video_worker.addTrackingLostCallbackU( TrackingLostCallback, this); _match_found_callback_id = video_worker.addMatchFoundCallbackU( MatchFoundCallback, this); // start threads _shutdown = false; _drawing_thread = new Thread(DrawingThreadFunc); _camera_thread = new Thread(CameraThreadFunc); _drawing_thread.Start(); _camera_thread.Start(); }
public RealSenseSource() { using (Context ctx = new Context()) { DeviceList devices = ctx.QueryDevices(); Console.WriteLine("There are {0} connected RealSense devices.", devices.Count); MAssert.Check(devices.Count > 0, "there is no RealSense devices"); for (int i = 0; i < devices.Count; ++i) { Console.WriteLine( "Device {0}: name {1}, serial {2}, firmware version: {3}", i, devices[i].Info[CameraInfo.Name], devices[i].Info[CameraInfo.SerialNumber], devices[i].Info[CameraInfo.FirmwareVersion]); } } Config config = new Config(); config.EnableStream(Stream.Depth, 640, 480, Format.Z16); config.EnableStream(Stream.Color, 640, 480, Format.Bgr8); pipeline = new Pipeline(); PipelineProfile pipeline_profile = pipeline.Start(config); depth_scale = pipeline_profile.Device.Sensors[0].DepthScale * 1000; MAssert.Check(depth_scale > 0); align_to_color = new Align(Stream.Color); }
void MatchFoundCallback( VideoWorker.MatchFoundCallbackData data, Object userdata) { MAssert.Check(data != null); MAssert.Check(userdata != null); int stream_id = (int)data.stream_id; int frame_id = (int)data.frame_id; RawSample sample = data.sample; float quality = (float)data.quality; Template templ = data.templ; VideoWorker.SearchResult[] search_results = data.search_results; // userdata is supposed to be pointer to this Worker // so cast the pointer Worker worker = (Worker)(userdata); // we care only about the worker._stream_id source // so just ignore any others if (stream_id != worker._stream_id) { return; } MAssert.Check(sample != null); MAssert.Check(templ != null); MAssert.Check(search_results.Length > 0); // just print distances in the console Console.WriteLine("stream {0} match track {1} : ", stream_id, sample.getID()); for (int i = 0; i < search_results.Length; ++i) { ulong element_id = search_results[i].element_id; if (element_id == (ulong)VideoWorker.MATCH_NOT_FOUND_ID) { MAssert.Check(i == 0); Console.WriteLine(" {0}: MATCH NOT FOUND", i); } else { MAssert.Check(element_id < (UInt64)worker._database.names.Count); Console.WriteLine(" {0}: with '{1}' distance: {2}", i, worker._database.names[(int)element_id], search_results[i].match_result.distance); } } Console.WriteLine(""); ulong match_element_id = search_results[0].element_id; if (match_element_id != (ulong)VideoWorker.MATCH_NOT_FOUND_ID) { MAssert.Check((int)match_element_id < worker._database.thumbnails.Count); // set the match info in the worker._drawing_data.faces worker._drawing_data_mutex.WaitOne(); FaceData face = worker._drawing_data.faces[sample.getID()]; MAssert.Check(!face.lost); face.match_database_index = (int)match_element_id; worker._drawing_data_mutex.ReleaseMutex(); } }
void TrackingCallback( VideoWorker.TrackingCallbackData data, Object userdata) { // check arguments MAssert.Check(data != null); MAssert.Check(userdata != null); int stream_id = (int)data.stream_id; int frame_id = (int)data.frame_id; RawSample[] samples = data.samples; bool[] samples_weak = data.samples_weak; float[] samples_quality = data.samples_quality; MAssert.Check(samples.Length == samples_weak.Length); MAssert.Check(samples.Length == samples_quality.Length); // userdata is supposed to be pointer to this Worker // so cast the pointer Worker worker = (Worker)(userdata); // we care only about the worker._stream_id source // so just ignore any others if (stream_id != worker._stream_id) { return; } // get the frame with frame_id id ImageAndDepth frame = new ImageAndDepth(); { worker._frames_mutex.WaitOne(); // searching in worker._frames for (;;) { // there already must be some frames MAssert.Check(worker._frames.Count > 0); // this frame_id can't be from future MAssert.Check(worker._frames.Peek().Item1 <= frame_id); if (worker._frames.Peek().Item1 == frame_id) { // we found it frame = worker._frames.Dequeue().Item2; break; } else { // this (i.e. the worker._frames.front() ) frame was skipeed by tracking Console.WriteLine("skiped {0}:{1}", stream_id, worker._frames.Dequeue().Item1); } } worker._frames_mutex.ReleaseMutex(); } // update the data { worker._drawing_data_mutex.WaitOne(); // frame worker._drawing_data.frame = frame.image; worker._drawing_data.depth = frame.depth; worker._drawing_data.frame_id = frame_id; worker._drawing_data.updated = true; // and samples for (int i = 0; i < samples.Length; ++i) { if (!worker._drawing_data.faces.ContainsKey(samples[i].getID())) { worker._drawing_data.faces[samples[i].getID()] = new FaceData(); } FaceData face = worker._drawing_data.faces[samples[i].getID()]; face.frame_id = samples[i].getFrameID(); // it can differ from the frame_id face.lost = false; face.weak = samples_weak[i]; face.sample = samples[i]; } worker._drawing_data_mutex.ReleaseMutex(); } }
static int Main(string[] args) { try { // print usage Console.WriteLine("Usage: dotnet csharp_video_recognition_demo.dll [OPTIONS] <video_source>..."); Console.WriteLine("Examples:"); Console.WriteLine(" Webcam: dotnet csharp_video_recognition_demo.dll --config_dir ../../../conf/facerec 0"); Console.WriteLine(" RTSP stream: dotnet csharp_video_recognition_demo.dll --config_dir ../../../conf/facerec rtsp://localhost:8554/"); Console.WriteLine(""); // parse arguments bool error = false; Options options = new Options(); CommandLine.Parser.Default.ParseArguments <Options>(args) .WithParsed <Options>(opts => options = opts) .WithNotParsed <Options>(errs => error = true); // exit if argument parsign error if (error) { return(1); } // print values of arguments Console.WriteLine("Arguments:"); foreach (var opt in options.GetType().GetProperties()) { if (opt.Name == "video_sources") { Console.Write("video sources = "); foreach (string vs in options.video_sources) { Console.Write(vs + " "); } Console.WriteLine(); } else { Console.WriteLine("--{0} = {1}", opt.Name, opt.GetValue(options, null)); } } Console.WriteLine("\n"); //parameters parse string config_dir = options.config_dir; string license_dir = options.license_dir; string database_dir = options.database_dir; string method_config = options.method_config; float recognition_distance_threshold = options.recognition_distance_threshold; float frame_fps_limit = options.frame_fps_limit; List <string> video_sources = new List <string>(options.video_sources); // check params MAssert.Check(config_dir != string.Empty, "Error! config_dir is empty."); MAssert.Check(database_dir != string.Empty, "Error! database_dir is empty."); MAssert.Check(method_config != string.Empty, "Error! method_config is empty."); MAssert.Check(recognition_distance_threshold > 0, "Error! Failed recognition distance threshold."); List <ImageAndDepthSource> sources = new List <ImageAndDepthSource>(); List <string> sources_names = new List <string>(); MAssert.Check(video_sources.Count > 0, "Error! video_sources is empty."); for (int i = 0; i < video_sources.Count; i++) { sources_names.Add(string.Format("OpenCvS source {0}", i)); sources.Add(new OpencvSource(video_sources[i])); } MAssert.Check(sources_names.Count == sources.Count); // print sources Console.WriteLine("\n{0} sources: ", sources.Count); for (int i = 0; i < sources_names.Count; ++i) { Console.WriteLine(" {0}", sources_names[i]); } Console.WriteLine(""); // create facerec servcie FacerecService service = FacerecService.createService( config_dir, license_dir); Console.WriteLine("Library version: {0}\n", service.getVersion()); // create database Recognizer recognizer = service.createRecognizer(method_config, true, false, false); Capturer capturer = service.createCapturer("common_capturer4_lbf_singleface.xml"); Database database = new Database( database_dir, recognizer, capturer, recognition_distance_threshold); recognizer.Dispose(); capturer.Dispose(); FacerecService.Config vw_config = new FacerecService.Config("video_worker_fdatracker_blf_fda.xml"); // vw_config.overrideParameter("single_match_mode", 1); vw_config.overrideParameter("search_k", 10); vw_config.overrideParameter("not_found_match_found_callback", 1); vw_config.overrideParameter("downscale_rawsamples_to_preferred_size", 0); //ActiveLiveness.CheckType[] checks = new ActiveLiveness.CheckType[3] //{ // ActiveLiveness.CheckType.BLINK, // ActiveLiveness.CheckType.TURN_RIGHT, // ActiveLiveness.CheckType.SMILE //}; // create one VideoWorker VideoWorker video_worker = service.createVideoWorker( new VideoWorker.Params() .video_worker_config(vw_config) .recognizer_ini_file(method_config) .streams_count(sources.Count) //.age_gender_estimation_threads_count(sources.Count) //.emotions_estimation_threads_count(sources.Count) //.active_liveness_checks_order(checks) .processing_threads_count(sources.Count) .matching_threads_count(sources.Count)); // set database video_worker.setDatabase(database.vwElements, Recognizer.SearchAccelerationType.SEARCH_ACCELERATION_1); for (int i = 0; i < sources_names.Count; ++i) { OpenCvSharp.Window window = new OpenCvSharp.Window(sources_names[i]); OpenCvSharp.Cv2.ImShow(sources_names[i], new OpenCvSharp.Mat(100, 100, OpenCvSharp.MatType.CV_8UC3, OpenCvSharp.Scalar.All(0))); } // prepare buffers for store drawed results Mutex draw_images_mutex = new Mutex(); List <OpenCvSharp.Mat> draw_images = new List <OpenCvSharp.Mat>(sources.Count); // create one worker per one source List <Worker> workers = new List <Worker>(); for (int i = 0; i < sources.Count; ++i) { draw_images.Add(new OpenCvSharp.Mat(100, 100, OpenCvSharp.MatType.CV_8UC3, OpenCvSharp.Scalar.All(0))); workers.Add(new Worker( database, video_worker, sources[i], i, // stream_id draw_images_mutex, draw_images[i], frame_fps_limit )); } // draw results until escape presssed for (;;) { { draw_images_mutex.WaitOne(); for (int i = 0; i < draw_images.Count; ++i) { OpenCvSharp.Mat drawed_im = workers[i]._draw_image; if (!drawed_im.Empty()) { OpenCvSharp.Cv2.ImShow(sources_names[i], drawed_im); draw_images[i] = new OpenCvSharp.Mat(); } } draw_images_mutex.ReleaseMutex(); } int key = OpenCvSharp.Cv2.WaitKey(20); if (27 == key) { foreach (Worker w in workers) { w.Dispose(); } break; } if (' ' == key) { Console.WriteLine("enable processing 0"); video_worker.enableProcessingOnStream(0); } if (13 == key) { Console.WriteLine("disable processing 0"); video_worker.disableProcessingOnStream(0); } if ('r' == key) { Console.WriteLine("reset trackerOnStream"); video_worker.resetTrackerOnStream(0); } // check exceptions in callbacks video_worker.checkExceptions(); } // force free resources // otherwise licence error may occur // when create sdk object in next time service.Dispose(); video_worker.Dispose(); } catch (Exception e) { Console.WriteLine("video_recognition_show exception catched: '{0}'", e.ToString()); return(1); } return(0); }
public override ImageAndDepth Get() { ImageAndDepth res = new ImageAndDepth(); using (FrameSet frameset = pipeline.WaitForFrames()) { DepthFrame depth_frame = frameset.FirstOrDefault <DepthFrame>(Stream.Depth, Format.Z16).DisposeWith(frameset); if (depth_frame != null && ((long)depth_frame.Number) > prev_depth_frame_number) { prev_depth_frame_number = (long)depth_frame.Number; FrameSet registered = align_to_color.Process(frameset).As <FrameSet>().DisposeWith(frameset); depth_frame = registered.FirstOrDefault <DepthFrame>(Stream.Depth, Format.Z16).DisposeWith(frameset); MAssert.Check(depth_frame != null); MAssert.Check(depth_frame.BitsPerPixel == 16); MAssert.Check(depth_frame.Stride >= depth_frame.Width * 2); float[] fov = depth_frame.Profile.As <VideoStreamProfile>().GetIntrinsics().FOV; res.depth_opts.horizontal_fov = fov[0]; res.depth_opts.vertical_fov = fov[1]; res.depth_opts.depth_unit_in_millimeters = depth_scale; res.depth_opts.depth_map_2_image_offset_x = 0; res.depth_opts.depth_map_2_image_offset_y = 0; res.depth_opts.depth_map_2_image_scale_x = 1; res.depth_opts.depth_map_2_image_scale_y = 1; res.depth_opts.depth_map_rows = depth_frame.Height; res.depth_opts.depth_map_cols = depth_frame.Width; res.depth_timestamp_microsec = (ulong)depth_frame.Timestamp * 1000; res.depth = new OpenCvSharp.MatOfUShort( depth_frame.Height, depth_frame.Width, depth_frame.Data, depth_frame.Stride).Clone(); } VideoFrame video_frame = frameset.FirstOrDefault <VideoFrame>(Stream.Color, Format.Bgr8).DisposeWith(frameset); if (video_frame != null && ((long)video_frame.Number > prev_video_frame_number)) { prev_video_frame_number = (long)video_frame.Number; MAssert.Check(video_frame.BitsPerPixel == 24); MAssert.Check(video_frame.Stride >= video_frame.Width * 3); res.image = new OpenCvSharp.Mat( video_frame.Height, video_frame.Width, OpenCvSharp.MatType.CV_8UC3, video_frame.Data, video_frame.Stride).Clone(); res.image_timestamp_microsec = (ulong)video_frame.Timestamp * 1000; } } return(res); }
// create the database public Database( string databaseDirPath, Recognizer recognizer, Capturer capturer, float distanceThreshold) { vwElements = new List <VideoWorker.DatabaseElement>(); samples = new List <RawSample>(); thumbnails = new List <OpenCvSharp.Mat>(); names = new List <string>(); // check paths MAssert.Check(Directory.Exists(databaseDirPath), "database not found"); // get directory content List <string> path_l1 = new List <string>(Directory.EnumerateDirectories(databaseDirPath)); // check every element in that directory ulong element_id_counter = 0; for (int il1 = 0; il1 < path_l1.Count; ++il1) { // ignore files if (!Directory.Exists(path_l1[il1])) { continue; } // so path_l1[il1] is supposed to be the path to the person directory // get files inside i List <string> path_l2 = new List <string>(Directory.EnumerateFiles(path_l1[il1])); string name = string.Empty; // search for the name.txt file for (int il2 = 0; il2 < path_l2.Count; ++il2) { if (Path.GetFileName(path_l2[il2]) == "name.txt") { // put file content in the name using (StreamReader sr = new StreamReader(path_l2[il2])) { name = sr.ReadToEnd(); } } } // try to open each file as an image for (int il2 = 0; il2 < path_l2.Count; ++il2) { if (Path.GetFileName(path_l2[il2]) == "name.txt") { continue; } Console.WriteLine("processing '{0}' name: '{1}'", path_l2[il2], name); // read image with opencv OpenCvSharp.Mat readed_image = OpenCvSharp.Cv2.ImRead(path_l2[il2]); if (readed_image.Empty() || readed_image.Type() != OpenCvSharp.MatType.CV_8UC3) { Console.WriteLine("\n\nWARNING: can't read image '{0}'\n\n", path_l2[il2]); continue; } byte[] data = new byte[readed_image.Total() * readed_image.Type().Channels]; Marshal.Copy(readed_image.DataStart, data, 0, (int)data.Length); RawImage image = new RawImage(readed_image.Width, readed_image.Height, RawImage.Format.FORMAT_BGR, data); // capture the face List <RawSample> capturedSamples = capturer.capture(image); if (capturedSamples.Count != 1) { Console.WriteLine("\n\nWARNING: detected {0} faces on '{1}' image instead of one, image ignored \n\n", capturedSamples.Count, path_l2[il2]); continue; } RawSample sample = capturedSamples[0]; // make template Template templ = recognizer.processing(sample); // prepare data for VideoWorker VideoWorker.DatabaseElement vwElement = new VideoWorker.DatabaseElement(element_id_counter++, (ulong)il1, templ, distanceThreshold); vwElements.Add(vwElement); samples.Add(sample); thumbnails.Add(makeThumbnail(sample, name)); names.Add(name); } } MAssert.Check((int)element_id_counter == vwElements.Count); MAssert.Check((int)element_id_counter == samples.Count); MAssert.Check((int)element_id_counter == thumbnails.Count); MAssert.Check((int)element_id_counter == names.Count); }
// make a thumbnail of a sample public static OpenCvSharp.Mat makeThumbnail( RawSample sample, string name = "") { int thumbnail_size = Worker.thumbnail_size; // buffer for the cutted image MemoryStream stream = new MemoryStream(); // make a cut in bmp format // so we don't waste time for encode/decode image // just copying it few times, which is irrelevant sample.cutFaceImage( stream, RawSample.ImageFormat.IMAGE_FORMAT_BMP, RawSample.FaceCutType.FACE_CUT_BASE); OpenCvSharp.Mat temp = OpenCvSharp.Mat.ImDecode(stream.ToArray(), OpenCvSharp.ImreadModes.Color); // so we got an image // check it MAssert.Check(!temp.Empty()); MAssert.Check(temp.Type() == OpenCvSharp.MatType.CV_8UC3); // and resize to the thumbnail_size OpenCvSharp.Rect resRect; if (temp.Rows >= temp.Cols) { resRect.Height = thumbnail_size; resRect.Width = temp.Cols * thumbnail_size / temp.Rows; } else { resRect.Width = thumbnail_size; resRect.Height = temp.Rows * thumbnail_size / temp.Cols; } resRect.X = (thumbnail_size - resRect.Width) / 2; resRect.Y = (thumbnail_size - resRect.Height) / 2; OpenCvSharp.Mat result = new OpenCvSharp.Mat( thumbnail_size, thumbnail_size, OpenCvSharp.MatType.CV_8UC3, OpenCvSharp.Scalar.All(0)); OpenCvSharp.Cv2.Resize( temp, result[resRect], resRect.Size); if (!string.IsNullOrEmpty(name)) { result[new OpenCvSharp.Rect(0, result.Rows - 27, result.Cols, 27)] = result.RowRange(result.Rows - 27, result.Rows) * 0.5f; OpenCvSharp.Cv2.PutText( result, name, new OpenCvSharp.Point(0, result.Rows - 7), OpenCvSharp.HersheyFonts.HersheyDuplex, 0.7, OpenCvSharp.Scalar.All(255), 1, OpenCvSharp.LineTypes.AntiAlias); } return(result); }