public async Task <bool> StopAsync() { bool Ret = false; bool err = false; Global.Log("Stopping Deepstack..."); Stopwatch sw = Stopwatch.StartNew(); //Try to get running processes in any case bool success = GetDeepStackRun(); //more than one python process we need to take care of... Sometimes MANY for (int i = 0; i < 20; i++) { if (Global.ProcessValid(PythonProc)) { try { await Task.Run(() => this.PythonProc.process.Kill()); await Task.Delay(100); this.PythonProc = Global.GetaProcessByPath(this.PythonEXE); } catch (Exception ex) { Global.Log("Error: Could not stop DeepStack python.exe process: " + Global.ExMsg(ex)); err = true; } } else { break; } } try { if (Global.ProcessValid(this.RedisProc)) { await Task.Run(() => this.RedisProc.process.Kill()); } } catch (Exception ex) { Global.Log("Error: Could not stop DeepStack redis-server.exe process: " + Global.ExMsg(ex)); err = true; } try { if (Global.ProcessValid(this.ServerProc)) { await Task.Run(() => this.ServerProc.process.Kill()); } } catch (Exception ex) { Global.Log("Could not stop DeepStack server.exe process: " + Global.ExMsg(ex)); err = true; } try { if (Global.ProcessValid(this.DeepStackProc)) { await Task.Run(() => this.DeepStackProc.process.Kill()); } } catch (Exception ex) { Global.Log("Error: Could not stop DeepStack.exe process: " + Global.ExMsg(ex)); err = true; } //takes a while for other python.exe processes to fully stop await Task.Delay(250); if (!err) { this.PythonProc = null; this.RedisProc = null; this.ServerProc = null; this.DeepStackProc = null; this.IsStarted = false; Global.Log("Stopped DeepStack in " + sw.ElapsedMilliseconds + "ms"); Ret = true; } else { Global.Log("Error: Could not stop - This can happen for a few reasons: 1) This tool did not originally START deepstack. 2) If this tool is 32 bit it cannot stop 64 bit Deepstack process. Kill manually via task manager - Server.exe, python.exe, redis-server.exe."); } this.HasError = !Ret; return(Ret); }
private async Task <bool> Start() { bool Ret = false; try { if (!this.IsInstalled) { Global.Log("Error: Cannot start because not installed."); this.IsStarted = false; return(Ret); } else { Global.Log("Starting DeepStack..."); } Stopwatch SW = Stopwatch.StartNew(); //First initialize with the py script Process InitProc = new Process(); InitProc.StartInfo.FileName = this.PythonEXE; InitProc.StartInfo.WorkingDirectory = Path.GetDirectoryName(this.PythonEXE); InitProc.StartInfo.Arguments = "../init.py"; InitProc.StartInfo.UseShellExecute = false; InitProc.StartInfo.CreateNoWindow = true; InitProc.StartInfo.RedirectStandardOutput = true; InitProc.StartInfo.RedirectStandardError = true; InitProc.EnableRaisingEvents = true; InitProc.OutputDataReceived += this.handleinitprocmsg; InitProc.ErrorDataReceived += this.handleinitprocerror; InitProc.Exited += (sender, e) => myProcess_Exited(sender, e, "Init:Python.exe"); //new EventHandler(myProcess_Exited); Global.Log($"Starting {InitProc.StartInfo.FileName} {InitProc.StartInfo.Arguments}..."); InitProc.Start(); InitProc.PriorityClass = ProcessPriorityClass.High; //always run this as high priority since it will initialize faster InitProc.BeginOutputReadLine(); InitProc.BeginErrorReadLine(); //next start the redis server... this.RedisProc = new Global.ClsProcess(); this.RedisProc.process.StartInfo.FileName = this.RedisEXE; this.RedisProc.process.StartInfo.WorkingDirectory = Path.GetDirectoryName(this.RedisEXE); this.RedisProc.process.StartInfo.UseShellExecute = false; this.RedisProc.process.StartInfo.CreateNoWindow = true; this.RedisProc.process.StartInfo.RedirectStandardOutput = true; this.RedisProc.process.StartInfo.RedirectStandardError = true; this.RedisProc.process.EnableRaisingEvents = true; this.RedisProc.process.OutputDataReceived += this.handleredisprocmsg; this.RedisProc.process.ErrorDataReceived += this.handleredisprocerror; this.RedisProc.process.Exited += (sender, e) => myProcess_Exited(sender, e, "Redis.exe"); //new EventHandler(myProcess_Exited); this.RedisProc.FileName = this.RedisEXE; this.RedisProc.CommandLine = this.RedisEXE; Global.Log($"Starting {this.RedisEXE}..."); this.RedisProc.process.Start(); if (AppSettings.Settings.deepstack_highpriority) { this.RedisProc.process.PriorityClass = ProcessPriorityClass.High; } this.RedisProc.process.BeginOutputReadLine(); this.RedisProc.process.BeginErrorReadLine(); //next, start the server this.ServerProc = new Global.ClsProcess(); this.ServerProc.process.StartInfo.FileName = this.ServerEXE; this.ServerProc.process.StartInfo.WorkingDirectory = Path.GetDirectoryName(this.ServerEXE); this.ServerProc.process.StartInfo.Arguments = $"-VISION-FACE={this.FaceAPIEnabled} -VISION-SCENE={this.SceneAPIEnabled} -VISION-DETECTION={this.DetectionAPIEnabled} -ADMIN-KEY={this.AdminKey} -API-KEY={this.APIKey} -PORT={this.Port}"; this.ServerProc.process.StartInfo.CreateNoWindow = true; this.ServerProc.process.StartInfo.UseShellExecute = false; this.ServerProc.process.StartInfo.RedirectStandardOutput = true; this.ServerProc.process.StartInfo.RedirectStandardError = true; this.ServerProc.process.EnableRaisingEvents = true; this.ServerProc.process.OutputDataReceived += this.handleserverprocmsg; this.ServerProc.process.ErrorDataReceived += this.handleserverprocerror; this.ServerProc.process.Exited += (sender, e) => myProcess_Exited(sender, e, "Server.exe"); //new EventHandler(myProcess_Exited); this.ServerProc.FileName = this.ServerEXE; this.ServerProc.CommandLine = this.ServerProc.process.StartInfo.Arguments; Global.Log($"Starting {this.ServerProc.process.StartInfo.FileName} {this.ServerProc.process.StartInfo.Arguments}..."); this.ServerProc.process.Start(); if (AppSettings.Settings.deepstack_highpriority) { this.ServerProc.process.PriorityClass = ProcessPriorityClass.High; } this.ServerProc.process.BeginOutputReadLine(); this.ServerProc.process.BeginErrorReadLine(); //start the python intelligence.py script this.PythonProc = new Global.ClsProcess(); this.PythonProc.process.StartInfo.FileName = this.PythonEXE; this.PythonProc.process.StartInfo.WorkingDirectory = Path.GetDirectoryName(this.PythonEXE); this.PythonProc.process.StartInfo.Arguments = $"../intelligence.py -MODE={this.Mode} -VFACE={this.FaceAPIEnabled} -VSCENE={this.SceneAPIEnabled} -VDETECTION={this.DetectionAPIEnabled}"; this.PythonProc.process.StartInfo.UseShellExecute = false; this.PythonProc.process.StartInfo.CreateNoWindow = true; this.PythonProc.process.EnableRaisingEvents = true; this.PythonProc.process.StartInfo.RedirectStandardOutput = true; this.PythonProc.process.StartInfo.RedirectStandardError = true; this.PythonProc.process.OutputDataReceived += this.handlepythonprocmsg; this.PythonProc.process.ErrorDataReceived += this.handlepythonprocerror; this.PythonProc.process.Exited += (sender, e) => myProcess_Exited(sender, e, "Main:Python.exe"); //new EventHandler(myProcess_Exited); this.PythonProc.FileName = this.PythonEXE; this.PythonProc.CommandLine = this.PythonProc.process.StartInfo.Arguments; Global.Log($"Starting {this.PythonProc.process.StartInfo.FileName} {this.PythonProc.process.StartInfo.Arguments}..."); this.PythonProc.process.Start(); if (AppSettings.Settings.deepstack_highpriority) { this.PythonProc.process.PriorityClass = ProcessPriorityClass.High; } this.PythonProc.process.BeginOutputReadLine(); this.PythonProc.process.BeginErrorReadLine(); this.IsStarted = true; this.HasError = false; Ret = true; //Lets wait for the rest of the python.exe processes to spawn and set their priority too (otherwise they are normal) int cnt = 0; do { List <Global.ClsProcess> montys = Global.GetProcessesByPath(this.PythonEXE); if (montys.Count >= 5) { //when deepstack is running normaly there will be 5 python.exe processes //Set priority for each this way since we didnt start them in the first place... cnt = montys.Count; if (AppSettings.Settings.deepstack_highpriority) { foreach (Global.ClsProcess prc in montys) { if (Global.ProcessValid(prc)) { try { prc.process.PriorityClass = ProcessPriorityClass.High; } catch { } } } } break; } await Task.Delay(100); } while (SW.ElapsedMilliseconds < 10000); //wait 10 seconds max if (cnt == 5) { Global.Log("Started in " + SW.ElapsedMilliseconds + "ms"); } else if (cnt > 5) { this.HasError = true; this.IsStarted = true; Global.Log("Error: More than 5 python.exe processes are running from the deepstack folder? Manually stop/restart. (" + SW.ElapsedMilliseconds + "ms)"); } else if (cnt == 0) { this.HasError = true; this.IsStarted = true; Global.Log("Error: 5 python.exe processes did not fully start in " + SW.ElapsedMilliseconds + "ms"); } } catch (Exception ex) { this.IsStarted = false; this.HasError = true; Global.Log("Error: Cannot start: " + Global.ExMsg(ex)); } return(Ret); }
public bool GetDeepStackRun() { bool Ret = false; //Note - deepstack.exe does NOT need to be running if (!Global.ProcessValid(this.DeepStackProc)) { this.DeepStackProc = Global.GetaProcessByPath(this.DeepStackEXE); } if (!Global.ProcessValid(this.ServerProc)) { this.ServerProc = Global.GetaProcessByPath(this.ServerEXE); } if (!Global.ProcessValid(this.PythonProc)) { this.PythonProc = Global.GetaProcessByPath(this.PythonEXE); } if (!Global.ProcessValid(this.RedisProc)) { this.RedisProc = Global.GetaProcessByPath(this.RedisEXE); } if (Global.ProcessValid(this.ServerProc) && Global.ProcessValid(this.PythonProc) && Global.ProcessValid(this.RedisProc)) { this.IsInstalled = true; this.HasError = false; Global.Log("DeepStack Desktop IS running from " + ServerProc.FileName); this.IsStarted = true; //C:\DeepStack\server\server.exe //check to see if it is a different path than default if (!this.ServerProc.FileName.ToLower().StartsWith(this.DeepStackFolder.ToLower())) { string dspath = this.ServerProc.FileName.ToLower().Replace(@"server\server.exe", ""); Global.Log("Deepstack running from non-default path: " + dspath); this.DeepStackFolder = dspath; this.DeepStackEXE = Path.Combine(this.DeepStackFolder, @"DeepStack.exe"); this.PythonEXE = Path.Combine(this.DeepStackFolder, @"interpreter\python.exe"); this.RedisEXE = Path.Combine(this.DeepStackFolder, @"redis\redis-server.exe"); this.ServerEXE = Path.Combine(this.DeepStackFolder, @"server\server.exe"); this.NeedsSaving = true; } //Try to get command line params to fill in correct running port, etc //"C:\DeepStack\server\server.exe" -VISION-FACE=False -VISION-SCENE=True -VISION-DETECTION=True -ADMIN-KEY= -API-KEY= -PORT=84 string face = Global.GetWordBetween(this.ServerProc.CommandLine, "-VISION-FACE=", " |-"); if (!string.IsNullOrEmpty(face)) { if (this.FaceAPIEnabled != Convert.ToBoolean(face)) { Global.Log($"...Face API detection setting found in running server.exe process changed from '{this.FaceAPIEnabled}' to '{Convert.ToBoolean(face)}'"); this.FaceAPIEnabled = Convert.ToBoolean(face); this.NeedsSaving = true; } } string scene = Global.GetWordBetween(this.ServerProc.CommandLine, "-VISION-SCENE=", " |-"); if (!string.IsNullOrEmpty(scene)) { if (Convert.ToBoolean(scene) != this.SceneAPIEnabled) { Global.Log($"...Scene API detection setting found in running server.exe process changed from '{this.SceneAPIEnabled}' to '{Convert.ToBoolean(scene)}'"); this.SceneAPIEnabled = Convert.ToBoolean(scene); this.NeedsSaving = true; } } ; string detect = Global.GetWordBetween(this.ServerProc.CommandLine, "-VISION-DETECTION=", " |-"); if (!string.IsNullOrEmpty(detect)) { if (this.DetectionAPIEnabled != Convert.ToBoolean(detect)) { Global.Log($"...Detection API detection setting found in running server.exe process changed from '{this.DetectionAPIEnabled}' to '{Convert.ToBoolean(detect)}'"); this.DetectionAPIEnabled = Convert.ToBoolean(detect); this.NeedsSaving = true; } } string admin = Global.GetWordBetween(this.ServerProc.CommandLine, "-ADMIN-KEY=", " |-"); if (!string.IsNullOrEmpty(admin)) { if (this.AdminKey != admin) { Global.Log($"...Admin key setting found in running server.exe process changed from '{this.AdminKey}' to '{admin}'"); this.AdminKey = admin; this.NeedsSaving = true; } } string api = Global.GetWordBetween(this.ServerProc.CommandLine, "-API-KEY=", " |-"); if (!string.IsNullOrEmpty(api)) { if (this.APIKey != api) { Global.Log($"...API key setting found in running server.exe process changed from '{this.APIKey}' to '{api}'"); this.APIKey = api; this.NeedsSaving = true; } } string port = Global.GetWordBetween(this.ServerProc.CommandLine, "-PORT=", " |-"); if (!string.IsNullOrEmpty(port)) { if (this.Port != port) { Global.Log($"...Port setting found in running server.exe process changed from '{this.Port}' to '{port}'"); this.Port = port; this.NeedsSaving = true; } } //Get mode: //"C:\DeepStack\interpreter\python.exe" ../intelligence.py -MODE=Medium -VFACE=False -VSCENE=True -VDETECTION=True string mode = Global.GetWordBetween(this.PythonProc.CommandLine, "-MODE=", " |-"); if (!string.IsNullOrEmpty(port)) { if (this.Mode != mode) { Global.Log($"...Mode setting found in running python.exe process changed from '{this.Mode}' to '{mode}'"); this.Mode = mode; this.NeedsSaving = true; } } //"C:\DeepStack\interpreter\python.exe" "-c" "from multiprocessing.spawn import spawn_main; spawn_main(parent_pid=17744, pipe_handle=328)" "--multiprocessing-fork" } else if (Global.ProcessValid(this.ServerProc) || Global.ProcessValid(this.PythonProc) || Global.ProcessValid(this.RedisProc)) { Global.Log("Error: Deepstack partially running. You many need to manually kill server.exe, python.exe, redis-server.exe"); this.HasError = true; this.IsStarted = true; } else { Global.Log("DeepStack Desktop NOT running."); this.IsStarted = false; this.HasError = false; } return(Ret); }
public bool StartDeepstack(bool ForceRestart = false) { //This error happens when you run out of video memory: //stderr.txt // File "C://DeepStack\windows_packages\torch\cuda\__init__.py", line 480, in _lazy_new // return super(_CudaBase, cls).__new__(cls, *args, **kwargs) //RuntimeError: CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 2.00 GiB total capacity; 35.77 MiB already allocated; 0 bytes free; 38.00 MiB reserved in total by PyTorch) //this error happens after sending an image to deepstack - I believe it is still running out of video memory: // File "C:\DeepStack\intelligencelayer\shared\detection.py", line 138, in objectdetection // os.remove(img_path) //FileNotFoundError: [WinError 2] The system cannot find the file specified: 'C:\\Users\\Vorlon\\AppData\\Local\\Temp\\DeepStack\\83e9c5b0-d698-44f3-a8df-d19655d9f7da' if (this.Starting.ReadFullFence()) { Log("Already starting?"); return(false); } this.Starting.WriteFullFence(true); using var Trace = new Trace(); //This c# 8.0 using feature will auto dispose when the function is done. bool Ret = false; try { if (!this.IsInstalled) { Log("Error: Cannot start because not installed."); this.IsStarted = false; return(Ret); } else { if (this.IsStarted) { if (this.StopBeforeStart || ForceRestart) { Log("Debug: Stopping already running DeepStack instance..."); this.StopDeepstack(); Thread.Sleep(250); } else { Log("Debug: Deepstack is already running, not re-starting due to 'deepstack_stopbeforestart' setting = false in aitool.settings.json file."); return(Ret); } } Log("Starting DeepStack..."); } Stopwatch SW = Stopwatch.StartNew(); this.URLS = ""; if (this.IsNewVersion) { List <string> ports = Global.Split(this.Port, ",|"); this.Count = ports.Count; this.CommandLine = ""; int pcnt = 0; foreach (string CurPort in ports) { if (Global.IsLocalPortInUse(Convert.ToInt32(CurPort))) { Log($"Error: Port {CurPort} is already open, so cannot start deepstack.exe using that port."); continue; } Global.ClsProcess prc = new Global.ClsProcess(); prc.process.StartInfo.FileName = this.DeepStackEXE; prc.process.StartInfo.WorkingDirectory = Path.GetDirectoryName(this.DeepStackEXE); if (this.CustomModelEnabled) { prc.process.StartInfo.Arguments = $"--MODELSTORE-DETECTION \"{this.CustomModelPath}\" --PORT {CurPort}"; } else { string face = ""; string scene = ""; string detect = ""; string admin = ""; string api = ""; string mode = ""; if (this.FaceAPIEnabled) { face = $"--VISION-FACE {this.FaceAPIEnabled} "; } if (this.SceneAPIEnabled) { scene = $"--VISION-SCENE {this.SceneAPIEnabled} "; } if (this.DetectionAPIEnabled) { detect = $"--VISION-DETECTION {this.DetectionAPIEnabled} "; } if (!string.IsNullOrEmpty(this.AdminKey)) { admin = $"--ADMIN-KEY {this.AdminKey} "; } if (!string.IsNullOrEmpty(this.APIKey)) { api = $"--API-KEY {this.APIKey} "; } if (!string.IsNullOrEmpty(this.Mode) && !this.DisplayVersion.Contains("2020")) { mode = $"--MODE {this.Mode} "; } prc.process.StartInfo.Arguments = $"{face}{scene}{detect}{admin}{api}{mode}--PORT {CurPort}"; } if (!AppSettings.Settings.deepstack_debug) { prc.process.StartInfo.CreateNoWindow = true; prc.process.StartInfo.UseShellExecute = false; prc.process.StartInfo.RedirectStandardOutput = true; prc.process.StartInfo.RedirectStandardError = true; prc.process.EnableRaisingEvents = true; prc.process.OutputDataReceived += this.DSHandleServerProcMSG; prc.process.ErrorDataReceived += this.DSHandleServerProcERROR; } else { prc.process.StartInfo.UseShellExecute = false; } if (this.DisplayVersion.Contains("2020") && !string.Equals(this.Mode, "medium", StringComparison.OrdinalIgnoreCase)) { prc.process.StartInfo.EnvironmentVariables["MODE"] = this.Mode; } prc.process.Exited += (sender, e) => this.DSProcess_Exited(sender, e, "deepstack.exe"); //new EventHandler(myProcess_Exited); prc.FileName = this.DeepStackEXE; prc.CommandLine = prc.process.StartInfo.Arguments; pcnt++; Log($"Starting {pcnt} of {ports.Count}: {prc.process.StartInfo.FileName} {prc.process.StartInfo.Arguments}..."); prc.process.Start(); Global.WaitForProcessToStart(prc.process, 3000, this.DeepStackEXE); if (AppSettings.Settings.deepstack_highpriority) { prc.process.PriorityClass = ProcessPriorityClass.High; } if (!AppSettings.Settings.deepstack_debug) { prc.process.BeginOutputReadLine(); prc.process.BeginErrorReadLine(); } this.ServerProc.Add(prc); ClsURLItem url = new ClsURLItem($"http://127.0.0.1:{CurPort}/v1/vision/detection", AppSettings.Settings.AIURLList.Count + 1, URLTypeEnum.DeepStack); this.CommandLine += $"{prc.FileName} {prc.CommandLine}\r\n"; this.URLS += $"{url.ToString()}\r\n"; if (!AppSettings.Settings.AIURLList.Contains(url)) { Log("Automatically adding local Windows Deepstack URL: " + url.ToString()); AppSettings.Settings.AIURLList.Add(url); } Thread.Sleep(250); } this.Count = pcnt; this.IsStarted = true; this.HasError = false; Ret = true; //Lets wait for the rest of the python.exe processes to spawn and set their priority too (otherwise they are normal) int PythonCnt = 0; int cc = 0; int ExpectedPythonCnt = 0; if (this.FaceAPIEnabled) { ExpectedPythonCnt = this.Count * 5; //face runs 5 copies of python.exe } else { ExpectedPythonCnt = this.Count * 2; //normal detection runs 2 copies } do { List <Global.ClsProcess> montys = Global.GetProcessesByPath(this.PythonEXE); PythonCnt = montys.Count; if (montys.Count >= ExpectedPythonCnt) { //when deepstack is running normaly there will be 2 python.exe processes running for each deepstack.exe //Set priority for each this way since we didnt start them in the first place... if (AppSettings.Settings.deepstack_highpriority) { foreach (Global.ClsProcess prc in montys) { cc++; if (Global.ProcessValid(prc)) { try { prc.process.PriorityClass = ProcessPriorityClass.High; prc.process.StartInfo.UseShellExecute = false; prc.process.StartInfo.RedirectStandardOutput = true; prc.process.StartInfo.RedirectStandardError = true; prc.process.EnableRaisingEvents = true; prc.process.OutputDataReceived += this.DSHandlePythonProcMSG; prc.process.ErrorDataReceived += this.DSHandlePythonProcERROR; prc.process.Exited += (sender, e) => this.DSProcess_Exited(sender, e, "python.exe"); //new EventHandler(myProcess_Exited); } catch { } } } } this.PythonProc = montys; break; } else { Log($"Debug: ...Waiting for {ExpectedPythonCnt} copies of {this.PythonEXE} to start (now={montys.Count})..."); Thread.Sleep(250); } } while (SW.ElapsedMilliseconds < 30000); //wait 30 seconds max this.RedisProc = Global.GetProcessesByPath(this.RedisEXE); if (Global.ProcessValid(this.RedisProc)) { if (AppSettings.Settings.deepstack_highpriority) { this.RedisProc[0].process.PriorityClass = ProcessPriorityClass.High; } } else { this.HasError = true; Log($"Error: 1 'redis-server.exe' processes did not start within " + SW.ElapsedMilliseconds + "ms"); } if (PythonCnt >= ExpectedPythonCnt) { Log("Started in " + SW.ElapsedMilliseconds + "ms"); } else { this.HasError = true; Log($"Error: {ExpectedPythonCnt} 'python.exe' processes did not start within " + SW.ElapsedMilliseconds + "ms"); } if (!this.HasError) { this.IsActivated = true; this.IsStarted = true; } } else { Log("Error: DeepStack for Windows v2020 or higher is not installed. https://github.com/johnolafenwa/DeepStack/releases"); } } catch (Exception ex) { this.IsStarted = false; this.HasError = true; Log("Error: Cannot start: " + Global.ExMsg(ex)); } finally { this.Starting.WriteFullFence(false); //this.PrintDeepStackError(); } Global.SendMessage(MessageType.UpdateDeepstackStatus, "Manual start"); return(Ret); }