// detect anomalies of given param data according to given model id. // this function is in one task, and might take time. // return null if failed. public ANOMALY Detect(int idModel, Predict_Data data) { string fileName = ""; List <CorrelatedFeatures> correlation = null; lock (L_NormalModels) { // get file name (json file). within our server program deleting files, // is only via the beginning of Main or via AnomalyDetectorsManager_instance.Remove(), and only 1 instance exists, // but we lock L_NormalModels so it won't be removed meanwhile if (L_NormalModels.ContainsKey(idModel) && L_NormalModels[idModel].status == MODEL.Status_Ready) { fileName = L_NormalModels[idModel].FileName(); } if (String.IsNullOrWhiteSpace(fileName)) { return(null); } // load normal model correlation = IO_Util.LoadNormalModel(fileName); } if (correlation == null) { return(null); } try { // get detection (default detection method = "hybrid" , which it's enough if the learning was with hybrid\regression method) var detection = AnomalyDetection.GetDetection(data.predict_data, correlation); if (detection == null) { return(null); } Dictionary <string, string> reason = AnomalyDetection.GetReportTypes(correlation, detection); Dictionary <String, List <Span> > spanDictionary = AnomalyDetection.ToSpanDictionary(detection); if (reason == null || spanDictionary == null) { return(null); } return(new ANOMALY() { anomalies = spanDictionary, reason = reason }); }catch { return(null); } }
// try to learn new normal model and save it in correspond json file. // the learning is in new Task and the return MODEL status field is therfore "pending". // does afterFinishingLearning Action after the learning finished. public MODEL LearnAndAddNewModel(string detectoionType, Train_Data data, Action afterFinishingLearning) { MODEL model; // random new model_id Random rnd = new Random(DateTime.Now.Millisecond); int id = rnd.Next(); // lock L_NormalModels in order to add the new mode_id lock (L_NormalModels) { // check it doesn't appear already, otherwise random new id. while (L_NormalModels.ContainsKey(id) || System.IO.File.Exists(new MODEL() { model_id = id }.FileName())) { id = rnd.Next(); } model = new MODEL() { model_id = id, status = MODEL.Status_Pending, upload_time = DateTime.Now }; L_NormalModels.Add(id, model); } // start new task of learning correlative features Task.Run(() => { try { // learn noraml mode [that might take while], only if id not deleted yet var correlation = AnomalyDetection.GetNormal(data.train_data, detectoionType, () => this.IsExist(id)); if (correlation == null) { throw new Exception(); } // save it to json file bool isSuccess = IO_Util.SaveNormalModel(model.FileName(), correlation, new MODEL() { model_id = model.model_id, status = MODEL.Status_Ready, upload_time = model.upload_time }); if (!isSuccess) { throw new Exception(); } lock (L_NormalModels) { if (L_NormalModels.ContainsKey(id)) { L_NormalModels[id].status = MODEL.Status_Ready; } else { // probably we won't get here since correlaion wil be null if id was deleted from L_NormalModels, // because we sent the lambda ()=>this.IsExist(id) to AnomalyDetection.GetNormal try { System.IO.File.Delete(new MODEL() { model_id = id }.FileName()); } catch { } } } } catch { lock (L_NormalModels) { if (L_NormalModels.ContainsKey(id)) { L_NormalModels[id].status = MODEL.Status_Corrupted; } } } afterFinishingLearning(); }); // return MODEL even before the learning finished // { model_id = id, status = MODEL.Status_Pending, upload_time = DateTime.Now }; return(model); }
public static void Main(string[] args) { Console.WriteLine("Server is on.\n"); // check the NormalModelsDBFolder exists if (!System.IO.Directory.Exists(NormalModelsDBFolder)) { Console.WriteLine("Error: Unable to find folder " + NormalModelsDBFolder); Console.WriteLine(" Which in use for database/IO for models storage"); Console.WriteLine("\nPress Enter to abort."); Console.ReadLine(); return; } // check this program can read\write to NormalModelsDBFolder try { string testFile = NormalModelsDBFolder + "testTestTest9999"; System.IO.File.WriteAllText(testFile, "something"); if (!System.IO.File.Exists(testFile)) { throw new Exception(); } System.IO.File.Delete(testFile); if (System.IO.File.Exists(testFile)) { throw new Exception(); } } catch { Console.WriteLine("Error: Unable to write and delete within " + NormalModelsDBFolder); Console.WriteLine(" Which in use for database/IO for models storage"); Console.WriteLine("\nPress Enter to abort."); Console.ReadLine(); return; } // ask what to do with old normal models, if exist in the NormalModelsDBFolder if (System.IO.Directory.GetFiles(NormalModelsDBFolder, "*" + NormalModelsFileExtension).Length != 0) { // ask if old normal models should be load Console.WriteLine("Do you want to restore prev Normal Models?"); Console.WriteLine($"y yes"); Console.WriteLine($"n no, but they will stay in folder" + NormalModelsDBFolder); Console.WriteLine($"remove no, and ALL (!) *{NormalModelsFileExtension} files in " + NormalModelsDBFolder + " will be removed!"); Console.Write("Enter option [y/n/remove] : "); string input = Console.ReadLine().ToLower().Trim(); while (input != "y" && input != "n" && input != "remove") { Console.Write("Enter option [y/n/remove] : "); input = Console.ReadLine().ToLower().Trim(); } List <MODEL> initList = new List <MODEL>(); // if we need to restore prev normal_model then load it from system-file // it's ok since it's only in the start of the server // and we take only the "info" field typeof MODEL which contains only 3 finite fields, // and therefore we won't need much memory for that. if (input == "y") { try { foreach (string file in System.IO.Directory.GetFiles(NormalModelsDBFolder, "*" + NormalModelsFileExtension)) { initList.Add(IO_Util.RestoreExtendedModelInfo(file).info); } } catch { Console.WriteLine($"Error: unable to load model (*{NormalModelsFileExtension}) files."); initList = new List <MODEL>(); } } // remove ALL NormalModelsFileExtension files within NormalModelsDBFolder if (input == "remove") { try { foreach (string file in System.IO.Directory.GetFiles(NormalModelsDBFolder, "*" + NormalModelsFileExtension)) { System.IO.File.Delete(file); } } catch { Console.WriteLine($"Error: unable to delete *{NormalModelsFileExtension} files."); } } // if input == "n" then nothing happened. Console.WriteLine(); AnomalyDetectionController.InitADM(new AnomalyDetectorsManager(initList)); } else { // no old normal-model, than start new fresh session AnomalyDetectionController.InitADM(new AnomalyDetectorsManager(new List <MODEL>())); } // run the server using mvc of .net asp core 5.0 CreateHostBuilder(args).Build().Run(); }