/// <summary> /// Allows to serialize Entry item /// </summary> /// <param name="list">List to be inserted to</param> /// <param name="id">item Name</param> /// <returns>Entry item</returns> public static Entry SerializeBaseClass(List<Entry> list, object id) { try { Entry item = list.Find(x => x.ID == Int64.Parse(id.ToString())); XmlHelper.SaveXml(item, string.Format(@"serialized\{0}_{1}.xml", item.Category, item.Name)); return item; } catch (DirectoryNotFoundException) { Directory.CreateDirectory("serialized"); return SerializeBaseClass(list, id); } catch { return null; } }
public ActionResult EditEmailParsingRule(EditRuleForEmailModel model) { var userId = UserHelper.GetUserByEmail(User.Identity.Name).Id; if (ModelState.IsValid && IsUsersEmailParsingRule(userId, model.Id)) { var rule = emailParsingRulesRepository.Find(w => w.Id == model.Id); bool isUpdatingRule = model.IsUpdatingRule.Equals("C"); List<SelectListItem> saveEmail = new List<SelectListItem> { new SelectListItem {Text = "Email without attachments", Value = "0"}, new SelectListItem {Text = "Email with attachments", Value = "1"} }; if (model.SaveEmailProperties != null) { if (model.SaveEmailProperties.Count > 0) { foreach (var saveEmailProp in model.SaveEmailProperties) { var saveProp = saveEmailPropertiesRepository.Find(w => w.Id == saveEmailProp.Id); if (saveProp != null && saveProp.SaveEmailProp != null) { saveProp.TableId = saveEmailProp.TableId; saveProp.Worksheet = saveEmailProp.Worksheet; saveProp.Column = saveEmailProp.Column; saveProp.SaveEmailProp = saveEmail.Find(w => w.Text == saveEmailProp.SaveEmailProp).Value; } else if (saveProp != null && saveProp.DefaultValue != null) { saveProp.TableId = saveEmailProp.TableId; saveProp.Worksheet = saveEmailProp.Worksheet; saveProp.Column = saveEmailProp.Column; saveProp.DefaultValue = saveEmailProp.DefaultValue; } else if (saveEmailProp.Id == 0 && saveEmailProp.SaveEmailProp != null) { saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty() { TableId = saveEmailProp.TableId, Worksheet = saveEmailProp.Worksheet, Column = saveEmailProp.Column, SaveEmailProp = saveEmailProp.SaveEmailProp, EmailParsingRulesModelsId = saveEmailProp.EmailParsingRulesModelsId }); } else if (saveEmailProp.Id == 0 && saveEmailProp.DefaultValue != null) { saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty() { TableId = saveEmailProp.TableId, Worksheet = saveEmailProp.Worksheet, Column = saveEmailProp.Column, DefaultValue = saveEmailProp.DefaultValue, EmailParsingRulesModelsId = saveEmailProp.EmailParsingRulesModelsId }); } } } } if (rule != null) { rule.AuthorId_Id = userId; rule.Email = HttpUtility.HtmlEncode(model.Email); rule.Subject = HttpUtility.HtmlEncode(model.Subject); rule.PlainContains = HttpUtility.HtmlEncode(model.PlainContains); rule.IsUpdatingRule = isUpdatingRule; rule.Status = Model.Enums.RuleStatusEnums.Active.ToString(); rule.CopyRuleFromPreviousOrder = model.CopyRuleFromPreviousOrder.Equals("C"); saveEmailPropertiesRepository.Commit(); emailParsingRulesRepository.Commit(); return RedirectToAction("EditEmailParsingRule", new { ruleId = rule.Id }); } } return View(); }
public long AddEmptySaveEmailProperty(EditRuleForEmailModel model) { var userId = UserHelper.GetUserByEmail(User.Identity.Name).Id; if (ModelState.IsValid && IsUsersEmailParsingRule(userId, model.Id)) { var tables = tablesRepository.FindAll(w => w.AspNetUsers.Id.Equals(userId)).Select(l => l.Name).ToList(); var rule = emailParsingRulesRepository.Find(w => w.Id == model.Id); long spreadsheetId = ParseHelper.GetSpreadsheetIdByName(userId, model.TableId); List<SelectListItem> saveEmail = new List<SelectListItem> { new SelectListItem {Text = "Email without attachments", Value = "0"}, new SelectListItem {Text = "Email with attachments", Value = "1"} }; if (rule != null) { rule.AuthorId_Id = userId; rule.Email = HttpUtility.HtmlEncode(model.Email); rule.Subject = HttpUtility.HtmlEncode(model.Subject); rule.PlainContains = HttpUtility.HtmlEncode(model.PlainContains); //rule.IsUpdatingRule = isUpdatingRule; rule.Status = Model.Enums.RuleStatusEnums.Active.ToString(); rule.TableId = spreadsheetId; var saveEmailProperties = model.SaveEmailProperties.Where(w => w.SaveEmailProp != null).ToList(); if (saveEmailProperties.Count == 0) { saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty() { TableId = tables[0], Worksheet = "0", Column = "A", SaveEmailProp = saveEmail[0].Value, EmailParsingRulesModelsId = rule.Id }); } else if (saveEmailProperties.Count > 0 && saveEmailProperties.First().Id == 0) { saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty() { TableId = model.SaveEmailProperties.First().TableId, Worksheet = model.SaveEmailProperties.First().Worksheet, Column = model.SaveEmailProperties.First().Column, SaveEmailProp = saveEmail.Find(w => w.Text == model.SaveEmailProperties.First().SaveEmailProp).Value, EmailParsingRulesModelsId = rule.Id }); saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty() { TableId = tables[0], Worksheet = "0", Column = "A", SaveEmailProp = saveEmail[0].Value, EmailParsingRulesModelsId = rule.Id }); } else { foreach (var saveEmailProp in saveEmailProperties) { var saveProp = saveEmailPropertiesRepository.Find(w => w.Id == saveEmailProp.Id); saveProp.TableId = saveEmailProp.TableId; saveProp.Worksheet = saveEmailProp.Worksheet; saveProp.Column = saveEmailProp.Column; saveProp.SaveEmailProp = saveEmail.Find(w => w.Text == saveEmailProp.SaveEmailProp).Value; } saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty() { TableId = tables[0], Worksheet = "0", Column = "A", SaveEmailProp = saveEmail[0].Value, EmailParsingRulesModelsId = rule.Id }); } saveEmailPropertiesRepository.Commit(); emailParsingRulesRepository.Commit(); return rule.Id; // return RedirectToAction("AddNewDataRuleForEmails", new { ruleId = rule.Id }); } return 0; } return 0; }
public ActionResult RedactorAdd(long spreadsheetId) { ViewBag.SpreadsheetId = spreadsheetId; var userId = UserHelper.GetUserByEmail(User.Identity.Name).Id; var readactorList = spreadsheetRepository.FindAll(t => t.AdministratorId_Id == userId); List<string> redactorsIds = new List<string>(); var redactorsOfThisTable = new List<string>(); var redactors = new List<RedactorOfTable>(); foreach (var table in readactorList) { redactorsIds = (from myObj in redactorRepository.FindAll(w => w.TableId == table.Id) orderby myObj.UserId_Id select myObj.UserId_Id).ToList(); foreach (var user in redactorsIds) { var userEmail = userRepository.Find(u => u.Id == user).Email; var rf = new RedactorOfTable() { IsRedactorOfThisTable = false, RedactorEmail = userEmail}; var rt = new RedactorOfTable() { IsRedactorOfThisTable = true, RedactorEmail = userEmail }; if (redactors.Find(r=>r.RedactorEmail == userEmail) == null && table.Id != spreadsheetId) { redactors.Add(rf); } else if (table.Id == spreadsheetId) { redactors.RemoveAll(r => r.RedactorEmail == userEmail); redactors.Add(rt); } } } return View("AddRedactorPopupContent", redactors); }
private static void ParseEmailData(long emailId, long ruleId) { // AddLog("ParseEmailData"); IRepository<EmailModels> emailRepository = new Repository<EmailModels>(); var email = emailRepository.Find(w => w.Id == emailId); if (email != null) { int row = -1; List<int> rows = new List<int>(); IRepository<EmailParsingRulesModels> emailParsingRulesRepository = new Repository<EmailParsingRulesModels>(); var emailParsingRule = emailParsingRulesRepository.Find(w => w.Id == ruleId); if (emailParsingRule != null) { // AddLog("emailParsingRule != null"); List<Models.TablesList> tablesList = new List<Models.TablesList>(); IRepository<Model.SaveEmailProperty> saveEmailPropertiesRepository = new Repository<Model.SaveEmailProperty>(); var allDefaultValues = saveEmailPropertiesRepository.FindAll(i => i.EmailParsingRulesModelsId == emailParsingRule.Id).ToList(); IRepository<Model.TableModels> tablesRepository = new Repository<TableModels>(); IRepository<EmailDataParsingRulesModels> emailDataParsingRulesRepository = new Repository<EmailDataParsingRulesModels>(); var emailDataParsingRules = emailDataParsingRulesRepository.FindAll( w => w.EmailParsingRulesModelsId_Id == emailParsingRule.Id && w.Status.Equals(Model.Enums.RuleStatusEnums.Active.ToString())).OrderBy(r => r.Name).ToList(); if (emailDataParsingRules != null && emailDataParsingRules.Count != 0) { // AddLog("emailDataParsingRules.Count != 0"); foreach (var dataParsingRule in emailDataParsingRules) { if (tablesList.Find(w => w.TableId == dataParsingRule.TableId_Id && w.IsChecked) == null && dataParsingRule.Status != Model.Enums.RuleStatusEnums.Paused.ToString()) { var name = dataParsingRule.TableId_Id.ToString(); var path = SpreadsheetHelper.GetSpreadsheetPathById((long)dataParsingRule.TableId_Id); try { SpreadsheetExtension.LoadDocument(name, path); IWorkbook workbook = SpreadsheetExtension.GetCurrentDocument(name); Worksheet worksheet = workbook.Worksheets[ emailParsingRule.EmailDataParsingRulesModels.FirstOrDefault().Worksheet]; if (emailParsingRule.IsUpdatingRule) { var changing = emailParsingRule.EmailDataParsingRulesModels.FirstOrDefault( w => w.IsUpdatindIndex && w.EmailParsingRulesModelsId_Id == emailParsingRule.Id); var changingIndex = BeforeParseValueFromEmail(email, changing); if (path != null) { foreach (var c in worksheet.Columns[dataParsingRule.Column].ExistingCells) { var value = c.Value.ToString(); if (value.Equals(changingIndex)) { row = c.RowIndex; rows.Add(row); } } byte[] size = workbook.SaveDocument(DocumentFormat.Xlsx); tablesRepository.Find(t => t.Id == dataParsingRule.TableId_Id).Size = size.Length; tablesRepository.Commit(); workbook.SaveDocument(path); SpreadsheetExtension.CloseDocument(name); } } else { worksheet.Rows.Insert(1); row = 1; } if (row != -1) { var author_Id = emailParsingRule.AuthorId_Id; foreach (var defaultValue in allDefaultValues) { if (emailParsingRule.Status != Model.Enums.RuleStatusEnums.Paused.ToString()) { InsertEmailLink(emailId, defaultValue, row, author_Id, emailParsingRule.CopyRuleFromPreviousOrder); } } foreach ( var dataParsing in emailParsingRule.EmailDataParsingRulesModels.Where( w => w.TableId_Id == dataParsingRule.TableId_Id)) { if (dataParsing.Status != Model.Enums.RuleStatusEnums.Paused.ToString()) { if (emailParsingRule.IsUpdatingRule) { foreach (var rowForUpdate in rows) { ParseCellValue(email, dataParsing, rowForUpdate, emailParsingRule.CopyRuleFromPreviousOrder); } rows.Clear(); } else { ParseCellValue(email, dataParsing, row, emailParsingRule.CopyRuleFromPreviousOrder); } } } tablesList.Add(new TablesList { IsChecked = true, TableId = (long)dataParsingRule.TableId_Id }); } } catch { AddLog("Table not found"); } } } } else if (allDefaultValues != null) { var tablesInallDefaultValues = allDefaultValues.GroupBy(a => a.TableId).Select(g => g.First()); foreach (var defaultValue in tablesInallDefaultValues) { try { var name = tablesRepository.Find(w => w.Name == defaultValue.TableId && w.AuthorId_Id == emailParsingRule.AuthorId_Id).Name; var path = SpreadsheetHelper.GetSpreadsheetPathById(tablesRepository.Find(w => w.Name == defaultValue.TableId && w.AuthorId_Id == emailParsingRule.AuthorId_Id).Id); SpreadsheetExtension.LoadDocument(name, path); IWorkbook workbook = SpreadsheetExtension.GetCurrentDocument(name); Worksheet worksheet = workbook.Worksheets[ Convert.ToInt32(defaultValue.Worksheet)]; worksheet.Rows.Insert(1); row = 1; byte[] size = workbook.SaveDocument(DocumentFormat.Xlsx); tablesRepository.Find(w => w.Name == defaultValue.TableId && w.AuthorId_Id == emailParsingRule.AuthorId_Id).Size = size.Length; tablesRepository.Commit(); workbook.SaveDocument(path); SpreadsheetExtension.CloseDocument(name); } catch (Exception ex) { AddLog(ex.Message); } } foreach (var defaultValue in allDefaultValues) { InsertEmailLink(emailId, defaultValue, row, emailParsingRule.AuthorId_Id, emailParsingRule.CopyRuleFromPreviousOrder); } } } } }
/// <summary> /// Metoda tworzenia i uczenia sieci neuronowych. /// </summary> /// <param name="maxMaxEpochs">Maksymalna liczba epok.</param> /// <param name="minMaxEpochs">Minimalna liczba epok.</param> /// <param name="maxEpochsMultiplierStep">Krok mnożący liczby epok dla kolejnego rodzaju sieci. </param> /// <param name="minHiddenLayersMultiplier">Minimalny mnożnik dla warstw ukrytych.</param> /// <param name="maxHiddenLayersMultiplier">Maksymalny mnożnik dla wartw ukrytych.</param> /// <param name="hiddenLayersMultiplierStep">Krok dodający dla mnożnika warstw ukrytych.</param> /// <param name="methodProgressPart">Wartość (0 - 100) określająca jaka część postępu inicjalizacji należy do tej metody.</param> private void CreateNetworks( string path, string trainDataFolder, string testDataFolder, int maxMaxEpochs, int minMaxEpochs, double maxEpochsMultiplierStep, double minHiddenLayersMultiplier, double maxHiddenLayersMultiplier, double hiddenLayersMultiplierStep, double methodProgressPart, double desiredMSE ) { double methodStartProgress = this.InitializationProgress; List<TrainingDataFileParameters> fileList = DataProvider.Instance.GetTrainingDataFilesParameters(trainDataFolder); List<NeuralNetworkParameters> parameters = new List<NeuralNetworkParameters>(); foreach (TrainingDataFileParameters file in fileList) { for (double i = minMaxEpochs; i <= maxMaxEpochs; i *= maxEpochsMultiplierStep) { for (double j = minHiddenLayersMultiplier; j <= maxHiddenLayersMultiplier; j += hiddenLayersMultiplierStep) { parameters.Add(new NeuralNetworkParameters() { fileParameters = file, hiddenLayersMultiplier = j, maxEpochs = (uint)i }); } } } int numberOfNetworksToCreate = parameters.Count; int numberOfCreatedNetworks = 0; NetworkMSE lowestNetworkMSE = null; try { StringReader reader = new StringReader(File.ReadAllText(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path + "LowestMSENetwork.xml")); XmlSerializer serializer = new XmlSerializer(typeof(NetworkMSE)); lowestNetworkMSE = (NetworkMSE)serializer.Deserialize(reader); } catch { lowestNetworkMSE = null; } //small hack if (lowestNetworkMSE != null && lowestNetworkMSE.MSE < desiredMSE) { this.InitializationProgress = 100; return; } string[] testFiles = Directory.GetFiles(testDataFolder); List<TrainingData> testDataList = new List<TrainingData>(); foreach (string testFile in testFiles) { TrainingData td = new TrainingData(); //if (td.ReadTrainFromFile(testDataFolder + "\\" + Path.GetFileName(testFile))) if (td.ReadTrainFromFile(testFile)) { testDataList.Add(td); } } List<TrainingData> trainingDataList = new List<TrainingData>(); foreach (TrainingDataFileParameters file in fileList) { TrainingData td = new TrainingData(); if (td.ReadTrainFromFile(trainDataFolder.Split('\\').Last() + "\\" + file.FileName)) { trainingDataList.Add(td); } } string initStatus = this.InitializationStatus; Directory.CreateDirectory(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path); List<Task> taskList = new List<Task>(); NeuralNet.CallbackType[] callbacksArray = new NeuralNet.CallbackType[this.MaxComputingThreads]; Semaphore threadProgressSemaphore = new Semaphore(1, 1); Semaphore allSem = new Semaphore(1, 1); TrainingData[] threadDataVars = new TrainingData[this.MaxComputingThreads]; for (int i = 1; i <= this.MaxComputingThreads; i++) { int taskNumber = i; threadDataVars[i - 1] = new TrainingData(); Task t = new Task((Action)( () => { while (true) { allSem.WaitOne(); if (parameters.Count == 0) { this.InitializationStatus = initStatus + " " + numberOfNetworksToCreate.ToString() + " / " + numberOfNetworksToCreate.ToString(); this.InitializationProgress = 100; break; } else { NeuralNetworkParameters usedParameters = parameters.First(); parameters.RemoveAt(0); if (!File.Exists(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path + usedParameters.FileName)) { NeuralNet net = new NeuralNet(); List<uint> layers = new List<uint>(); layers.Add((uint)((usedParameters.fileParameters.NumberOfPeriods * 2) - 1)); // inputs layers.Add((uint)(layers[0] * usedParameters.hiddenLayersMultiplier)); // hidden layers.Add(3); // output net.CreateStandardArray(layers.ToArray()); net.SetLearningRate((float)0.7); net.SetActivationFunctionHidden(ActivationFunction.SigmoidSymmetric); net.SetActivationFunctionOutput(ActivationFunction.SigmoidSymmetric); net.Callback += callbacksArray[taskNumber - 1]; threadDataVars[taskNumber - 1] = trainingDataList.Find((e) => ((e.NumInputTrainData == layers[0]) && (e.Input.Length == usedParameters.fileParameters.NumberOfPatterns))); allSem.Release(); net.TrainOnData(threadDataVars[taskNumber - 1], usedParameters.maxEpochs, // max iterations 0,// iterations between report 0 //desired error ); allSem.WaitOne(); net.TestData(testDataList.Find((e) => e.NumInputTrainData == layers[0])); double mse = net.GetMSE(); if (lowestNetworkMSE == null || lowestNetworkMSE.MSE > mse) { lowestNetworkMSE = new NetworkMSE() { MSE = mse, NetworkFileName = usedParameters.FileName }; StringWriter writer = new StringWriter(); XmlSerializer serializer = new XmlSerializer(typeof(NetworkMSE)); serializer.Serialize(writer, lowestNetworkMSE); File.WriteAllText(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path + "LowestMSENetwork.xml", writer.ToString()); } net.Save(path + usedParameters.FileName); } this.InitializationStatus = initStatus + " " + numberOfCreatedNetworks.ToString() + " / " + numberOfNetworksToCreate.ToString(); numberOfCreatedNetworks++; this.InitializationProgress = (numberOfCreatedNetworks * methodProgressPart / numberOfNetworksToCreate) + methodStartProgress; } allSem.Release(); } } ), TaskCreationOptions.LongRunning ); taskList.Add(t); } foreach (Task t in taskList) { t.Start(); } }
private static void CheckTagSetTest(List<TagProcessResult> tagList, TagProcessResult tagToCheck) { var tagWordSynonyms = GetSynonymsForTagWords(tagToCheck); var result = (from x in tagList where x.TagId != tagToCheck.TagId where !x.DuplicateTagId.HasValue where !x.RelatedTagId.HasValue let foundDuplicateTagId = !tagToCheck.IsDuplicateReference && x.TagValue.ToLower() == tagToCheck.TagValue.ToLower() ? x.TagId : (int?)null let foundPartialDuplicateTagId = !tagToCheck.IsRelatedReference && x.WordProcessResultList.Any(a => tagToCheck.WordProcessResultList.Select(wpr => wpr.Word.ToLower()).Contains(a.Word.ToLower())) ? x.TagId : (int?)null let foundSynonymTagId = !tagToCheck.IsRelatedReference && x.WordProcessResultList.Any(w => tagWordSynonyms.Contains(w.Word.ToLower())) ? x.TagId : (int?)null select new { DuplicateTagId = foundDuplicateTagId, RelatedTagId = foundPartialDuplicateTagId ?? foundSynonymTagId }).FirstOrDefault(); if (result != null) { if (result.DuplicateTagId.HasValue) { tagToCheck.DuplicateTagId = result.DuplicateTagId; tagList.Find(t => t.TagId == result.DuplicateTagId).IsDuplicateReference = true; } if (result.RelatedTagId.HasValue) { tagToCheck.RelatedTagId = result.RelatedTagId; tagList.Find(t => t.TagId == result.RelatedTagId).IsRelatedReference = true; } } }