/// <summary>
/// Allows to serialize Entry item
/// </summary>
/// <param name="list">List to be inserted to</param>
/// <param name="id">item Name</param>
/// <returns>Entry item</returns>
public static Entry SerializeBaseClass(List<Entry> list, object id)
{
try
{
Entry item = list.Find(x => x.ID == Int64.Parse(id.ToString()));
XmlHelper.SaveXml(item, string.Format(@"serialized\{0}_{1}.xml", item.Category, item.Name));
return item;
}
catch (DirectoryNotFoundException)
{
Directory.CreateDirectory("serialized");
return SerializeBaseClass(list, id);
}
catch
{
return null;
}
}
public ActionResult EditEmailParsingRule(EditRuleForEmailModel model)
{
var userId = UserHelper.GetUserByEmail(User.Identity.Name).Id;
if (ModelState.IsValid && IsUsersEmailParsingRule(userId, model.Id))
{
var rule = emailParsingRulesRepository.Find(w => w.Id == model.Id);
bool isUpdatingRule = model.IsUpdatingRule.Equals("C");
List<SelectListItem> saveEmail = new List<SelectListItem>
{
new SelectListItem {Text = "Email without attachments", Value = "0"},
new SelectListItem {Text = "Email with attachments", Value = "1"}
};
if (model.SaveEmailProperties != null)
{
if (model.SaveEmailProperties.Count > 0)
{
foreach (var saveEmailProp in model.SaveEmailProperties)
{
var saveProp = saveEmailPropertiesRepository.Find(w => w.Id == saveEmailProp.Id);
if (saveProp != null && saveProp.SaveEmailProp != null)
{
saveProp.TableId = saveEmailProp.TableId;
saveProp.Worksheet = saveEmailProp.Worksheet;
saveProp.Column = saveEmailProp.Column;
saveProp.SaveEmailProp = saveEmail.Find(w => w.Text == saveEmailProp.SaveEmailProp).Value;
}
else if (saveProp != null && saveProp.DefaultValue != null)
{
saveProp.TableId = saveEmailProp.TableId;
saveProp.Worksheet = saveEmailProp.Worksheet;
saveProp.Column = saveEmailProp.Column;
saveProp.DefaultValue = saveEmailProp.DefaultValue;
}
else if (saveEmailProp.Id == 0 && saveEmailProp.SaveEmailProp != null)
{
saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty()
{
TableId = saveEmailProp.TableId,
Worksheet = saveEmailProp.Worksheet,
Column = saveEmailProp.Column,
SaveEmailProp = saveEmailProp.SaveEmailProp,
EmailParsingRulesModelsId = saveEmailProp.EmailParsingRulesModelsId
});
}
else if (saveEmailProp.Id == 0 && saveEmailProp.DefaultValue != null)
{
saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty()
{
TableId = saveEmailProp.TableId,
Worksheet = saveEmailProp.Worksheet,
Column = saveEmailProp.Column,
DefaultValue = saveEmailProp.DefaultValue,
EmailParsingRulesModelsId = saveEmailProp.EmailParsingRulesModelsId
});
}
}
}
}
if (rule != null)
{
rule.AuthorId_Id = userId;
rule.Email = HttpUtility.HtmlEncode(model.Email);
rule.Subject = HttpUtility.HtmlEncode(model.Subject);
rule.PlainContains = HttpUtility.HtmlEncode(model.PlainContains);
rule.IsUpdatingRule = isUpdatingRule;
rule.Status = Model.Enums.RuleStatusEnums.Active.ToString();
rule.CopyRuleFromPreviousOrder = model.CopyRuleFromPreviousOrder.Equals("C");
saveEmailPropertiesRepository.Commit();
emailParsingRulesRepository.Commit();
return RedirectToAction("EditEmailParsingRule", new { ruleId = rule.Id });
}
}
return View();
}
public long AddEmptySaveEmailProperty(EditRuleForEmailModel model)
{
var userId = UserHelper.GetUserByEmail(User.Identity.Name).Id;
if (ModelState.IsValid && IsUsersEmailParsingRule(userId, model.Id))
{
var tables = tablesRepository.FindAll(w => w.AspNetUsers.Id.Equals(userId)).Select(l => l.Name).ToList();
var rule = emailParsingRulesRepository.Find(w => w.Id == model.Id);
long spreadsheetId = ParseHelper.GetSpreadsheetIdByName(userId, model.TableId);
List<SelectListItem> saveEmail = new List<SelectListItem>
{
new SelectListItem {Text = "Email without attachments", Value = "0"},
new SelectListItem {Text = "Email with attachments", Value = "1"}
};
if (rule != null)
{
rule.AuthorId_Id = userId;
rule.Email = HttpUtility.HtmlEncode(model.Email);
rule.Subject = HttpUtility.HtmlEncode(model.Subject);
rule.PlainContains = HttpUtility.HtmlEncode(model.PlainContains);
//rule.IsUpdatingRule = isUpdatingRule;
rule.Status = Model.Enums.RuleStatusEnums.Active.ToString();
rule.TableId = spreadsheetId;
var saveEmailProperties = model.SaveEmailProperties.Where(w => w.SaveEmailProp != null).ToList();
if (saveEmailProperties.Count == 0)
{
saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty()
{
TableId = tables[0],
Worksheet = "0",
Column = "A",
SaveEmailProp = saveEmail[0].Value,
EmailParsingRulesModelsId = rule.Id
});
}
else if (saveEmailProperties.Count > 0 && saveEmailProperties.First().Id == 0)
{
saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty()
{
TableId = model.SaveEmailProperties.First().TableId,
Worksheet = model.SaveEmailProperties.First().Worksheet,
Column = model.SaveEmailProperties.First().Column,
SaveEmailProp = saveEmail.Find(w => w.Text == model.SaveEmailProperties.First().SaveEmailProp).Value,
EmailParsingRulesModelsId = rule.Id
});
saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty()
{
TableId = tables[0],
Worksheet = "0",
Column = "A",
SaveEmailProp = saveEmail[0].Value,
EmailParsingRulesModelsId = rule.Id
});
}
else
{
foreach (var saveEmailProp in saveEmailProperties)
{
var saveProp = saveEmailPropertiesRepository.Find(w => w.Id == saveEmailProp.Id);
saveProp.TableId = saveEmailProp.TableId;
saveProp.Worksheet = saveEmailProp.Worksheet;
saveProp.Column = saveEmailProp.Column;
saveProp.SaveEmailProp = saveEmail.Find(w => w.Text == saveEmailProp.SaveEmailProp).Value;
}
saveEmailPropertiesRepository.Add(new Model.SaveEmailProperty()
{
TableId = tables[0],
Worksheet = "0",
Column = "A",
SaveEmailProp = saveEmail[0].Value,
EmailParsingRulesModelsId = rule.Id
});
}
saveEmailPropertiesRepository.Commit();
emailParsingRulesRepository.Commit();
return rule.Id;
// return RedirectToAction("AddNewDataRuleForEmails", new { ruleId = rule.Id });
}
return 0;
}
return 0;
}
public ActionResult RedactorAdd(long spreadsheetId)
{
ViewBag.SpreadsheetId = spreadsheetId;
var userId = UserHelper.GetUserByEmail(User.Identity.Name).Id;
var readactorList = spreadsheetRepository.FindAll(t => t.AdministratorId_Id == userId);
List<string> redactorsIds = new List<string>();
var redactorsOfThisTable = new List<string>();
var redactors = new List<RedactorOfTable>();
foreach (var table in readactorList)
{
redactorsIds = (from myObj in redactorRepository.FindAll(w => w.TableId == table.Id)
orderby myObj.UserId_Id
select myObj.UserId_Id).ToList();
foreach (var user in redactorsIds)
{
var userEmail = userRepository.Find(u => u.Id == user).Email;
var rf = new RedactorOfTable() { IsRedactorOfThisTable = false, RedactorEmail = userEmail};
var rt = new RedactorOfTable() { IsRedactorOfThisTable = true, RedactorEmail = userEmail };
if (redactors.Find(r=>r.RedactorEmail == userEmail) == null && table.Id != spreadsheetId)
{
redactors.Add(rf);
}
else if (table.Id == spreadsheetId)
{
redactors.RemoveAll(r => r.RedactorEmail == userEmail);
redactors.Add(rt);
}
}
}
return View("AddRedactorPopupContent", redactors);
}
private static void ParseEmailData(long emailId, long ruleId)
{
// AddLog("ParseEmailData");
IRepository<EmailModels> emailRepository = new Repository<EmailModels>();
var email = emailRepository.Find(w => w.Id == emailId);
if (email != null)
{
int row = -1;
List<int> rows = new List<int>();
IRepository<EmailParsingRulesModels> emailParsingRulesRepository = new Repository<EmailParsingRulesModels>();
var emailParsingRule = emailParsingRulesRepository.Find(w => w.Id == ruleId);
if (emailParsingRule != null)
{
// AddLog("emailParsingRule != null");
List<Models.TablesList> tablesList = new List<Models.TablesList>();
IRepository<Model.SaveEmailProperty> saveEmailPropertiesRepository = new Repository<Model.SaveEmailProperty>();
var allDefaultValues = saveEmailPropertiesRepository.FindAll(i => i.EmailParsingRulesModelsId == emailParsingRule.Id).ToList();
IRepository<Model.TableModels> tablesRepository = new Repository<TableModels>();
IRepository<EmailDataParsingRulesModels> emailDataParsingRulesRepository =
new Repository<EmailDataParsingRulesModels>();
var emailDataParsingRules =
emailDataParsingRulesRepository.FindAll(
w => w.EmailParsingRulesModelsId_Id == emailParsingRule.Id &&
w.Status.Equals(Model.Enums.RuleStatusEnums.Active.ToString())).OrderBy(r => r.Name).ToList();
if (emailDataParsingRules != null && emailDataParsingRules.Count != 0)
{
// AddLog("emailDataParsingRules.Count != 0");
foreach (var dataParsingRule in emailDataParsingRules)
{
if (tablesList.Find(w => w.TableId == dataParsingRule.TableId_Id && w.IsChecked) == null && dataParsingRule.Status != Model.Enums.RuleStatusEnums.Paused.ToString())
{
var name = dataParsingRule.TableId_Id.ToString();
var path = SpreadsheetHelper.GetSpreadsheetPathById((long)dataParsingRule.TableId_Id);
try
{
SpreadsheetExtension.LoadDocument(name, path);
IWorkbook workbook = SpreadsheetExtension.GetCurrentDocument(name);
Worksheet worksheet =
workbook.Worksheets[
emailParsingRule.EmailDataParsingRulesModels.FirstOrDefault().Worksheet];
if (emailParsingRule.IsUpdatingRule)
{
var changing =
emailParsingRule.EmailDataParsingRulesModels.FirstOrDefault(
w =>
w.IsUpdatindIndex &&
w.EmailParsingRulesModelsId_Id == emailParsingRule.Id);
var changingIndex = BeforeParseValueFromEmail(email, changing);
if (path != null)
{
foreach (var c in worksheet.Columns[dataParsingRule.Column].ExistingCells)
{
var value = c.Value.ToString();
if (value.Equals(changingIndex))
{
row = c.RowIndex;
rows.Add(row);
}
}
byte[] size = workbook.SaveDocument(DocumentFormat.Xlsx);
tablesRepository.Find(t => t.Id == dataParsingRule.TableId_Id).Size = size.Length;
tablesRepository.Commit();
workbook.SaveDocument(path);
SpreadsheetExtension.CloseDocument(name);
}
}
else
{
worksheet.Rows.Insert(1);
row = 1;
}
if (row != -1)
{
var author_Id = emailParsingRule.AuthorId_Id;
foreach (var defaultValue in allDefaultValues)
{
if (emailParsingRule.Status != Model.Enums.RuleStatusEnums.Paused.ToString())
{
InsertEmailLink(emailId, defaultValue, row, author_Id, emailParsingRule.CopyRuleFromPreviousOrder);
}
}
foreach (
var dataParsing in
emailParsingRule.EmailDataParsingRulesModels.Where(
w => w.TableId_Id == dataParsingRule.TableId_Id))
{
if (dataParsing.Status != Model.Enums.RuleStatusEnums.Paused.ToString())
{
if (emailParsingRule.IsUpdatingRule)
{
foreach (var rowForUpdate in rows)
{
ParseCellValue(email, dataParsing, rowForUpdate, emailParsingRule.CopyRuleFromPreviousOrder);
}
rows.Clear();
}
else
{
ParseCellValue(email, dataParsing, row, emailParsingRule.CopyRuleFromPreviousOrder);
}
}
}
tablesList.Add(new TablesList
{
IsChecked = true,
TableId = (long)dataParsingRule.TableId_Id
});
}
}
catch
{
AddLog("Table not found");
}
}
}
}
else if (allDefaultValues != null)
{
var tablesInallDefaultValues = allDefaultValues.GroupBy(a => a.TableId).Select(g => g.First());
foreach (var defaultValue in tablesInallDefaultValues)
{
try
{
var name = tablesRepository.Find(w => w.Name == defaultValue.TableId && w.AuthorId_Id == emailParsingRule.AuthorId_Id).Name;
var path = SpreadsheetHelper.GetSpreadsheetPathById(tablesRepository.Find(w => w.Name == defaultValue.TableId && w.AuthorId_Id == emailParsingRule.AuthorId_Id).Id);
SpreadsheetExtension.LoadDocument(name, path);
IWorkbook workbook = SpreadsheetExtension.GetCurrentDocument(name);
Worksheet worksheet =
workbook.Worksheets[
Convert.ToInt32(defaultValue.Worksheet)];
worksheet.Rows.Insert(1);
row = 1;
byte[] size = workbook.SaveDocument(DocumentFormat.Xlsx);
tablesRepository.Find(w => w.Name == defaultValue.TableId && w.AuthorId_Id == emailParsingRule.AuthorId_Id).Size = size.Length;
tablesRepository.Commit();
workbook.SaveDocument(path);
SpreadsheetExtension.CloseDocument(name);
}
catch (Exception ex)
{
AddLog(ex.Message);
}
}
foreach (var defaultValue in allDefaultValues)
{
InsertEmailLink(emailId, defaultValue, row, emailParsingRule.AuthorId_Id, emailParsingRule.CopyRuleFromPreviousOrder);
}
}
}
}
}
/// <summary>
/// Metoda tworzenia i uczenia sieci neuronowych.
/// </summary>
/// <param name="maxMaxEpochs">Maksymalna liczba epok.</param>
/// <param name="minMaxEpochs">Minimalna liczba epok.</param>
/// <param name="maxEpochsMultiplierStep">Krok mnożący liczby epok dla kolejnego rodzaju sieci. </param>
/// <param name="minHiddenLayersMultiplier">Minimalny mnożnik dla warstw ukrytych.</param>
/// <param name="maxHiddenLayersMultiplier">Maksymalny mnożnik dla wartw ukrytych.</param>
/// <param name="hiddenLayersMultiplierStep">Krok dodający dla mnożnika warstw ukrytych.</param>
/// <param name="methodProgressPart">Wartość (0 - 100) określająca jaka część postępu inicjalizacji należy do tej metody.</param>
private void CreateNetworks(
string path,
string trainDataFolder,
string testDataFolder,
int maxMaxEpochs,
int minMaxEpochs,
double maxEpochsMultiplierStep,
double minHiddenLayersMultiplier,
double maxHiddenLayersMultiplier,
double hiddenLayersMultiplierStep,
double methodProgressPart,
double desiredMSE
)
{
double methodStartProgress = this.InitializationProgress;
List<TrainingDataFileParameters> fileList = DataProvider.Instance.GetTrainingDataFilesParameters(trainDataFolder);
List<NeuralNetworkParameters> parameters = new List<NeuralNetworkParameters>();
foreach (TrainingDataFileParameters file in fileList)
{
for (double i = minMaxEpochs; i <= maxMaxEpochs; i *= maxEpochsMultiplierStep)
{
for (double j = minHiddenLayersMultiplier; j <= maxHiddenLayersMultiplier; j += hiddenLayersMultiplierStep)
{
parameters.Add(new NeuralNetworkParameters()
{
fileParameters = file,
hiddenLayersMultiplier = j,
maxEpochs = (uint)i
});
}
}
}
int numberOfNetworksToCreate = parameters.Count;
int numberOfCreatedNetworks = 0;
NetworkMSE lowestNetworkMSE = null;
try
{
StringReader reader = new StringReader(File.ReadAllText(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path + "LowestMSENetwork.xml"));
XmlSerializer serializer = new XmlSerializer(typeof(NetworkMSE));
lowestNetworkMSE = (NetworkMSE)serializer.Deserialize(reader);
}
catch
{
lowestNetworkMSE = null;
}
//small hack
if (lowestNetworkMSE != null && lowestNetworkMSE.MSE < desiredMSE)
{
this.InitializationProgress = 100;
return;
}
string[] testFiles = Directory.GetFiles(testDataFolder);
List<TrainingData> testDataList = new List<TrainingData>();
foreach (string testFile in testFiles)
{
TrainingData td = new TrainingData();
//if (td.ReadTrainFromFile(testDataFolder + "\\" + Path.GetFileName(testFile)))
if (td.ReadTrainFromFile(testFile))
{
testDataList.Add(td);
}
}
List<TrainingData> trainingDataList = new List<TrainingData>();
foreach (TrainingDataFileParameters file in fileList)
{
TrainingData td = new TrainingData();
if (td.ReadTrainFromFile(trainDataFolder.Split('\\').Last() + "\\" + file.FileName))
{
trainingDataList.Add(td);
}
}
string initStatus = this.InitializationStatus;
Directory.CreateDirectory(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path);
List<Task> taskList = new List<Task>();
NeuralNet.CallbackType[] callbacksArray = new NeuralNet.CallbackType[this.MaxComputingThreads];
Semaphore threadProgressSemaphore = new Semaphore(1, 1);
Semaphore allSem = new Semaphore(1, 1);
TrainingData[] threadDataVars = new TrainingData[this.MaxComputingThreads];
for (int i = 1; i <= this.MaxComputingThreads; i++)
{
int taskNumber = i;
threadDataVars[i - 1] = new TrainingData();
Task t = new Task((Action)(
() =>
{
while (true)
{
allSem.WaitOne();
if (parameters.Count == 0)
{
this.InitializationStatus = initStatus + " " + numberOfNetworksToCreate.ToString() + " / " + numberOfNetworksToCreate.ToString();
this.InitializationProgress = 100;
break;
}
else
{
NeuralNetworkParameters usedParameters = parameters.First();
parameters.RemoveAt(0);
if (!File.Exists(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path + usedParameters.FileName))
{
NeuralNet net = new NeuralNet();
List<uint> layers = new List<uint>();
layers.Add((uint)((usedParameters.fileParameters.NumberOfPeriods * 2) - 1)); // inputs
layers.Add((uint)(layers[0] * usedParameters.hiddenLayersMultiplier)); // hidden
layers.Add(3); // output
net.CreateStandardArray(layers.ToArray());
net.SetLearningRate((float)0.7);
net.SetActivationFunctionHidden(ActivationFunction.SigmoidSymmetric);
net.SetActivationFunctionOutput(ActivationFunction.SigmoidSymmetric);
net.Callback += callbacksArray[taskNumber - 1];
threadDataVars[taskNumber - 1] = trainingDataList.Find((e) => ((e.NumInputTrainData == layers[0]) && (e.Input.Length == usedParameters.fileParameters.NumberOfPatterns)));
allSem.Release();
net.TrainOnData(threadDataVars[taskNumber - 1],
usedParameters.maxEpochs, // max iterations
0,// iterations between report
0 //desired error
);
allSem.WaitOne();
net.TestData(testDataList.Find((e) => e.NumInputTrainData == layers[0]));
double mse = net.GetMSE();
if (lowestNetworkMSE == null || lowestNetworkMSE.MSE > mse)
{
lowestNetworkMSE = new NetworkMSE()
{
MSE = mse,
NetworkFileName = usedParameters.FileName
};
StringWriter writer = new StringWriter();
XmlSerializer serializer = new XmlSerializer(typeof(NetworkMSE));
serializer.Serialize(writer, lowestNetworkMSE);
File.WriteAllText(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + path + "LowestMSENetwork.xml", writer.ToString());
}
net.Save(path + usedParameters.FileName);
}
this.InitializationStatus = initStatus + " " + numberOfCreatedNetworks.ToString() + " / " + numberOfNetworksToCreate.ToString();
numberOfCreatedNetworks++;
this.InitializationProgress = (numberOfCreatedNetworks * methodProgressPart / numberOfNetworksToCreate) + methodStartProgress;
}
allSem.Release();
}
}
), TaskCreationOptions.LongRunning
);
taskList.Add(t);
}
foreach (Task t in taskList)
{
t.Start();
}
}
private static void CheckTagSetTest(List<TagProcessResult> tagList, TagProcessResult tagToCheck)
{
var tagWordSynonyms = GetSynonymsForTagWords(tagToCheck);
var result = (from x in tagList
where x.TagId != tagToCheck.TagId
where !x.DuplicateTagId.HasValue
where !x.RelatedTagId.HasValue
let foundDuplicateTagId = !tagToCheck.IsDuplicateReference && x.TagValue.ToLower() == tagToCheck.TagValue.ToLower() ? x.TagId : (int?)null
let foundPartialDuplicateTagId = !tagToCheck.IsRelatedReference && x.WordProcessResultList.Any(a => tagToCheck.WordProcessResultList.Select(wpr => wpr.Word.ToLower()).Contains(a.Word.ToLower())) ? x.TagId : (int?)null
let foundSynonymTagId = !tagToCheck.IsRelatedReference && x.WordProcessResultList.Any(w => tagWordSynonyms.Contains(w.Word.ToLower())) ? x.TagId : (int?)null
select new
{
DuplicateTagId = foundDuplicateTagId,
RelatedTagId = foundPartialDuplicateTagId ?? foundSynonymTagId
}).FirstOrDefault();
if (result != null)
{
if (result.DuplicateTagId.HasValue)
{
tagToCheck.DuplicateTagId = result.DuplicateTagId;
tagList.Find(t => t.TagId == result.DuplicateTagId).IsDuplicateReference = true;
}
if (result.RelatedTagId.HasValue)
{
tagToCheck.RelatedTagId = result.RelatedTagId;
tagList.Find(t => t.TagId == result.RelatedTagId).IsRelatedReference = true;
}
}
}
