public bool HandleFilterFault(
EventBean theEvent,
long version)
{
return Realization.HandleFilterFault(theEvent, version);
}
public ICollection<int> GetAgentInstanceIds(ContextPartitionSelector selector)
{
return Realization.GetAgentInstanceIds(selector);
}
public void Add(Realization realization)
{
Context.AddRealization(realization);
}
public void OnPostParallel()
{
var files = new DirectoryInfo(Path.Combine(_environment.ContentRootPath, "database", "parallelizedDocuments")).GetFiles();
var documents = new List <ParallelDocument>();
if (files.Length > 0)
{
for (int f = 0; f < files.Length; f++)
{
using (StreamReader r = new StreamReader(new FileStream(files[f].FullName, FileMode.Open, FileAccess.Read)))
{
documents.Add(JsonConvert.DeserializeObject <ParallelDocument>(r.ReadToEnd()));
}
}
List <string> lemmata = new List <string>();
chosenTexts.ForEach(x =>
{
var docName = x.Split(": ")[0];
var textID = x.Split(": ")[1].Split(" (")[1].Split(')')[0];
var currentLemmata = documents
.Where(d => d.name == docName)
.SelectMany(d => d.parallelTokens)
.SelectMany(t => t.ToList())
.SelectMany(rg => rg.ToList())
.Where(r => r.clauseID == textID)
.Select(r => r.realizationFields is null ? r.lexemeTwo : r.realizationFields.Any(f => f.ContainsKey("Lemma")) ? r.realizationFields.Where(t => t.ContainsKey("Lemma")).SelectMany(t => t["Lemma"]).Select(v => v.name).FirstOrDefault() : r.lexemeTwo)
.Distinct()
.ToList();
currentLemmata.ForEach(l => lemmata.Add(l));
});
lemmata = lemmata.Distinct().ToList();
var finalDictionary = new List <ParallelDictionaryUnit>();
lemmata.ForEach(lemma => finalDictionary.Add(new ParallelDictionaryUnit(lemma, documents
.SelectMany(d => d.parallelTokens)
.Where(t => t.Any(rg => rg.Any(r => r.lexemeTwo == lemma || (r.realizationFields != null ? (r.realizationFields.Any(f => f.ContainsKey("Lemma")) ? r.realizationFields.Where(f => f.ContainsKey("Lemma")).SelectMany(kvp => kvp["Lemma"]).Any(v => v.name == lemma) : false) : false))))
.ToList())));
finalDictionary = finalDictionary.OrderBy(unit => unit.lemma).ToList();
if (parallelDictsToFiles == true)
{
string path = Path.Combine(_environment.ContentRootPath, "database");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
path = Path.Combine(path, "parallelDictionary");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
using (StreamWriter w = new StreamWriter(new FileStream(Path.Combine(path, Regex.Replace(string.Join("_", chosenTexts), @"[*""><:/\\|?\.,]", "") + "_parallelDict.json"), FileMode.CreateNew, FileAccess.Write)))
{
for (int u = 0; u < finalDictionary.Count; u++)
{
w.Write(JsonConvert.SerializeObject(finalDictionary[u], Formatting.Indented));
}
}
}
foreach (var unit in finalDictionary)
{
string output = "<b>" + unit.lemma + "</b>";
output += " (" + unit.realizations.Count.ToString() + ")<br />";
output += "<ul>";
foreach (var token in unit.realizations)
{
RealizationGroup coreGroup = null;
Realization coreRealization = null;
foreach (var rg in token)
{
foreach (var realization in rg)
{
if (realization.realizationFields != null)
{
if (realization.realizationFields.Any(f => f.ContainsKey("Lemma")))
{
if (realization.realizationFields.Where(f => f.ContainsKey("Lemma")).SelectMany(kvp => kvp["Lemma"]).Any(v => v.name == unit.lemma))
{
coreGroup = rg;
coreRealization = realization;
break;
}
}
}
if (realization.lexemeTwo == unit.lemma)
{
coreGroup = rg;
coreRealization = realization;
break;
}
}
}
output += "<li>";
var currentDocument = documents.Where(d => d.id == coreRealization.documentID).Single();
output += coreRealization.lexemeTwo + " (" + String.Join(' ', coreGroup.Select(r => r.lexemeTwo)) + ", " + currentDocument.name + " - " + currentDocument.parallelClauses[Convert.ToInt32(coreRealization.textID), Convert.ToInt32(coreRealization.clauseID)].textName + "); <span class=\"clause\" id=\"clauseExtractionButton\" clause=\"clause: " + currentDocument.parallelClauses[Convert.ToInt32(coreRealization.textID), Convert.ToInt32(coreRealization.clauseID)].clause.clauseText + "\">see text segment</span><br />";
output += ((coreRealization.realizationFields != null && coreRealization.realizationFields.Count > 0) ? String.Join(" - ", coreRealization.realizationFields.SelectMany(t => t).Where(a => a.Key != "Lemma").Select(a => String.Join("", a.Value.SelectMany(v => v.name).ToList())).Distinct().ToList()) + "<br />" : "");
output += coreRealization.letters.Any(l => !(l.graphemeFields is null) && l.graphemeFields.Count > 0) ? ("<ul class=\"graphemeFeatures\">" + String.Join("", coreRealization.letters.Select(l => ((l.graphemeFields != null && l.graphemeFields.Count > 0) ? "<li>" + l.grapheme + ": " + String.Join(" - ", l.graphemeFields.SelectMany(t => t).Select(a => a.Key + "(" + String.Join("", a.Value.SelectMany(v => v.name).ToList()) + ")").Distinct().ToList()) + "<br />" : ""))) + "</ul>") : "";
output += "<ul class=\"parallels\"><li>" + String.Join(";<br/><li>", token.GetParallels(coreGroup).Select(rg => String.Join(' ', rg.Select(r => r.lexemeTwo).ToList()) + "(" + currentDocument.name + " - " + currentDocument.parallelClauses[Convert.ToInt32(rg[0].textID), Convert.ToInt32(rg[0].clauseID)].textName + ")" + "; <span class=\"clause\" id=\"clauseExtractionButton\" clause=\"text segment:\r" + String.Join('\r', rg.Select(r => r.clauseID).Distinct().ToList().OrderBy(id => Convert.ToInt32(id)).Select(id => currentDocument.parallelClauses[Convert.ToInt32(rg.textID), Convert.ToInt32(id)].clause.clauseText)) + "\">see text segment</span>")) + "</ul></li>";
}
output += "</ul><br /><br />";
convertedTexts.Add(output);
}
}
}
public void Edit(Realization realization)
{
Context.EditRealization(realization);
}
private void Button5_Click(object sender, EventArgs e)
{
if (text_is_ready)
{
bool res = false;
do
{
MessageBox.Show("Укажите папку, в которой размещается корпус", "Сообщение программы");
var choice = folderBrowserDialog1.ShowDialog();
if (choice == DialogResult.OK)
{
string folderPath = folderBrowserDialog1.SelectedPath;
Document serializedDoc = new Document(new DirectoryInfo(folderPath).GetFiles().Length.ToString(), textBox2.Text,
new DirectoryInfo(folderPath).GetFiles().Length.ToString() + "_" + textBox2.Text, "_");
var units = Regex.Split(richTextBox2.Text, "\n");
progressBar1.Minimum = 0;
progressBar1.Value = 0;
progressBar1.Step = 1;
progressBar1.Maximum = units.Length;
Text currentText = new Text(serializedDoc, "0", serializedDoc.documentName);
for (int i = 0; i < units.Length; i++)
{
try
{
Clause currentClause = new Clause(currentText, i.ToString(), units[i]);
var lexemes = currentClause.clauseText.Split(' ');
for (int j = 0; j < lexemes.Length; j++)
{
Realization currentRealization;
if (lexemes[j].StartsWith("!"))
{
if (Regex.Match(lexemes[j], @"[!]{1}[Ц|ц|W|w|Ѱ|ѱ|Х|х|Ф|ф|Ѹ|ѹ|У|у|Т|т|С|с|Р|р]{0,1}[Ч|ч|П|п|О|о|Ѯ|ѯ|Н|н|М|м|Л|л|К|к|I|i|И|и]{0,1}[Ѳ|ѳ|И|и|З|з|Е|е|Д|д|Г|г|В|в|Б|б|А|а]{0,1}$").Success)
{
currentRealization = new Realization(currentClause, j.ToString(), lexemes[j], "~" + Regex.Replace(lexemes[j], @"[\[\]\?\-\'\`\^\~\(\)\!\\]", "") + "~");
}
else
{
currentRealization = new Realization(currentClause, j.ToString(), lexemes[j], "~" + Regex.Replace(lexemes[j], @"[\[\]\?\-\'\`\^\~\(\)\!\\]", ""));
}
}
else
{
currentRealization = new Realization(currentClause, j.ToString(), lexemes[j], Regex.Replace(lexemes[j], @"[\[\]\?\-\'\`\^\~\(\)\!\\]", ""));
}
if (currentRealization.realizationFields == null)
{
currentRealization.realizationFields = new List <Dictionary <string, List <IValue> > >();
}
currentRealization.realizationFields.Add(new Dictionary <string, List <IValue> >());
List <Grapheme> letters(Realization word)
{
List <Grapheme> graphemes = new List <Grapheme>();
for (int k = 0; k < word.lexemeOne.Length; k++)
{
graphemes.Add(new Grapheme(currentRealization, k.ToString(), word.lexemeOne[k].ToString()));
}
return(graphemes);
}
currentRealization.letters = letters(currentRealization);
if (currentClause.realizations == null)
{
currentClause.realizations = new List <Realization>();
}
currentClause.realizations.Add(currentRealization);
}
if (currentText.clauses == null)
{
currentText.clauses = new List <Clause>();
}
currentText.clauses.Add(currentClause);
progressBar1.PerformStep();
}
catch (Exception r)
{
if (r is IOException)
{
Console.WriteLine(r.Message);
}
continue;
}
}
serializedDoc.texts.Add(currentText);
using (StreamWriter sw = new StreamWriter(Path.Combine(folderPath, serializedDoc.documentName + ".json")))
{
sw.Write(serializedDoc.Jsonize());
}
DialogResult dialogResult = MessageBox.Show("Требуются ли частеречная разметка и лемматизация?", "Сообщение программы", MessageBoxButtons.YesNo);
if (dialogResult == DialogResult.Yes)
{
SimpleValue period = new SimpleValue(listBox2.Text);
List <IValue> periodFieldValues = new List <IValue>();
periodFieldValues.Add(period);
SimpleValue tagged = new SimpleValue("Automatically_tagged");
List <IValue> taggedFieldValues = new List <IValue>();
taggedFieldValues.Add(tagged);
Dictionary <string, List <IValue> > fields = new Dictionary <string, List <IValue> >();
fields.Add("Period", periodFieldValues);
fields.Add("Tagged", taggedFieldValues);
serializedDoc.documentMetaData.Add(fields);
using (StreamWriter sw = new StreamWriter(Path.Combine(folderPath, serializedDoc.documentName + ".json")))
{
sw.Write(serializedDoc.Jsonize());
}
res = false;
string model_path = "";
do
{
MessageBox.Show("Укажите папку с моделью, обученной для частеречной разметки и файлом Python", "Сообщение системы", MessageBoxButtons.OK, MessageBoxIcon.Information);
FolderBrowserDialog folderBrowserDialog = new FolderBrowserDialog();
DialogResult result = folderBrowserDialog.ShowDialog();
if (result == DialogResult.OK)
{
res = true;
model_path = folderBrowserDialog.SelectedPath;
}
}while (!res);
string command = "python " + Path.Combine(model_path, "main.py") + " --data " + Path.Combine(folderPath, serializedDoc.documentName + ".json") + " --modus prediction --folder " + model_path;
var processInfo = new ProcessStartInfo("cmd.exe", "/c " + command);
processInfo.CreateNoWindow = true;
processInfo.UseShellExecute = false;
processInfo.RedirectStandardError = true;
processInfo.RedirectStandardOutput = true;
var process = Process.Start(processInfo);
process.WaitForExit();
process.Close();
res = false;
model_path = "";
do
{
MessageBox.Show("Укажите папку с моделью, обученной для лемматизации и файлом Python", "Сообщение системы", MessageBoxButtons.OK, MessageBoxIcon.Information);
FolderBrowserDialog folderBrowserDialog = new FolderBrowserDialog();
DialogResult result = folderBrowserDialog.ShowDialog();
if (result == DialogResult.OK)
{
res = true;
model_path = folderBrowserDialog.SelectedPath;
}
}while (!res);
command = "python " + Path.Combine(model_path, "main.py") + " --data " + Path.Combine(folderPath, serializedDoc.documentName + ".json") + " --modus prediction --folder " + model_path;
processInfo = new ProcessStartInfo("cmd.exe", "/c " + command);
processInfo.CreateNoWindow = true;
processInfo.UseShellExecute = false;
processInfo.RedirectStandardError = true;
processInfo.RedirectStandardOutput = true;
process = Process.Start(processInfo);
process.WaitForExit();
process.Close();
MessageBox.Show("Частеречная разметка и лемматизация проведены для текста", "Сообщение системы");
res = true;
}
else if (dialogResult == DialogResult.No)
{
SimpleValue period = new SimpleValue(listBox2.Text);
List <IValue> periodFieldValues = new List <IValue>();
periodFieldValues.Add(period);
SimpleValue tagged = new SimpleValue("Not_tagged");
List <IValue> taggedFieldValues = new List <IValue>();
taggedFieldValues.Add(tagged);
Dictionary <string, List <IValue> > fields = new Dictionary <string, List <IValue> >();
fields.Add("Period", periodFieldValues);
fields.Add("Tagged", taggedFieldValues);
serializedDoc.documentMetaData.Add(fields);
using (StreamWriter sw = new StreamWriter(Path.Combine(folderPath, serializedDoc.documentName + ".json")))
{
sw.Write(serializedDoc.Jsonize());
}
MessageBox.Show("Частеречная разметка и лемматизация не проведены для текста", "Сообщение системы");
res = true;
}
res = true;
}
res = true;
MessageBox.Show("Текст занесён в базу данных", "Сообщение системы", MessageBoxButtons.OK, MessageBoxIcon.Information);
}while (!res);
}
else
{
MessageBox.Show("Прежде чем заносить текст в базу данных, нужно разбить его по знакам пунктуации", "Сообщение программы", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
static void Main()
{
Generalization algorithm = new Realization();
algorithm.FindSolution();
}
public IActionResult OnPost()
{
try
{
MatchCollection units = Regex.Matches(currentText, @"\{(\d*\|){2}.*?\}");
string document_edited = units[0].Value.Split('|')[0].Replace("{", "");
string text_edited = units[0].Value.Split('|')[1];
var files = new DirectoryInfo(Path.Combine(_environment.ContentRootPath, "database", "documents")).GetFiles();
Document editedDocument = new Document();
foreach (var file in files)
{
using (StreamReader r = new StreamReader(file.FullName))
{
Document analyzedDocument = JsonConvert.DeserializeObject <Document>(r.ReadToEnd());
if (analyzedDocument.documentID == document_edited)
{
editedDocument = analyzedDocument;
}
}
}
foreach (Clause clause in editedDocument.texts.Where(t => t.textID == text_edited).ToList()[0].clauses)
{
List <string> forEditing = units.Select(c => c.Value).Where(v => Regex.Replace(v.Split('|')[2], @"(\s.*|\})", "") == clause.clauseID).ToList();
for (int i = 0; i < forEditing.Count; i++)
{
if (Regex.IsMatch(forEditing[i], @"\{(\d*\|){2}(\d*)(\s.*?\}|\})"))
{
try
{
List <string> addedFields = forEditing[i].Split(" => ")[1].Replace("{", "").Split(";<br />").ToList();
List <Dictionary <string, List <Value> > > newFields = new List <Dictionary <string, List <Value> > >();
Dictionary <string, List <Value> > addedField = new Dictionary <string, List <Value> >();
for (int j = 0; j < addedFields.Count; j++)
{
if (addedFields[j] != "")
{
List <string> currentFields = addedFields[j].Replace(" ;}", "").Split(";").ToList();
for (int f = 0; f < currentFields.Count; f++)
{
if (currentFields[f] != "}")
{
string field = currentFields[f].Split(':')[0];
string values = currentFields[f].Split(':')[1];
List <Value> addedValues = new List <Value>();
if (values.Contains(','))
{
List <string> splitValues = values.Split(",").ToList();
for (int k = 0; k < splitValues.Count; k++)
{
addedValues.Add(new Value(splitValues[k].Trim()));
}
}
else
{
addedValues.Add(new Value(values.Trim()));
}
addedField[field] = addedValues;
}
}
}
}
newFields.Add(addedField);
clause.clauseFields = newFields;
}
catch (IndexOutOfRangeException)
{
try
{
if (clause.clauseFields.Count < 1)
{
continue;
}
else
{
clause.clauseFields.Clear();
}
}
catch (NullReferenceException)
{
continue;
}
}
}
else if (Regex.IsMatch(forEditing[i], @"\{(\d*\|){4}(\d*)(\s.*?\}|\})"))
{
string realizationID = Regex.Replace(forEditing[i].Split('|')[3], @"(\s.*|\})", "");
string graphemeID = Regex.Replace(forEditing[i].Split('|')[4], @"(\s.*|\})", "");
Grapheme grapheme = clause.realizations.Where(r => r.realizationID == realizationID).Select(r => r.letters).ToList()[0].Where(g => g.graphemeID == graphemeID).ToList()[0];
try
{
List <string> addedFields = forEditing[i].Split(" => ")[1].Replace("{", "").Split(";<br />").ToList();
List <Dictionary <string, List <Value> > > newFields = new List <Dictionary <string, List <Value> > >();
Dictionary <string, List <Value> > addedField = new Dictionary <string, List <Value> >();
for (int j = 0; j < addedFields.Count; j++)
{
List <string> currentFields = addedFields[j].Replace(" ;}", "").Split(";").ToList();
for (int f = 0; f < currentFields.Count; f++)
{
if (currentFields[f] != "}")
{
string field = currentFields[f].Split(':')[0];
string values = currentFields[f].Split(':')[1];
List <Value> addedValues = new List <Value>();
if (values.Contains(','))
{
List <string> splitValues = values.Split(",").ToList();
for (int k = 0; k < splitValues.Count; k++)
{
addedValues.Add(new Value(splitValues[k].Trim()));
}
}
else
{
addedValues.Add(new Value(values.Trim()));
}
addedField[field] = addedValues;
}
}
}
newFields.Add(addedField);
grapheme.graphemeFields = newFields;
}
catch (IndexOutOfRangeException)
{
try
{
if (grapheme.graphemeFields.Count < 1)
{
continue;
}
else
{
grapheme.graphemeFields.Clear();
}
}
catch (NullReferenceException)
{
continue;
}
}
}
else if (Regex.IsMatch(forEditing[i], @"\{(\d*\|){3}(\d*)(\s.*?\}|\})"))
{
Realization realization = clause.realizations.Where(r => r.realizationID == Regex.Replace(forEditing[i].Split('|')[3], @"(\s.*|\})", "")).ToList()[0];
try
{
List <string> addedFields = forEditing[i].Split(" => ")[1].Split("***").ToList();
List <Dictionary <string, List <Value> > > newFields = new List <Dictionary <string, List <Value> > >();
for (int j = 0; j < addedFields.Count; j++)
{
Dictionary <string, List <Value> > addedTagging = new Dictionary <string, List <Value> >();
if (addedFields[j] != "")
{
List <string> tagging = addedFields[j].Split(";<br />").ToList();
for (int f = 0; f < tagging.Count; f++)
{
if (tagging[f] != "}")
{
List <string> fieldsToAdd = tagging[f].Split(" ;").ToList();
for (int n = 0; n < fieldsToAdd.Count; n++)
{
if (fieldsToAdd[n] != "" && fieldsToAdd[n] != ";")
{
List <string> splitFields = fieldsToAdd[n].Split(';').ToList();
for (int s = 0; s < splitFields.Count; s++)
{
if (splitFields[s] != ";" && splitFields[s] != "")
{
string field = splitFields[s].Split(':')[0];
string values = splitFields[s].Split(':')[1].Replace(";", "");
List <Value> addedValues = new List <Value>();
if (values.Contains(','))
{
List <string> splitValues = values.Split(",").ToList();
for (int k = 0; k < splitValues.Count; k++)
{
addedValues.Add(new Value(splitValues[k].Trim()));
}
}
else
{
addedValues.Add(new Value(values.Trim()));
}
addedTagging[field] = addedValues;
}
}
}
}
}
}
}
newFields.Add(addedTagging);
}
realization.realizationFields = newFields;
realization.realizationFields = realization.realizationFields.Where(d => d.Count != 0).ToList();
}
catch (IndexOutOfRangeException)
{
try
{
if (realization.realizationFields.Count < 1)
{
continue;
}
else
{
realization.realizationFields.Clear();
}
}
catch
{
continue;
}
}
}
}
}
string documentInJSON = editedDocument.Jsonize();
var documentDBFile = Path.Combine(_environment.ContentRootPath, "database", "documents", editedDocument.documentID + "_" + editedDocument.documentName + ".json");
FileStream fs = new FileStream(documentDBFile, FileMode.Create);
using (StreamWriter w = new StreamWriter(fs))
{
w.Write(documentInJSON);
}
}
catch
{
}
docList = getDocs();
fieldsList = getFields();
return(RedirectToPage());
}
static void Main()
{
Realization rl = new Realization();
rl.Menu_();
}
