private static void refindPatternds()
{
bool logAll = false;
string loadFilename = ResultFile;
string splitter = "\t\t";
NodeDeduplicator deduplicator = new InverseNodeDeduplicator();
Tree tree = Tree.GetRootedTree();
tree.Print = Print;
tree.Root.Print = Print;
tree.LogEnabled = logAll;
FileHelper fh = new FileHelper(splitter);
var resultLines = fh.LoadFromFile(loadFilename);
int count = 0;
foreach (var res in resultLines)
{
var sent = res.Sentence.Replace("^ ", "").Trim();
Node current = tree.Root;
string[] words = sent.Split(' ');
foreach (string word in words)
{
current = current.AddOrGet(word);
current.IsTemplateValue = Node.IsTemplateNode(current.Value);
}
count++;
if (count % 10 == 0)
{
deduplicator.Deduplicate(tree.Root);
}
current.HintCount = res.HintCount;
}
if (logAll)
{
tree.BeautyLog();
}
deduplicator.Deduplicate(tree.Root);
tree.BeautyLog();
string toFile = "new_file_02.txt";
if (File.Exists(toFile))
{
File.Delete(toFile);
}
fh.SaveToFile(toFile, tree);
}
private static void findPatterns()
{
//File.Delete(FileName);
//CreatePatterns(15000);
NodeDeduplicator deduplicator = new InverseNodeDeduplicator();
Tree tree = Tree.GetRootedTree(); // new Tree();
tree.Print = Print;
int count = 0;
using (var fileStream = File.OpenRead(FileName))
{
using (StreamReader reader = new StreamReader(fileStream, Encoding.UTF8))
{
string newLine = string.Empty;
while ((newLine = reader.ReadLine()) != null)
{
if (!string.IsNullOrWhiteSpace(newLine))
{
newLine = newLine + " " + Node.LineEnd;
string[] words = newLine.Split(' ');
Node current = tree.Root;
foreach (string word in words)
{
current = current.AddOrGet(word);
current.Print = Print;
}
count++;
}
}
}
}
Console.WriteLine();
Stopwatch cwDeduplicate = new Stopwatch();
cwDeduplicate.Start();
deduplicator.Deduplicate(tree.Root);
cwDeduplicate.Stop();
tree.BeautyLog();
Console.WriteLine($"{count}. Duration: {cwDeduplicate.ElapsedMilliseconds / 1000.0}");
//FileHelper fileHelper = new FileHelper();
//fileHelper.SaveToFile(ResultFile, tree);
}
