static LogParser()
{
var parsers = new List <LogSectionParser>(LogSections.Count);
foreach (var sectionDescription in LogSections)
{
var parser = new LogSectionParser
{
OnLineCheckAsync = sectionDescription.OnNewLineAsync ?? ((l, s) => Task.CompletedTask),
OnSectionEnd = sectionDescription.OnSectionEnd,
EndTrigger = sectionDescription.EndTrigger.ToLatin8BitEncoding(),
};
// the idea here is to construct Aho-Corasick parser that will look for any data marker and run the associated regex to extract the data into state
if (sectionDescription.Extractors?.Count > 0)
{
var act = new AhoCorasickDoubleArrayTrie <Action <string, LogParseState> >(sectionDescription.Extractors.Select(extractorPair =>
new SectionAction(
extractorPair.Key.ToLatin8BitEncoding(),
(buffer, state) => OnExtractorHit(buffer, extractorPair.Value, state)
)
), true);
parser.OnExtract = (line, buffer, state) => { act.ParseText(line, h => { h.Value(buffer, state); }); };
}
parsers.Add(parser);
}
SectionParsers = parsers.AsReadOnly();
}
internal static (bool needToChill, bool needToThank) NeedToSilence(DiscordMessage msg)
{
if (string.IsNullOrEmpty(msg?.Content))
{
return(false, false);
}
var needToChill = false;
var needToThank = false;
var msgContent = msg.Content.ToLowerInvariant();
ChillCheck.ParseText(msgContent, h =>
{
if (h.Value)
{
needToChill = true;
}
else
{
needToThank = true;
}
});
var mentionsBot = msgContent.Contains("bot") || (msg.MentionedUsers?.Any(u => { try { return(u.IsCurrent); } catch { return(false); } }) ?? false);
return(needToChill && mentionsBot, needToThank&& mentionsBot);
}
public static Task <string> FindTriggerAsync(string str)
{
string result = null;
matcher?.ParseText(str, h =>
{
result = h.Value;
return(false);
});
return(Task.FromResult(result));
}
public void GetSkillSet(Resume resume, string line)
{
var collectedIndexes = new HashSet <int>();
__skillSetMatcher.ParseText(line, hit => { collectedIndexes.Add(int.Parse(hit.Value)); return(true); });
foreach (var index in collectedIndexes)
{
var element = SkillSet.ElementAtOrDefault(index);
if (element != null)
{
resume.Skills.Add(element.Trim());
}
}
}
public void TestAhoCorasickDoubleArrayTrieForSingleLine(string line)
{
var acdat = new AhoCorasickDoubleArrayTrie <string>();
var pairs = SkillSetMapper.SkillSet.Select((k, i) => new KeyValuePair <string, string>(k, i.ToString()));
acdat.Build(pairs, true);
var collectedValues = new List <string>();
acdat.ParseText(line, hit => { collectedValues.Add(hit.Value); return(true); });
Assert.IsNotEmpty(collectedValues);
var collectedValuesresult = collectedValues.Where(i => SkillSetMapper.SkillSet.ElementAtOrDefault(int.Parse(i)) == null);
Assert.IsEmpty(collectedValuesresult);
var keyWord = SkillSetMapper.SkillSet[int.Parse(collectedValues.FirstOrDefault())].Trim();
Assert.True(keyWord == "MATLAB");
}
internal static bool NeedToSilence(DiscordMessage msg)
{
if (string.IsNullOrEmpty(msg.Content))
{
return(false);
}
var needToChill = false;
ChillCheck.ParseText(msg.Content, h =>
{
needToChill = true;
return(false);
});
if (!needToChill)
{
return(false);
}
return(msg.Content.Contains("bot") || (msg.MentionedUsers?.Any(u => u.IsCurrent) ?? false));
}
// In order this test to run, you need to create a Resumes folder in test execution directory and put some test resumés.
// Only technical skill matching is set up, to test with technical profiles.
public void TestAhoCorasickDoubleArrayTrieForManyResumes()
{
var processor = new ResumeProcessor(new JsonOutputFormatter());
var filePaths = Directory.GetFiles("Resumes").Select(Path.GetFullPath);
var acdat = new AhoCorasickDoubleArrayTrie <string>();
var pairs = SkillSetMapper.SkillSet.Select((k, i) => new KeyValuePair <string, string>(k, i.ToString()));
acdat.Build(pairs, true);
foreach (var filePath in filePaths)
{
var fileName = Path.GetFileName(filePath);
var rawInput = processor._inputReaders.ReadIntoList(filePath);
var collectedValues = new List <string>();
foreach (var line in rawInput)
{
acdat.ParseText(line, hit => { collectedValues.Add(hit.Value); return(true); });
}
Assert.IsNotEmpty(collectedValues, $"No match found in file: {filePath}");
}
}
private static void AhoCorasickDoubleArrayTrieSearch(List <string> list, string txt)
{
var keywords = new Dictionary <string, string>();
for (int i = 0; i < list.Count; i++)
{
keywords[list[i]] = list[i];
}
var matcher = new AhoCorasickDoubleArrayTrie <string>(keywords);
var fs = File.OpenWrite("AhoCorasickDoubleArrayTrie.dat");
matcher.Save(fs, true);
fs.Close();
Stopwatch watch = new Stopwatch();
watch.Start();
for (int i = 0; i < 100000; i++)
{
matcher.ParseText(txt);
}
watch.Stop();
Console.WriteLine(" AhoCorasickDoubleArrayTrie: " + watch.ElapsedMilliseconds.ToString("N0") + "ms");
}
private static bool EntitiesReader(string temp_txt_Location, string length, string dllPath, Dictionary <string, int> entityMissDic1, Dictionary <string, int> entityMissDic2, bool flag3)
{
StreamReader sr = new StreamReader(temp_txt_Location);
var line = sr.ReadToEnd();
int currentCount = 0;
var keywords = new Dictionary <string, int>()
{
{ ".class", 0 },
{ ".method", 0 },
{ "interface", 0 },
{ ".property", 0 },
{ ".assembly", 0 },
};
var matcher = new AhoCorasickDoubleArrayTrie <int>(keywords);
var text = line;
matcher.ParseText(text, (hit) =>
{
switch (text.Substring(hit.Begin, hit.Length))
{
case ".class":
{
keywords.TryGetValue(".class", out currentCount);
keywords[".class"] = currentCount + 1;
break;
}
case ".method":
{
keywords.TryGetValue(".method", out currentCount);
keywords[".method"] = currentCount + 1;
break;
}
case "interface":
{
keywords.TryGetValue("interface", out currentCount);
keywords["interface"] = currentCount + 1;
break;
}
case ".property":
{
keywords.TryGetValue(".property", out currentCount);
keywords[".property"] = currentCount + 1;
break;
}
case ".assembly":
{
keywords.TryGetValue(".assembly", out currentCount);
keywords[".assembly"] = currentCount + 1;
break;
}
default:
{
break;
}
}
});
if (flag3)
{
entityMissDic1[".class"] = keywords[".class"];
entityMissDic1[".method"] = keywords[".method"];
entityMissDic1["interface"] = keywords["interface"];
entityMissDic1[".property"] = keywords[".property"];
entityMissDic1[".assembly"] = keywords[".assembly"];
entityMissDic1["length"] = Convert.ToInt32(length);
}
else
{
entityMissDic2[".class"] = keywords[".class"];
entityMissDic2[".method"] = keywords[".method"];
entityMissDic2["interface"] = keywords["interface"];
entityMissDic2[".property"] = keywords[".property"];
entityMissDic2[".assembly"] = keywords[".assembly"];
entityMissDic2["length"] = Convert.ToInt32(length);
}
//close the file
sr.Close();
return(true);
}