public Dictionary <string, int> GetWordOccurences(string text, List <Stopwords> stopwords)
{
Dictionary <string, int> occurenceDictionary = new Dictionary <string, int>();
var wordList = RegexExtensions.listMatchingRegex(RegexExtensions.isWord, text);
foreach (var word in wordList)
{
var searchWord = word.ToLower().Trim();
if (!stopwords
.Select(x => x.Stopword)
.Contains(searchWord))
{
if (!occurenceDictionary.ContainsKey(searchWord))
{
occurenceDictionary.Add(searchWord, 1);
}
else
{
occurenceDictionary[searchWord] += 1;
}
}
}
return(occurenceDictionary);
}
public async Task <ScrapedData> ScrapeData(string text)
{
var scrapedData = new ScrapedData();
if (!string.IsNullOrEmpty(text))
{
scrapedData.BodyContent = text;
var links = RegexExtensions.listMatchingRegex(RegexExtensions.isLink, text);
foreach (var link in links)
{
scrapedData.Links.Add(link);
}
}
return(scrapedData);
}