public Dictionary <string, int> GetWordOccurences(string text, List <Stopwords> stopwords) { Dictionary <string, int> occurenceDictionary = new Dictionary <string, int>(); var wordList = RegexExtensions.listMatchingRegex(RegexExtensions.isWord, text); foreach (var word in wordList) { var searchWord = word.ToLower().Trim(); if (!stopwords .Select(x => x.Stopword) .Contains(searchWord)) { if (!occurenceDictionary.ContainsKey(searchWord)) { occurenceDictionary.Add(searchWord, 1); } else { occurenceDictionary[searchWord] += 1; } } } return(occurenceDictionary); }
public async Task <ScrapedData> ScrapeData(string text) { var scrapedData = new ScrapedData(); if (!string.IsNullOrEmpty(text)) { scrapedData.BodyContent = text; var links = RegexExtensions.listMatchingRegex(RegexExtensions.isLink, text); foreach (var link in links) { scrapedData.Links.Add(link); } } return(scrapedData); }