private async void button3_Click(object sender, EventArgs e)
{
//var strData = File.ReadAllText(@"D:\wamp\tmp\3.txt");
//var data = Newtonsoft.Json.JsonConvert.DeserializeObject<List<Word>>(strData);
//var wordsWithOccurences = await MongoDBHandler.GetWordCollection();
var wordsWithOccurences = _intermediateWordContainer;
var articles = await MySQLHandler.GetAllArticlesAsync();
//articles.Reverse();
//articles = articles.Take(110).ToList();
// take top 1000 words
var wordsToProcess = wordsWithOccurences.Take(1000).ToList();
// get occurence of each word by individual article
var articleWordOccurenceList = new List <ArticleWordOccurence>();
int counter = 0;
await MongoDBHandler.ClearArticleWordOccurences();
var thread = new Thread(async() =>
{
foreach (var article in articles)
{
foreach (var word in wordsToProcess)
{
var count = GeneralHandlers.CountStringOccurrences(article.Text, word.WordName);
articleWordOccurenceList.Add(new ArticleWordOccurence
{
ArticleId = article.Id,
Word = word.WordName,
Occurence = count
});
}
if (counter > 1000)
{
counter = 0;
await MongoDBHandler.InsertArticleWordOccurences(articleWordOccurenceList);
articleWordOccurenceList.Clear();
}
counter++;
}
await MongoDBHandler.InsertArticleWordOccurences(articleWordOccurenceList);
});
thread.Start();
}
