private static void mergeStatistics(ArticleParserContext ctx, Dictionary <string, int> localStats, ArticleHeader header)
{
// First of all lock the global stats to
// avoid collisions caused by the workers
// running in parallel
lock (ctx.globalStats)
{
ctx.reportAction(MessageKind.Info, String.Format("[10]Worker {0}: Merging stats...", ctx.id));
// Run through the worker's local stas and merge it
foreach (var wordInfo in localStats)
{
string word = wordInfo.Key;
// Make sure the word exists
if (!ctx.globalStats.ContainsKey(word))
{
ctx.globalStats[word] = new WordStatsInfo();
}
// Add the local counter to the global one
ctx.globalStats[word].count = ctx.globalStats[word].count + wordInfo.Value;
// Add the article's reference
ctx.globalStats[word].articles.Add(header.url.ToString());
}
}
ctx.reportAction(MessageKind.Info, String.Format("[10]Worker {0}: Stats merged", ctx.id));
}
public static void processArticles(object context)
{
ArticleParserContext ctx = (context as ArticleParserContext);
if (ctx == null)
{
return; // Shouldn't be here
}
ctx.reportAction(MessageKind.Info, String.Format("Worker {0}: Started", ctx.id));
while (!ctx.shouldStop.IsCancellationRequested)
{
// Prepare to acquire a task from the shared queue
ArticleHeader task = null;
// Try to pick a task from the queue. For this
// we need to seriliaze access to the queue among
// all the workers, so we use a simple critical section
lock (ctx.articles)
{
// Check if the queue is not empty
if (ctx.articles.Count > 0)
{
// Take a task from it
task = ctx.articles.Dequeue();
}
}
// Now either we acquired a task or there are no more tasks
if (task == null || ctx.shouldStop.IsCancellationRequested)
{
break; // exit the working loop
}
// Report the allocation. Due to the multiline output
// we will have to lock other workers attempt to print
// at the same time
lock (ctx.reportAction)
{
ctx.reportAction(MessageKind.Info, String.Format("[10]Worker {0}: Parsing the article \"{1}\"", ctx.id, task.title));
}
// Process the article and measure its local
// word frequency
var localFrequency = processArticle(task, ctx.reportAction);
// Check what's up
if (localFrequency == null)
{
ctx.reportAction(MessageKind.Warning, "Failed to parse the article");
continue;
}
// Merge the local frequency into the global one
mergeStatistics(ctx, localFrequency, task);
// Report task completed
ctx.reportAction(MessageKind.Info, String.Format("[10]Worker {0}: Parsed the article", ctx.id));
}
bool wasStopped = ctx.shouldStop.IsCancellationRequested;
ctx.reportAction(wasStopped ? MessageKind.Warning : MessageKind.Info,
String.Format("Worker {0}: {1}", ctx.id,
wasStopped ? "Terminated" : "Finished"
));
}
