public CrawlMaster(CrawlJob job)
{
Job = job;
RunningStatus = new JobStatusUpdate(Job);
TotalStats = new CrawlJobStats(Job);
Context.SetReceiveTimeout(TimeSpan.FromSeconds(5));
WaitingForTracker();
}
public DownloadCoordinator(CrawlJob job, IActorRef commander, IActorRef downloadsTracker, long maxConcurrentDownloads)
{
Job = job;
DownloadsTracker = downloadsTracker;
MaxConcurrentDownloads = maxConcurrentDownloads;
Commander = commander;
Stats = new CrawlJobStats(Job);
Receiving();
}
private void Receiving()
{
Receive<PublishStatsTick>(stats =>
{
if (!Stats.IsEmpty)
{
_logger.Info("Publishing {0} to parent", Stats);
Commander.Tell(Stats.Copy());
//reset our stats after publishing
Stats = Stats.Reset();
}
});
//Received word from a ParseWorker that we need to check for new documents
Receive<CheckDocuments>(documents =>
{
//forward this onto the downloads tracker, but have it reply back to us
((ICanTell) DownloadsTracker).Tell(documents, Self);
});
//Update our local stats
Receive<DiscoveredDocuments>(discovered =>
{
Stats = Stats.WithDiscovered(discovered);
});
//Received word from the DownloadTracker that we need to process some docs
Receive<ProcessDocuments>(process =>
{
foreach (var doc in process.Documents)
{
// Context.Parent is the router between the coordinators and the Commander
if (doc.IsImage)
{
Context.Parent.Tell(new DownloadWorker.DownloadImage(doc));
}
else
{
Context.Parent.Tell(new DownloadWorker.DownloadHtmlDocument(doc));
}
}
});
//hand the work off to the downloaders
Receive<DownloadWorker.IDownloadDocument>(download =>
{
DownloaderRouter.Tell(download);
});
Receive<CompletedDocument>(completed =>
{
//TODO: send verbose status messages to commander here?
Stats = Stats.WithCompleted(completed);
});
/* Set all of our local downloaders to message our local parsers */
Receive<DownloadWorker.RequestParseActor>(request =>
{
Sender.Tell(new DownloadWorker.SetParseActor(ParserRouter));
});
/* Set all of our local parsers to message our local downloaders */
Receive<ParseWorker.RequestDownloadActor>(request =>
{
Sender.Tell(new ParseWorker.SetDownloadActor(DownloaderRouter));
});
}
