public CrawlMaster(CrawlJob job) { Job = job; RunningStatus = new JobStatusUpdate(Job); TotalStats = new CrawlJobStats(Job); Context.SetReceiveTimeout(TimeSpan.FromSeconds(5)); WaitingForTracker(); }
public DownloadCoordinator(CrawlJob job, IActorRef commander, IActorRef downloadsTracker, long maxConcurrentDownloads) { Job = job; DownloadsTracker = downloadsTracker; MaxConcurrentDownloads = maxConcurrentDownloads; Commander = commander; Stats = new CrawlJobStats(Job); Receiving(); }
private void Receiving() { Receive<PublishStatsTick>(stats => { if (!Stats.IsEmpty) { _logger.Info("Publishing {0} to parent", Stats); Commander.Tell(Stats.Copy()); //reset our stats after publishing Stats = Stats.Reset(); } }); //Received word from a ParseWorker that we need to check for new documents Receive<CheckDocuments>(documents => { //forward this onto the downloads tracker, but have it reply back to us ((ICanTell) DownloadsTracker).Tell(documents, Self); }); //Update our local stats Receive<DiscoveredDocuments>(discovered => { Stats = Stats.WithDiscovered(discovered); }); //Received word from the DownloadTracker that we need to process some docs Receive<ProcessDocuments>(process => { foreach (var doc in process.Documents) { // Context.Parent is the router between the coordinators and the Commander if (doc.IsImage) { Context.Parent.Tell(new DownloadWorker.DownloadImage(doc)); } else { Context.Parent.Tell(new DownloadWorker.DownloadHtmlDocument(doc)); } } }); //hand the work off to the downloaders Receive<DownloadWorker.IDownloadDocument>(download => { DownloaderRouter.Tell(download); }); Receive<CompletedDocument>(completed => { //TODO: send verbose status messages to commander here? Stats = Stats.WithCompleted(completed); }); /* Set all of our local downloaders to message our local parsers */ Receive<DownloadWorker.RequestParseActor>(request => { Sender.Tell(new DownloadWorker.SetParseActor(ParserRouter)); }); /* Set all of our local parsers to message our local downloaders */ Receive<ParseWorker.RequestDownloadActor>(request => { Sender.Tell(new ParseWorker.SetDownloadActor(DownloaderRouter)); }); }