public void Done(ICrawlerAgent agent) { context.SaveChanges(); foreach (var s in storeAgents.Values) { s.Close(); } isRunning = false; }
public void Init(ICrawlerAgent agent, IMatcherManager manager) { log.Debug("Initializing Matcher"); if (agent == null) { throw new ArgumentNullException(nameof(agent)); } if (manager == null) { throw new ArgumentNullException(nameof(manager)); } this.agent = agent; this.manager = manager; this.agent.PageCrawled += Agent_PageCrawledAsync; log.Debug($"Attached event listener to PageCrawled of agent with id {this.agent.GetId()}"); init = true; }
public void Start() { log.Debug("Site Processor starting..."); if (isRunning) { log.Warn("Site Processor is already running"); return; } isRunning = true; crawler = new CrawlerAgent(); crawler.Init(this, site.Id); matcher = new Matcher(); matcher.Init(crawler, this); foreach (var cat in site.Categories) { var agent = new StoreAgent(); agent.Init(cat.Id, site.OutputFolder); storeAgents.Add(cat.Name, agent); } crawler.Start(); log.Debug("Site Processor started"); }