public void Done(ICrawlerAgent agent)
{
context.SaveChanges();
foreach (var s in storeAgents.Values)
{
s.Close();
}
isRunning = false;
}
public void Init(ICrawlerAgent agent, IMatcherManager manager)
{
log.Debug("Initializing Matcher");
if (agent == null)
{
throw new ArgumentNullException(nameof(agent));
}
if (manager == null)
{
throw new ArgumentNullException(nameof(manager));
}
this.agent = agent;
this.manager = manager;
this.agent.PageCrawled += Agent_PageCrawledAsync;
log.Debug($"Attached event listener to PageCrawled of agent with id {this.agent.GetId()}");
init = true;
}
public void Start()
{
log.Debug("Site Processor starting...");
if (isRunning)
{
log.Warn("Site Processor is already running");
return;
}
isRunning = true;
crawler = new CrawlerAgent();
crawler.Init(this, site.Id);
matcher = new Matcher();
matcher.Init(crawler, this);
foreach (var cat in site.Categories)
{
var agent = new StoreAgent();
agent.Init(cat.Id, site.OutputFolder);
storeAgents.Add(cat.Name, agent);
}
crawler.Start();
log.Debug("Site Processor started");
}
