/// <summary>
/// Start crawl process
/// </summary>
public virtual async Task CrawlAsync()
{
if (this.m_OnlyOneCrawlPerInstance)
{
throw new InvalidOperationException("Crawler instance cannot be reused");
}
this.m_OnlyOneCrawlPerInstance = true;
var parameters = new Parameter[]
{
new TypedParameter(typeof(Uri), this.m_BaseUri),
new NamedParameter("crawlStart", this.m_BaseUri),
new TypedParameter(typeof(Crawler), this),
};
this.m_CrawlerQueue = this.m_LifetimeScope.Resolve <ICrawlerQueue>(parameters);
parameters = parameters.AddToEnd(new TypedParameter(typeof(ICrawlerQueue), this.m_CrawlerQueue)).ToArray();
this.m_CrawlerHistory = this.m_LifetimeScope.Resolve <ICrawlerHistory>(parameters);
parameters = parameters.AddToEnd(new TypedParameter(typeof(ICrawlerHistory), this.m_CrawlerHistory)).ToArray();
this.m_TaskRunner = this.m_LifetimeScope.Resolve <ITaskRunner>(parameters);
parameters = parameters.AddToEnd(new TypedParameter(typeof(ITaskRunner), this.m_TaskRunner)).ToArray();
this.m_Logger = this.m_LifetimeScope.Resolve <ILog>(parameters);
parameters = parameters.AddToEnd(new TypedParameter(typeof(ILog), this.m_Logger)).ToArray();
this.m_CrawlerRules = this.m_LifetimeScope.Resolve <ICrawlerRules>(parameters);
this.m_Logger.Verbose("Crawl started @ {0}", this.m_BaseUri);
this.m_WebDownloaderFactory = this.m_LifetimeScope.Resolve <Func <IWebDownloader> >();
using (this.m_CrawlCompleteEvent = new ManualResetEvent(false))
{
this.m_Crawling = true;
this.m_Runtime = Stopwatch.StartNew();
if (this.m_CrawlerQueue.Count > 0)
{
// Resume enabled
ProcessQueue();
}
else
{
await this.AddStepAsync(this.m_BaseUri, 0);
}
if (!this.m_CrawlStopped)
{
this.m_CrawlCompleteEvent.WaitOne();
}
this.m_Runtime.Stop();
this.m_Crawling = false;
}
if (this.m_Cancelled)
{
OnCancelled();
}
this.m_Logger.Verbose("Crawl ended @ {0} in {1}", this.m_BaseUri, this.m_Runtime.Elapsed);
OnCrawlFinished();
}