/// <summary> /// Start crawl process /// </summary> public virtual async Task CrawlAsync() { if (this.m_OnlyOneCrawlPerInstance) { throw new InvalidOperationException("Crawler instance cannot be reused"); } this.m_OnlyOneCrawlPerInstance = true; var parameters = new Parameter[] { new TypedParameter(typeof(Uri), this.m_BaseUri), new NamedParameter("crawlStart", this.m_BaseUri), new TypedParameter(typeof(Crawler), this), }; this.m_CrawlerQueue = this.m_LifetimeScope.Resolve <ICrawlerQueue>(parameters); parameters = parameters.AddToEnd(new TypedParameter(typeof(ICrawlerQueue), this.m_CrawlerQueue)).ToArray(); this.m_CrawlerHistory = this.m_LifetimeScope.Resolve <ICrawlerHistory>(parameters); parameters = parameters.AddToEnd(new TypedParameter(typeof(ICrawlerHistory), this.m_CrawlerHistory)).ToArray(); this.m_TaskRunner = this.m_LifetimeScope.Resolve <ITaskRunner>(parameters); parameters = parameters.AddToEnd(new TypedParameter(typeof(ITaskRunner), this.m_TaskRunner)).ToArray(); this.m_Logger = this.m_LifetimeScope.Resolve <ILog>(parameters); parameters = parameters.AddToEnd(new TypedParameter(typeof(ILog), this.m_Logger)).ToArray(); this.m_CrawlerRules = this.m_LifetimeScope.Resolve <ICrawlerRules>(parameters); this.m_Logger.Verbose("Crawl started @ {0}", this.m_BaseUri); this.m_WebDownloaderFactory = this.m_LifetimeScope.Resolve <Func <IWebDownloader> >(); using (this.m_CrawlCompleteEvent = new ManualResetEvent(false)) { this.m_Crawling = true; this.m_Runtime = Stopwatch.StartNew(); if (this.m_CrawlerQueue.Count > 0) { // Resume enabled ProcessQueue(); } else { await this.AddStepAsync(this.m_BaseUri, 0); } if (!this.m_CrawlStopped) { this.m_CrawlCompleteEvent.WaitOne(); } this.m_Runtime.Stop(); this.m_Crawling = false; } if (this.m_Cancelled) { OnCancelled(); } this.m_Logger.Verbose("Crawl ended @ {0} in {1}", this.m_BaseUri, this.m_Runtime.Elapsed); OnCrawlFinished(); }