private bool ShouldCrawlNextPages()
        {
            CrawlQueueItem currentCrawler = null;

            if (_pagesToCrawl.Count > 0)
            {
                currentCrawler = _pagesToCrawl.Peek();
            }

            return(!_contentFound &&
                   _crawledUri.Count + _currentTasks.Count < _configuration.MaxPages &&
                   (currentCrawler == null || currentCrawler.PageLevel < _configuration.MaxPageLevel));
        }
        private async Task <Tuple <CrawlPageResults, int> > CrawlPage(CrawlQueueItem crawlItem, CancellationToken cancellationToken)
        {
            CrawlPageResults results = null;

            try
            {
                // Раскладываем в корень страницы.
                using (FileStream pageStream = new FileStream(
                           Path.Combine(
                               _domainDirectory,
                               GetPageFileNameByUri(crawlItem.PageUri)),
                           FileMode.Create))
                {
                    var pageCrawler = new PageCrawler(crawlItem.PageUri);

                    results =
                        await pageCrawler.StartCrawling(pageStream, cancellationToken);

                    // В прекрасном мире, здесь был бы стрим, который пропускал через себя страницу,
                    // которую по мере вычитывания ему давал PageCrawler и, тем самым проверялось
                    // бы и стоп условие и все шло бы без излишних проверок и прочего, но имеем то,
                    // что имеем.
                    if (!string.IsNullOrEmpty(_configuration.StopString) &&
                        results.PageContent.Contains(_configuration.StopString))
                    {
                        Logger.Log(
                            LogLevel.Info,
                            $"Stop string found at {crawlItem.PageUri}");

                        _contentFound = true;
                    }
                }

                await LoadContent(results);
            }
            catch (Exception e)
            {
                Logger.Log(
                    LogLevel.Error,
                    e,
                    $"Page crawl exception with uri {crawlItem.PageUri}");
            }
            finally
            {
            }

            return(new Tuple <CrawlPageResults, int>(results, crawlItem.PageLevel));
        }
        private void EnqeueCrawlers(IEnumerable <Uri> nextPages, int parentLevel)
        {
            foreach (var page in nextPages)
            {
                if (!_queuedUri.Contains(page) && !_crawledUri.Contains(page))
                {
                    var item = new CrawlQueueItem()
                    {
                        PageUri   = page,
                        PageLevel = parentLevel + 1
                    };

                    _queuedUri.Add(page);
                    _pagesToCrawl.Enqueue(item);
                }
            }
        }
        public async Task <DomainCrawlStatistics> CrawlDomain(CancellationToken cancellationToken)
        {
            CancellationToken resultToken =
                CancellationTokenSource.CreateLinkedTokenSource(
                    cancellationToken,
                    _internalCTS.Token).Token;

            _contentFound = false;

            await Task.Factory.StartNew(
                async() =>
            {
                InitQueue();

                Directory.CreateDirectory(_domainDirectory);

                RobotsParams robotsParams = await _robotsReader.GetRobotsParams();

                while ((_pagesToCrawl.Count > 0 && ShouldCrawlNextPages() || _currentTasks.Count > 0))
                {
                    foreach (var task in _currentTasks.Where(t => t.IsCompleted).ToArray())
                    {
                        if (!task.IsFaulted && !task.IsCanceled)
                        {
                            var results = task.Result;

                            UpdateProgress(results.Item1);

                            if (results.Item2 < _configuration.MaxPageLevel)
                            {
                                EnqeueCrawlers(results.Item1.References, results.Item2);
                            }

                            _crawledUri.Add(task.Result.Item1.CrawledUri);
                            _queuedUri.Remove(task.Result.Item1.CrawledUri);
                        }

                        _currentTasks.Remove(task);
                    }

                    if (_currentTasks.Count >= _configuration.MaxTasks)
                    {
                        await Task.WhenAny(_currentTasks.ToArray());
                        continue;
                    }

                    if (_pagesToCrawl.Count > 0 &&
                        ShouldCrawlNextPages())
                    {
                        CrawlQueueItem currentCrawler = _pagesToCrawl.Dequeue();

                        _currentTasks.Add(CrawlPage(currentCrawler, resultToken));

                        if (robotsParams.CrawlDelay > 0)
                        {
                            await Task.Delay(robotsParams.CrawlDelay, resultToken);
                        }
                    }
                }


                SaveProgress();
            },
                TaskCreationOptions.LongRunning);

            return(_statistics);
        }