public CrawlScheduler(IUrlHasher urlHasher, IConfiguration configuration, ISinglePageCrawler pageCrawler, ICrawlUrlRepository crawlUrlRepository) { this.urlHasher = urlHasher; this.configuration = configuration; this.pageCrawler = pageCrawler; this.crawlUrlRepository = crawlUrlRepository; schedulingBlock = new ActionBlock<PageCrawlResult>(result => { var websiteDefinition = result.CrawlUrl.WebsiteDefinition; Interlocked.Increment(ref websiteDefinition.ProcessedUrlsCount); RaisePageCrawled(result); if (result.Links.Any()) { Schedule(result.Links); } if (websiteDefinition.UrlsToProcessCount == websiteDefinition.ProcessedUrlsCount) { websiteProcessingDefinitions[websiteDefinition].Complete(); } ScheduleNext(); }); }
public CrawlScheduler(IUrlHasher urlHasher, IConfiguration configuration, ISinglePageCrawler pageCrawler, ICrawlUrlRepository crawlUrlRepository) { this.urlHasher = urlHasher; this.configuration = configuration; this.pageCrawler = pageCrawler; this.crawlUrlRepository = crawlUrlRepository; schedulingBlock = new ActionBlock <PageCrawlResult>(result => { var websiteDefinition = result.CrawlUrl.WebsiteDefinition; Interlocked.Increment(ref websiteDefinition.ProcessedUrlsCount); RaisePageCrawled(result); if (result.Links.Any()) { Schedule(result.Links); } if (websiteDefinition.UrlsToProcessCount == websiteDefinition.ProcessedUrlsCount) { websiteProcessingDefinitions[websiteDefinition].Complete(); } ScheduleNext(); }); }
public ShortenUrlCommandHandler( IUrlHasher urlHasher, IShortenUrlRepository shortenUrlRepository, ICodeServiceClient codeServiceClient, IPublishEndpoint publishEndpoint, IDistributedCache distributedCache) { _urlHasher = urlHasher; _shortenUrlRepository = shortenUrlRepository; _codeServiceClient = codeServiceClient; _publishEndpoint = publishEndpoint; _distributedCache = distributedCache; }