internal void Crawl(Type type, ICrawlToken token, object value)
{
if (_crawledItems.Contains(value))
{
return;
}
_crawledItems.Add(value);
CrawlingType crawlingType = Configuration.CrawlingTypeDetection.DetectType(type, token);
ICrawler crawler;
switch (crawlingType)
{
case CrawlingType.Reference:
crawler = new ReferenceCrawler(value, type, token, this);
break;
case CrawlingType.Value:
crawler = new ValueCrawler(value, type, token, this);
break;
case CrawlingType.Enumerable:
crawler = new EnumerableCrawler(value, type, token, this);
break;
case CrawlingType.Dictionary:
crawler = new DictionaryCrawler(value, type, token, this);
break;
default:
throw new ArgumentOutOfRangeException();
}
crawler.Crawl();
}
private async Task <string> GetSourceId(CrawlingType crawlingType, string boardName)
{
if (crawlingType == CrawlingType.Rss)
{
return(string.Empty);
}
var source = await _sourceService.GetByName(crawlingType, boardName);
if (source == null)
{
throw new DeveloperException(ResultCode.NotFoundSource);
}
return(source.Id);
}
public async Task <Source> GetByName(CrawlingType crawlingType, string boardName)
{
return(await _mongoDbSource.FindOneAsync(Builders <Source> .Filter.Eq(x => x.Type, crawlingType)&
Builders <Source> .Filter.Eq(x => x.Name, boardName)));
}
private FilterDefinition <Notification> GetFilterDefinition(string sourceId, CrawlingType crawlingType, NotificationType notificationType)
{
return(Builders <Notification> .Filter.Eq(x => x.SourceId, sourceId) &
Builders <Notification> .Filter.Eq(x => x.CrawlingType, crawlingType) &
Builders <Notification> .Filter.Eq(x => x.Type, notificationType));
}