public async Task <Protocols.Response.Crawling> Execute(Protocols.Request.Crawling crawling)
{
var onCrawlDataDelegate = new CrawlerBase.CrawlDataDelegate(OnNewCrawlData);
var sources = crawling.All ? (await _sourceService.All()).ConvertAll(x => x.ToProtocol()) : crawling.Sources;
Parallel.ForEach(sources, new ParallelOptions {
MaxDegreeOfParallelism = 16
},
async source =>
{
switch (source.Type)
{
case CrawlingType.Ruliweb:
await new RuliwebCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.Clien:
await new ClienCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.SlrClub:
await new SlrclubCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.Ppomppu:
await new PpomppuCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.TodayHumor:
await new TodayhumorCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.FmKorea:
await new FmkoreaCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.InvenNews:
await new InvenNewsCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.HumorUniv:
await new HumorUnivCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
case CrawlingType.Itcm:
await new ItcmCrawler(onCrawlDataDelegate, _mongoDbService.Database, source.ToModel()).RunAsync();
break;
default:
throw new DeveloperException(Code.ResultCode.NotImplementedYet);
}
}
);
return(new Protocols.Response.Crawling
{
ResultCode = Code.ResultCode.Success
});
}
public async Task <Protocols.Response.Crawling> Crawling([FromBody] Protocols.Request.Crawling crawling)
{
return(await _crawlingService.Execute(crawling));
}
