public ResponseData DownloadResponse(CrawlingOption option)
{
if (option.Uri.Contains("://docs.google.com"))
{
return(_googleWorker.DownloadResponse(option));
}
return(_worker.DownloadResponse(option));
}
public void Download(DownloadServiceOption options)
{
var fileLinks = new Dictionary <string, IEnumerable <string> >();
foreach (var urlAndPattern in options.UrlsAndPatterns)
{
var uri = urlAndPattern.Key;
var filePattern = urlAndPattern.Value;
Console.WriteLine("Start getting " + uri);
var articleResult = _worker.DownloadResponse(new CrawlingOption(uri));
if (articleResult.StatusCode == HttpStatusCode.OK)
{
fileLinks.Add(uri, GetFileLinks(articleResult.ReadAsText(), filePattern));
}
else
{
Console.WriteLine(articleResult.StatusDescription);
}
}
var count = 0;
var total = fileLinks.Values.SelectMany(x => x).Count();
foreach (var kvp in fileLinks)
{
foreach (var file in kvp.Value)
{
count += 1;
Console.WriteLine("Process " + count + "/" + total);
ProcessDownload(options, file, kvp);
}
}
}
