/// <summary>Gets resource urls.</summary>
/// <param name="endsWith">Resource pattern if any.</param>
private List <Uri> GetResources(string[] endsWith = null)
{
var res = new List <Uri>();
foreach (var uri in this.urls)
{
bool isResource = UrlHelper.IsResource(uri);
if (!isResource)
{
continue;
}
if (Sitemap.EndsWith(uri.LocalPath, endsWith))
{
res.Add(uri);
}
}
return(res);
}
public Scraper(Sitemap sitemap, string htmlDownloadPath, Action <double> reportProgress, CancellationToken token)
{
if (sitemap == null)
{
throw new NullReferenceException("sitemap");
}
if (htmlDownloadPath == null)
{
throw new NullReferenceException("htmlDownloadPath");
}
if (Directory.Exists(htmlDownloadPath))
{
Directory.Delete(htmlDownloadPath, true);
}
Directory.CreateDirectory(htmlDownloadPath);
this.sitemap = sitemap;
this.htmlDownloadPath = htmlDownloadPath;
this.reportProgress = reportProgress;
this.cancellationToken = token;
}