public virtual async Task <ICrawlerResultList <T> > Crawl(
CrawlerParameters parameters,
CancellationToken token)
{
if (parameters.VolatileParameters.UseCache && Cache.IsCached(parameters))
{
return(CrawlerResultBase <T> .FromCache(Cache.Get(parameters)));
}
try
{
var source = await Source.ObtainSource(parameters, token);
var result = await Parser.Parse(source, parameters);
Cache.Set(result.Results, parameters);
return(result);
}
catch (Exception e)
{
return(new CrawlerResultBase <T>
{
Success = false
});
}
}
public CacheResult <T> Get(CrawlerParameters parameters)
{
return(new CacheResult <T>()
{
Cache = _cache[ToKey(parameters)],
CacheTime = _cacheTimes[ToKey(parameters)]
});
}
private string ToKey(CrawlerParameters crawlerParameters)
{
return($"{crawlerParameters.Parameters.SearchQuery}_{crawlerParameters.VolatileParameters.Page}");
}
public bool IsCached(CrawlerParameters parameters)
{
return(_cache.ContainsKey(ToKey(parameters)));
}
public void Set(IEnumerable <T> items, CrawlerParameters parameters)
{
_cache[ToKey(parameters)] = new List <T>(items);
_cacheTimes[ToKey(parameters)] = DateTime.UtcNow;
}
async Task <ICrawlerResultList <ICrawlerResultItem> > ICrawler.Crawl(CrawlerParameters parameters,
CancellationToken token)
{
return((ICrawlerResultList <ICrawlerResultItem>) await Crawl(parameters, token));
}
public bool IsCached(CrawlerParameters parameters)
{
return(Cache.IsCached(parameters));
}