public virtual async Task <ICrawlerResultList <T> > Crawl( CrawlerParameters parameters, CancellationToken token) { if (parameters.VolatileParameters.UseCache && Cache.IsCached(parameters)) { return(CrawlerResultBase <T> .FromCache(Cache.Get(parameters))); } try { var source = await Source.ObtainSource(parameters, token); var result = await Parser.Parse(source, parameters); Cache.Set(result.Results, parameters); return(result); } catch (Exception e) { return(new CrawlerResultBase <T> { Success = false }); } }
public CacheResult <T> Get(CrawlerParameters parameters) { return(new CacheResult <T>() { Cache = _cache[ToKey(parameters)], CacheTime = _cacheTimes[ToKey(parameters)] }); }
private string ToKey(CrawlerParameters crawlerParameters) { return($"{crawlerParameters.Parameters.SearchQuery}_{crawlerParameters.VolatileParameters.Page}"); }
public bool IsCached(CrawlerParameters parameters) { return(_cache.ContainsKey(ToKey(parameters))); }
public void Set(IEnumerable <T> items, CrawlerParameters parameters) { _cache[ToKey(parameters)] = new List <T>(items); _cacheTimes[ToKey(parameters)] = DateTime.UtcNow; }
async Task <ICrawlerResultList <ICrawlerResultItem> > ICrawler.Crawl(CrawlerParameters parameters, CancellationToken token) { return((ICrawlerResultList <ICrawlerResultItem>) await Crawl(parameters, token)); }
public bool IsCached(CrawlerParameters parameters) { return(Cache.IsCached(parameters)); }