public Option <string> GetPage()
{
using (var client = _redisProvider.GetDatabase().GetReadOnlyClient())
{
var page = client.GetRandomItemFromSet("to_crawl");
if (page == null)
{
_logger.LogInformation("No more pages to crawl. Returning None.");
return(Option <string> .None);
}
return(page);
}
}
public void Save(PageCrawlResult page)
{
_logger.LogInformation($"Saving crawl result of page [{page.Url}] to the database.");
var db = _redisProvider.GetDatabase();
var serialized = JsonConvert.SerializeObject(page);
db.StringSet(page.Url, serialized);
}