public void SetUp()
{
var config = new ConfigurationOptions() { EndPoints = { { _host, _port } }, Password = _password, DefaultDatabase = _database, AllowAdmin = true };
_state = SchedulerFunc.Create(config, "Test", _database);
var server = _state.Connection.GetServer(_state.Connection.GetEndPoints().First());
server.FlushDatabase(_state.Database);
}
public static void Add(SchedulerState state, PageToCrawl page)
{
var json = JsonConvert.SerializeObject(page);
var url = page.Uri.AbsoluteUri;
var trans = CreateTransaction(state);
var crawledPageKey = CrawledPageKey(state.SiteName, url);
var pageToCrawlKey = PageToCrawlKey(state.SiteName);
trans.AddCondition(Condition.KeyNotExists(crawledPageKey));
trans.StringSetAsync(crawledPageKey, "");
trans.ListLeftPushAsync(pageToCrawlKey, json);
trans.ExecuteAsync().Wait();
}
private static PageToCrawl GetNextInner(SchedulerState state, int attempt)
{
if (attempt > 10) return null;
var pageToCrawlKey = PageToCrawlKey(state.SiteName);
var trans = CreateTransaction(state);
trans.AddCondition(Condition.KeyExists(pageToCrawlKey));
var taskJson = trans.ListRightPopAsync(pageToCrawlKey);
var committed = trans.ExecuteAsync().Result;
if (!committed)
{
Thread.Sleep(500);
return GetNextInner(state, attempt + 1);
}
var json = taskJson.Result;
var pageToCrawl = JsonConvert.DeserializeObject<PageToCrawl>(json);
pageToCrawl.PageBag = new ExpandoObject();
return pageToCrawl;
}
public RedisScheduler(SchedulerState state)
{
_state = state;
}
public static void Clear(SchedulerState state)
{
var database = state.Connection.GetDatabase();
var server = state.Connection.GetServer(state.Connection.GetEndPoints().First());
foreach(var key in server.Keys(state.Database, pattern: CrawledPageKey(state.SiteName, "*")))
{
database.KeyDelete(key);
}
database.KeyDelete(PageToCrawlKey(state.SiteName));
}
public static PageToCrawl GetNext(SchedulerState state)
{
return GetNextInner(state, 1);
}
public static int Count(SchedulerState state)
{
var count = GetDatabase(state).ListLengthAsync(PageToCrawlKey(state.SiteName)).Result;
return Convert.ToInt32(count);
}
public static ITransaction CreateTransaction(SchedulerState scheduler)
{
return scheduler.Connection.GetDatabase().CreateTransaction();
}
public static IDatabase GetDatabase(SchedulerState state)
{
return state.Connection.GetDatabase();
}