public void SetUp() { var config = new ConfigurationOptions() { EndPoints = { { _host, _port } }, Password = _password, DefaultDatabase = _database, AllowAdmin = true }; _state = SchedulerFunc.Create(config, "Test", _database); var server = _state.Connection.GetServer(_state.Connection.GetEndPoints().First()); server.FlushDatabase(_state.Database); }
public static void Add(SchedulerState state, PageToCrawl page) { var json = JsonConvert.SerializeObject(page); var url = page.Uri.AbsoluteUri; var trans = CreateTransaction(state); var crawledPageKey = CrawledPageKey(state.SiteName, url); var pageToCrawlKey = PageToCrawlKey(state.SiteName); trans.AddCondition(Condition.KeyNotExists(crawledPageKey)); trans.StringSetAsync(crawledPageKey, ""); trans.ListLeftPushAsync(pageToCrawlKey, json); trans.ExecuteAsync().Wait(); }
private static PageToCrawl GetNextInner(SchedulerState state, int attempt) { if (attempt > 10) return null; var pageToCrawlKey = PageToCrawlKey(state.SiteName); var trans = CreateTransaction(state); trans.AddCondition(Condition.KeyExists(pageToCrawlKey)); var taskJson = trans.ListRightPopAsync(pageToCrawlKey); var committed = trans.ExecuteAsync().Result; if (!committed) { Thread.Sleep(500); return GetNextInner(state, attempt + 1); } var json = taskJson.Result; var pageToCrawl = JsonConvert.DeserializeObject<PageToCrawl>(json); pageToCrawl.PageBag = new ExpandoObject(); return pageToCrawl; }
public RedisScheduler(SchedulerState state) { _state = state; }
public static void Clear(SchedulerState state) { var database = state.Connection.GetDatabase(); var server = state.Connection.GetServer(state.Connection.GetEndPoints().First()); foreach(var key in server.Keys(state.Database, pattern: CrawledPageKey(state.SiteName, "*"))) { database.KeyDelete(key); } database.KeyDelete(PageToCrawlKey(state.SiteName)); }
public static PageToCrawl GetNext(SchedulerState state) { return GetNextInner(state, 1); }
public static int Count(SchedulerState state) { var count = GetDatabase(state).ListLengthAsync(PageToCrawlKey(state.SiteName)).Result; return Convert.ToInt32(count); }
public static ITransaction CreateTransaction(SchedulerState scheduler) { return scheduler.Connection.GetDatabase().CreateTransaction(); }
public static IDatabase GetDatabase(SchedulerState state) { return state.Connection.GetDatabase(); }