public CrawlingIndexableGenerator( CrawlPlan crawlPlan, IndexableBuilder indexableBuilder, ManifestChangeTracker changeTracker ) { Debug.Assert (crawlPlan != null); Debug.Assert (indexableBuilder != null); Debug.Assert (changeTracker != null); this.crawlPlan = crawlPlan; this.indexableBuilder = indexableBuilder; this.changeTracker = changeTracker; }
void SaveStateAndReschedule(AccountSyncState syncState, AccountSyncStateStore store, CrawlPlan crawlPlan) { Log.Info ("Sync finished"); // If the crawl plan completed successfully, then update the // last update time, otherwise leave the last update time the same // so that we can try to crawl again next time around. // The crawl may abort if delicious starts returning 503s if (! crawlPlan.HasNextCommand ()) { syncState.UpdateLastSyncTime (); } try { store.SaveSyncState (syncState); } catch (Exception e) { Log.Error (e, "Error saving sync state"); } RescheduleCrawl (); }
CrawlingIndexableGenerator BuildIndexableGenerator() { Debug.Assert (account.IsValid); Log.Info ("Beginning sync for " + account.Username); // Create some common services DeliciousRequester requester = new DeliciousRequester (account); RemoteDataAccess remoteDao = new RemoteDataAccess (requester); // Load the sync state AccountSyncStateStore store = new AccountSyncStateStore (requester, remoteDao, queryable); AccountSyncState syncState = null; try { syncState = store.LoadSyncState (); } catch (DeliciousUnavailableException e) { Log.Warn (e.ToString()); return null; } catch (Exception e) { Log.Error (e, "Error loading sync state"); return null; } if (! syncState.ShouldCrawl ()) { Log.Info ("Skipping crawl"); return null; } // Build the crawl plan CrawlPlan crawlPlan = new CrawlPlan( syncState.LocalManifest, syncState.RemoteManifest, remoteDao); // Build indexables from the steps of the crawl plan IndexableBuilder indexableBuilder = new IndexableBuilder (requester); // Update the sync state's local manifest as bookmarks are indexed ManifestChangeTracker changeTracker = new ManifestChangeTracker (syncState.LocalManifest); // Create the indexable generator that will execute the crawl plan CrawlingIndexableGenerator indexableGenerator = new CrawlingIndexableGenerator( crawlPlan, indexableBuilder, changeTracker); // After the crawl is done, we'll need to save the sync state, // and set up another scheduled crawl indexableGenerator.FinishedCrawl +=()=> SaveStateAndReschedule (syncState, store, crawlPlan); return indexableGenerator; }
public Indexable GetNextIndexable() { CrawlCommand command = null; try { Debug.Assert (crawlPlan != null); command = crawlPlan.GetNextCommand (); } catch (DeliciousUnavailableException e) { Log.Warn (e.ToString ()); // By setting the crawl plan to null, HasNextIndexable will return false // and the crawl will end. crawlPlan = null; } if (command == null) { return null; } try { Indexable newIndexable = indexableBuilder.Build (command); if (newIndexable != null) { changeTracker.UpdateManifest (command); } return newIndexable; } catch (Exception e) { Log.Error (e, "Exception building indexable"); return null; } }