public Indexable Build(CrawlCommand command)
{
Debug.Assert (command != null);
switch (command.Action)
{
case CrawlAction.Add:
return BuildAddIndexable (command.Post);
case CrawlAction.Update:
return BuildUpdateIndexable (command.Post);
case CrawlAction.Remove:
return BuildRemoveIndexable (command.Post);
default:
Debug.Fail ("Unexpected action");
return null;
}
}
void FillCommandQueue()
{
IEnumerable <PartialCrawlCommand> nextPartialCommands = DequeueNextPartialCommands ();
IDictionary <string, CrawlAction> urlActionDict = new Dictionary <string, CrawlAction> ();
IList <RemoteManifestEntry> entryList = new List <RemoteManifestEntry> ();
foreach (PartialCrawlCommand partialCommand in nextPartialCommands)
{
entryList.Add (partialCommand.Entry);
urlActionDict [partialCommand.Entry.UrlHash] = partialCommand.Action;
}
try
{
foreach (Post post in dao.GetPostsFromEntries (entryList))
{
try
{
CrawlCommand newCommand = new CrawlCommand ();
newCommand.Action = urlActionDict [post.Hash];
newCommand.Post = post;
commands.Enqueue (newCommand);
}
catch (KeyNotFoundException e)
{
// This one won't get indexed
Log.Error (e, "Failed to create command for " + post);
}
}
}
catch (DeliciousUnavailableException)
{
// Let caller know we should stop
throw;
}
catch (Exception e)
{
// A lot of stuff's not getting indexed
Log.Error (e, "Failed to retrieve posts from delicious");
}
}
void BuildCrawlPlan(LocalManifest localManifest, RemoteManifest remoteManifest)
{
Log.Debug ("Building crawl plan");
foreach (RemoteManifestEntry remoteEntry in remoteManifest)
{
PartialCrawlCommand newCommand = PreparePartialCommandFromRemoteEntry (remoteEntry, localManifest);
if (newCommand != null)
{
partialCommands.Enqueue (newCommand);
}
// Remove entries from the local manifest, so we can later
// tell which entries only exist locally
if (localManifest.ContainsKey (remoteEntry.UrlHash))
{
localManifest.Remove (remoteEntry.UrlHash);
}
}
// Any entries left in the local manifest don't exist remotely,
// so they need to be removed
foreach (KeyValuePair <string, LocalManifestEntry> entryPair in localManifest)
{
CrawlCommand newCommand = new CrawlCommand ();
newCommand.Action = CrawlAction.Remove;
newCommand.Post = BuildFakePost (entryPair.Value);
commands.Enqueue (newCommand);
}
}
