override protected void DoTaskReal()
{
// Since this is a generator, we want the task to
// get re-scheduled after it is run.
Reschedule = true;
// Number of times a null indexable was returned. We don't want
// to spin tightly in a loop here if we're not actually indexing
// things.
int misfires = 0;
bool flushed = false;
do
{
if (!generator.HasNextIndexable())
{
// Of course, don't reschedule if there is no more work to do.
Reschedule = false;
break;
}
Indexable generated;
generated = generator.GetNextIndexable();
// Note that the indexable generator can return null.
// This means that the generator didn't have an indexable
// to return this time through, but it does not mean that
// its processing queue is empty.
if (generated == null)
{
misfires++;
if (misfires > 179) // Another totally arbitrary number
{
break;
}
else
{
continue;
}
}
if (queryable.PreAddIndexableHook(generated))
{
queryable.AddIndexable(generated);
}
else
{
generated.Cleanup();
}
// We keep adding indexables until a flush goes through.
} while (!(flushed = queryable.ConditionalFlush()));
if (!flushed)
{
queryable.Flush();
}
generator.PostFlushHook();
}
override protected void DoTaskReal()
{
// If our last generator is still doing stuff, just reschedule
// and return. This keeps us from generating more tasks until
// the last one we started runs to completion.
if ((current_generator != null && current_generator.HasNextIndexable()) ||
current_dir != null)
{
Reschedule = true;
return;
}
lock (big_lock) {
Log.Debug("Running file crawl task");
current_dir = queryable.GetNextDirectoryToCrawl();
if (current_dir == null)
{
Log.Debug("Done crawling files!!!");
SetIsActive(false, current_dir);
return;
}
SetIsActive(true, current_dir);
}
if (!current_dir.IsAttached)
{
Reschedule = true;
return;
}
if (FileSystemQueryable.Debug)
{
Logger.Log.Debug("Starting crawl of '{0}'", current_dir.FullName);
if (current_dir.State == DirectoryState.PossiblyClean)
{
Log.Debug("It looks as though we've crawled '{0}' before", current_dir.FullName);
}
}
// Schedule a DirectoryIndexableGenerator
// for that directory, and then reschedule ourselves.
try {
current_generator = new DirectoryIndexableGenerator(queryable, current_dir);
} catch (DirectoryNotFoundException ex) {
Logger.Log.Debug("Couldn't crawl '{0}'", current_dir.FullName);
// FIXME: If our attempt to crawl the directory fails, just
// mark it as uncrawlable and move on. This isn't optimal behavior,
// but works around bugs involving weird permissions for now.
current_dir.MarkAsUncrawlable();
current_dir = null;
}
if (current_generator != null)
{
Scheduler.TaskGroup group;
group = Scheduler.NewTaskGroup("Crawl task group", null, our_post_hook);
Scheduler.Task task;
task = queryable.NewAddTask(current_generator);
task.AddTaskGroup(group);
SpawnChild(task);
}
Reschedule = true;
}