private IndexerReceipt [] FlushGeneratedIndexables_Unlocked(IndexerRequest request)
{
int num_indexed = 0;
ArrayList receipt_queue;
receipt_queue = new ArrayList();
if (text_cache != null)
{
text_cache.BeginTransaction();
}
IndexWriter primary_writer, secondary_writer;
primary_writer = new IndexWriter(PrimaryStore, IndexingAnalyzer, false);
secondary_writer = null;
IndexerAddedReceipt r;
Log.Debug("Continuing indexing generated indexables from {0} indexables", deferred_indexables.Count);
// Access using index so that we can add more deferred_indexable at the front
// deferred_indexables are added at the front and fetched from the front like a stack
while (deferred_indexables.Count > 0)
{
DeferredInfo di = (DeferredInfo)deferred_indexables [0];
if (di.Indexable.LocalState ["HasNextIndexable"] != null)
{
// Finally, good to index
// Should we do a sanity check ? deferred_indexables [0] =?= di
deferred_indexables.RemoveAt(0);
Document persistent_prop_doc = null;
if (di.PersistentPropDocs != null)
{
persistent_prop_doc = (Document)di.PersistentPropDocs [di.Indexable.Uri];
}
if (di.Indexable.DisplayUri != di.Indexable.ContentUri)
{
Log.Debug("+{0} ({1}) [deferred]", di.Indexable.DisplayUri, di.Indexable.ContentUri);
}
else
{
Log.Debug("+{0} [deferred]", di.Indexable.DisplayUri);
}
AddDocumentToIndex(di.Indexable, persistent_prop_doc, primary_writer, ref secondary_writer);
// Add the receipt if the indexable was submitted and not generated
if (di.Indexable.LocalState ["GeneratedIndexable"] == null)
{
r = new IndexerAddedReceipt(di.Indexable.Id);
r.FilterName = di.Filter.GetType().ToString();
r.FilterVersion = di.Filter.Version;
receipt_queue.Add(r);
}
// Cleanup, and text cache maintenance.
di.Cleanup();
if (disable_textcache && text_cache != null)
{
text_cache.Delete(di.Indexable.Uri);
}
num_indexed++;
continue;
}
Log.Debug("Processing deferred indexable from {0}", di.Indexable.DisplayUri);
bool next = false;
while (!next && !Shutdown.ShutdownRequested && num_indexed <= RequestFlushThreshold)
{
Indexable generated_indexable = null;
bool next_indexable = false;
try {
next_indexable = di.Filter.GenerateNextIndexable(out generated_indexable);
} catch (Exception e) {
Log.Error(e, "Error while generating next indexable from {0}", di.Indexable.DisplayUri);
}
if (!next_indexable)
{
// Mark it for indexing and leave it in the stack
di.Indexable.LocalState ["HasNextIndexable"] = false;
next = true;
break;
}
if (generated_indexable == null)
{
continue;
}
Log.Debug("Adding generated indexable {0}", generated_indexable.DisplayUri);
// Mark this indexable
generated_indexable.LocalState ["GeneratedIndexable"] = true;
// IndexerGenerated indexables have a common parenturi, which has been used before
// to remove all docs from the lucene index with that parenturi. So, now we can safely
// go ahead and just add the new information.
r = AddIndexableToIndex(generated_indexable, primary_writer, ref secondary_writer, di.PersistentPropDocs);
// But do not add r to the receipt queue, since this was generated
if (r != null) // null receipt is returned if generated_indexable is deferred
{
num_indexed++;
}
}
if (Shutdown.ShutdownRequested || num_indexed > RequestFlushThreshold)
{
break;
}
}
if (text_cache != null)
{
text_cache.CommitTransaction();
}
if (Shutdown.ShutdownRequested)
{
foreach (DeferredInfo di in deferred_indexables)
{
di.Cleanup();
}
deferred_indexables.Clear();
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
return(null);
}
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
// Send a single IndexerIndexablesReceipt if there were deferred indexables
if (deferred_indexables.Count > 0)
{
Log.Debug("{0} more indexable-generating indexable remainding to index; asking daemon to schedule their indexing.", deferred_indexables.Count);
IndexerIndexablesReceipt paused_receipt = new IndexerIndexablesReceipt();
receipt_queue.Add(paused_receipt);
}
IndexerReceipt [] receipt_array;
receipt_array = new IndexerReceipt [receipt_queue.Count];
for (int i = 0; i < receipt_queue.Count; ++i)
{
receipt_array [i] = (IndexerReceipt)receipt_queue [i];
}
return(receipt_array);
}
private IndexerAddedReceipt AddIndexableToIndex(Indexable indexable,
IndexWriter primary_writer,
ref IndexWriter secondary_writer,
Hashtable prop_change_docs)
{
Filter filter = null;
if (FileFilterNotifier != null)
{
FileFilterNotifier(indexable.DisplayUri, indexable.ContentUri, null); // We don't know what filter yet.
}
// If we have content, try to find a filter
// we we can use to process the indexable
bool filter_content = false;
try {
filter_content = FilterFactory.FilterIndexable(indexable, (disable_textcache ? null : text_cache), out filter);
} catch { }
if (!filter_content)
{
indexable.NoContent = true;
filter = null;
}
if (FileFilterNotifier != null)
{
FileFilterNotifier(indexable.DisplayUri, indexable.ContentUri, filter); // Update with our filter
}
IndexerAddedReceipt r = new IndexerAddedReceipt(indexable.Id);
if (filter != null)
{
if (filter.HasGeneratedIndexable)
{
Log.Debug("{0} might generate indexables from {1}; deferring until later",
indexable.DisplayUri,
filter.GetType().ToString());
// This indexable can potentially generate indexables,
// so defer its indexing
DeferredInfo di;
di = new DeferredInfo(indexable, filter, prop_change_docs);
deferred_indexables.Insert(0, di);
// Since we are deferred, continue. Do not cleanup indexable or remove text-cache yet.
// FIXME: Make sure all indexable.Cleanup is called for all indexables if
// shutdown is signalled.
if (FileFilterNotifier != null)
{
FileFilterNotifier(null, null, null); // reset
}
// Return null to signal the indexable was deferred
return(null);
}
// Force the clean-up of temporary files, just in case.
// FIXME: I am not sure if the cleanup should happen now.
// What is the difference between filter.Cleanup and Indexable.Cleanup ?
filter.Cleanup();
r.FilterName = filter.GetType().ToString();
r.FilterVersion = filter.Version;
}
// If this indexables is not deferred, add it to the index.
if (indexable.DisplayUri != indexable.ContentUri)
{
Log.Debug("+{0} ({1})", indexable.DisplayUri, indexable.ContentUri);
}
else
{
Log.Debug("+{0}", indexable.DisplayUri);
}
Document persistent_prop_doc = null;
if (prop_change_docs != null)
{
persistent_prop_doc = (Document)prop_change_docs [indexable.Uri];
}
AddDocumentToIndex(indexable, persistent_prop_doc, primary_writer, ref secondary_writer);
if (FileFilterNotifier != null)
{
FileFilterNotifier(null, null, null); // reset
}
// Clean up any temporary files associated with filtering this indexable.
indexable.Cleanup();
// Remove any existing text cache for this item
if (disable_textcache && text_cache != null)
{
text_cache.Delete(indexable.Uri);
}
return(r);
}
private IndexerAddedReceipt AddIndexableToIndex (Indexable indexable,
IndexWriter primary_writer,
ref IndexWriter secondary_writer,
Hashtable prop_change_docs)
{
Filter filter = null;
if (FileFilterNotifier != null)
FileFilterNotifier (indexable.DisplayUri, indexable.ContentUri, null); // We don't know what filter yet.
// If we have content, try to find a filter
// we we can use to process the indexable
bool filter_content = false;
try {
filter_content = FilterFactory.FilterIndexable (indexable, (disable_textcache ? null : text_cache), out filter);
} catch { }
if (! filter_content) {
indexable.NoContent = true;
filter = null;
}
if (FileFilterNotifier != null)
FileFilterNotifier (indexable.DisplayUri, indexable.ContentUri, filter); // Update with our filter
IndexerAddedReceipt r = new IndexerAddedReceipt (indexable.Id);
if (filter != null) {
if (filter.HasGeneratedIndexable) {
Log.Debug ("{0} might generate indexables from {1}; deferring until later",
indexable.DisplayUri,
filter.GetType ().ToString ());
// This indexable can potentially generate indexables,
// so defer its indexing
DeferredInfo di;
di = new DeferredInfo (indexable, filter, prop_change_docs);
deferred_indexables.Insert (0, di);
// Since we are deferred, continue. Do not cleanup indexable or remove text-cache yet.
// FIXME: Make sure all indexable.Cleanup is called for all indexables if
// shutdown is signalled.
if (FileFilterNotifier != null)
FileFilterNotifier (null, null, null); // reset
// Return null to signal the indexable was deferred
return null;
}
// Force the clean-up of temporary files, just in case.
// FIXME: I am not sure if the cleanup should happen now.
// What is the difference between filter.Cleanup and Indexable.Cleanup ?
filter.Cleanup ();
r.FilterName = filter.GetType ().ToString ();
r.FilterVersion = filter.Version;
}
// If this indexables is not deferred, add it to the index.
if (indexable.DisplayUri != indexable.ContentUri)
Log.Debug ("+{0} ({1})", indexable.DisplayUri, indexable.ContentUri);
else
Log.Debug ("+{0}", indexable.DisplayUri);
Document persistent_prop_doc = null;
if (prop_change_docs != null)
persistent_prop_doc = (Document) prop_change_docs [indexable.Uri];
AddDocumentToIndex (indexable, persistent_prop_doc, primary_writer, ref secondary_writer);
if (FileFilterNotifier != null)
FileFilterNotifier (null, null, null); // reset
// Clean up any temporary files associated with filtering this indexable.
indexable.Cleanup ();
// Remove any existing text cache for this item
if (disable_textcache && text_cache != null)
text_cache.Delete (indexable.Uri);
return r;
}
