private IndexerReceipt [] FlushGeneratedIndexables_Unlocked(IndexerRequest request) { int num_indexed = 0; ArrayList receipt_queue; receipt_queue = new ArrayList(); if (text_cache != null) { text_cache.BeginTransaction(); } IndexWriter primary_writer, secondary_writer; primary_writer = new IndexWriter(PrimaryStore, IndexingAnalyzer, false); secondary_writer = null; IndexerAddedReceipt r; Log.Debug("Continuing indexing generated indexables from {0} indexables", deferred_indexables.Count); // Access using index so that we can add more deferred_indexable at the front // deferred_indexables are added at the front and fetched from the front like a stack while (deferred_indexables.Count > 0) { DeferredInfo di = (DeferredInfo)deferred_indexables [0]; if (di.Indexable.LocalState ["HasNextIndexable"] != null) { // Finally, good to index // Should we do a sanity check ? deferred_indexables [0] =?= di deferred_indexables.RemoveAt(0); Document persistent_prop_doc = null; if (di.PersistentPropDocs != null) { persistent_prop_doc = (Document)di.PersistentPropDocs [di.Indexable.Uri]; } if (di.Indexable.DisplayUri != di.Indexable.ContentUri) { Log.Debug("+{0} ({1}) [deferred]", di.Indexable.DisplayUri, di.Indexable.ContentUri); } else { Log.Debug("+{0} [deferred]", di.Indexable.DisplayUri); } AddDocumentToIndex(di.Indexable, persistent_prop_doc, primary_writer, ref secondary_writer); // Add the receipt if the indexable was submitted and not generated if (di.Indexable.LocalState ["GeneratedIndexable"] == null) { r = new IndexerAddedReceipt(di.Indexable.Id); r.FilterName = di.Filter.GetType().ToString(); r.FilterVersion = di.Filter.Version; receipt_queue.Add(r); } // Cleanup, and text cache maintenance. di.Cleanup(); if (disable_textcache && text_cache != null) { text_cache.Delete(di.Indexable.Uri); } num_indexed++; continue; } Log.Debug("Processing deferred indexable from {0}", di.Indexable.DisplayUri); bool next = false; while (!next && !Shutdown.ShutdownRequested && num_indexed <= RequestFlushThreshold) { Indexable generated_indexable = null; bool next_indexable = false; try { next_indexable = di.Filter.GenerateNextIndexable(out generated_indexable); } catch (Exception e) { Log.Error(e, "Error while generating next indexable from {0}", di.Indexable.DisplayUri); } if (!next_indexable) { // Mark it for indexing and leave it in the stack di.Indexable.LocalState ["HasNextIndexable"] = false; next = true; break; } if (generated_indexable == null) { continue; } Log.Debug("Adding generated indexable {0}", generated_indexable.DisplayUri); // Mark this indexable generated_indexable.LocalState ["GeneratedIndexable"] = true; // IndexerGenerated indexables have a common parenturi, which has been used before // to remove all docs from the lucene index with that parenturi. So, now we can safely // go ahead and just add the new information. r = AddIndexableToIndex(generated_indexable, primary_writer, ref secondary_writer, di.PersistentPropDocs); // But do not add r to the receipt queue, since this was generated if (r != null) // null receipt is returned if generated_indexable is deferred { num_indexed++; } } if (Shutdown.ShutdownRequested || num_indexed > RequestFlushThreshold) { break; } } if (text_cache != null) { text_cache.CommitTransaction(); } if (Shutdown.ShutdownRequested) { foreach (DeferredInfo di in deferred_indexables) { di.Cleanup(); } deferred_indexables.Clear(); primary_writer.Close(); if (secondary_writer != null) { secondary_writer.Close(); } return(null); } primary_writer.Close(); if (secondary_writer != null) { secondary_writer.Close(); } // Send a single IndexerIndexablesReceipt if there were deferred indexables if (deferred_indexables.Count > 0) { Log.Debug("{0} more indexable-generating indexable remainding to index; asking daemon to schedule their indexing.", deferred_indexables.Count); IndexerIndexablesReceipt paused_receipt = new IndexerIndexablesReceipt(); receipt_queue.Add(paused_receipt); } IndexerReceipt [] receipt_array; receipt_array = new IndexerReceipt [receipt_queue.Count]; for (int i = 0; i < receipt_queue.Count; ++i) { receipt_array [i] = (IndexerReceipt)receipt_queue [i]; } return(receipt_array); }
private IndexerAddedReceipt AddIndexableToIndex(Indexable indexable, IndexWriter primary_writer, ref IndexWriter secondary_writer, Hashtable prop_change_docs) { Filter filter = null; if (FileFilterNotifier != null) { FileFilterNotifier(indexable.DisplayUri, indexable.ContentUri, null); // We don't know what filter yet. } // If we have content, try to find a filter // we we can use to process the indexable bool filter_content = false; try { filter_content = FilterFactory.FilterIndexable(indexable, (disable_textcache ? null : text_cache), out filter); } catch { } if (!filter_content) { indexable.NoContent = true; filter = null; } if (FileFilterNotifier != null) { FileFilterNotifier(indexable.DisplayUri, indexable.ContentUri, filter); // Update with our filter } IndexerAddedReceipt r = new IndexerAddedReceipt(indexable.Id); if (filter != null) { if (filter.HasGeneratedIndexable) { Log.Debug("{0} might generate indexables from {1}; deferring until later", indexable.DisplayUri, filter.GetType().ToString()); // This indexable can potentially generate indexables, // so defer its indexing DeferredInfo di; di = new DeferredInfo(indexable, filter, prop_change_docs); deferred_indexables.Insert(0, di); // Since we are deferred, continue. Do not cleanup indexable or remove text-cache yet. // FIXME: Make sure all indexable.Cleanup is called for all indexables if // shutdown is signalled. if (FileFilterNotifier != null) { FileFilterNotifier(null, null, null); // reset } // Return null to signal the indexable was deferred return(null); } // Force the clean-up of temporary files, just in case. // FIXME: I am not sure if the cleanup should happen now. // What is the difference between filter.Cleanup and Indexable.Cleanup ? filter.Cleanup(); r.FilterName = filter.GetType().ToString(); r.FilterVersion = filter.Version; } // If this indexables is not deferred, add it to the index. if (indexable.DisplayUri != indexable.ContentUri) { Log.Debug("+{0} ({1})", indexable.DisplayUri, indexable.ContentUri); } else { Log.Debug("+{0}", indexable.DisplayUri); } Document persistent_prop_doc = null; if (prop_change_docs != null) { persistent_prop_doc = (Document)prop_change_docs [indexable.Uri]; } AddDocumentToIndex(indexable, persistent_prop_doc, primary_writer, ref secondary_writer); if (FileFilterNotifier != null) { FileFilterNotifier(null, null, null); // reset } // Clean up any temporary files associated with filtering this indexable. indexable.Cleanup(); // Remove any existing text cache for this item if (disable_textcache && text_cache != null) { text_cache.Delete(indexable.Uri); } return(r); }
private IndexerAddedReceipt AddIndexableToIndex (Indexable indexable, IndexWriter primary_writer, ref IndexWriter secondary_writer, Hashtable prop_change_docs) { Filter filter = null; if (FileFilterNotifier != null) FileFilterNotifier (indexable.DisplayUri, indexable.ContentUri, null); // We don't know what filter yet. // If we have content, try to find a filter // we we can use to process the indexable bool filter_content = false; try { filter_content = FilterFactory.FilterIndexable (indexable, (disable_textcache ? null : text_cache), out filter); } catch { } if (! filter_content) { indexable.NoContent = true; filter = null; } if (FileFilterNotifier != null) FileFilterNotifier (indexable.DisplayUri, indexable.ContentUri, filter); // Update with our filter IndexerAddedReceipt r = new IndexerAddedReceipt (indexable.Id); if (filter != null) { if (filter.HasGeneratedIndexable) { Log.Debug ("{0} might generate indexables from {1}; deferring until later", indexable.DisplayUri, filter.GetType ().ToString ()); // This indexable can potentially generate indexables, // so defer its indexing DeferredInfo di; di = new DeferredInfo (indexable, filter, prop_change_docs); deferred_indexables.Insert (0, di); // Since we are deferred, continue. Do not cleanup indexable or remove text-cache yet. // FIXME: Make sure all indexable.Cleanup is called for all indexables if // shutdown is signalled. if (FileFilterNotifier != null) FileFilterNotifier (null, null, null); // reset // Return null to signal the indexable was deferred return null; } // Force the clean-up of temporary files, just in case. // FIXME: I am not sure if the cleanup should happen now. // What is the difference between filter.Cleanup and Indexable.Cleanup ? filter.Cleanup (); r.FilterName = filter.GetType ().ToString (); r.FilterVersion = filter.Version; } // If this indexables is not deferred, add it to the index. if (indexable.DisplayUri != indexable.ContentUri) Log.Debug ("+{0} ({1})", indexable.DisplayUri, indexable.ContentUri); else Log.Debug ("+{0}", indexable.DisplayUri); Document persistent_prop_doc = null; if (prop_change_docs != null) persistent_prop_doc = (Document) prop_change_docs [indexable.Uri]; AddDocumentToIndex (indexable, persistent_prop_doc, primary_writer, ref secondary_writer); if (FileFilterNotifier != null) FileFilterNotifier (null, null, null); // reset // Clean up any temporary files associated with filtering this indexable. indexable.Cleanup (); // Remove any existing text cache for this item if (disable_textcache && text_cache != null) text_cache.Delete (indexable.Uri); return r; }