public IndexerReceipt [] Flush (IndexerRequest request)
{
// If there isn't actually any work to do, just return
// an empty array.
if (request.IsEmpty)
return new IndexerReceipt [0];
// Iterate through the items in the IndexerRequest to
// store the streams before passing them to the helper.
foreach (Indexable indexable in request.Indexables) {
if (indexable.Type == IndexableType.Add)
indexable.StoreStream ();
}
RemoteIndexerRequest remote_request;
remote_request = new RemoteIndexerRequest ();
remote_request.RemoteIndexName = this.remote_index_name;
remote_request.RemoteIndexMinorVersion = this.remote_index_minor_version;
remote_request.Request = request;
RemoteIndexerResponse response;
response = SendRequest (remote_request);
if (response == null) {
Logger.Log.Error ("Something terrible happened --- Flush failed");
request.Cleanup ();
return null;
}
last_item_count = response.ItemCount;
return response.Receipts;
}
public IndexerReceipt [] Flush(IndexerRequest request)
{
// If there isn't actually any work to do, just return
// an empty array.
if (request.IsEmpty)
{
return(new IndexerReceipt [0]);
}
// Iterate through the items in the IndexerRequest to
// store the streams before passing them to the helper.
foreach (Indexable indexable in request.Indexables)
{
if (indexable.Type == IndexableType.Add)
{
indexable.StoreStream();
}
}
RemoteIndexerRequest remote_request;
remote_request = new RemoteIndexerRequest();
remote_request.RemoteIndexName = this.remote_index_name;
remote_request.RemoteIndexMinorVersion = this.remote_index_minor_version;
remote_request.Request = request;
RemoteIndexerResponse response;
response = SendRequest(remote_request);
if (response == null)
{
Logger.Log.Error("Something terrible happened --- Flush failed");
request.Cleanup();
return(null);
}
last_item_count = response.ItemCount;
return(response.Receipts);
}
static void Main (string[] args)
{
if (args.Length != 2) {
Console.WriteLine ("Usage: beagle-master-delete-button index-name uri-to-delete");
return;
}
string index_name = args [0];
LuceneQueryingDriver driver = new LuceneQueryingDriver (index_name, -1, true);
Uri uri = new Uri (args [1], false);
Uri uri_to_delete = RemapUri (driver, uri);
LuceneIndexingDriver indexer = new LuceneIndexingDriver (index_name, false);
Indexable indexable = new Indexable (uri_to_delete);
indexable.Type = IndexableType.Remove;
IndexerRequest request = new IndexerRequest ();
request.Add (indexable);
IndexerReceipt [] receipts = indexer.Flush (request);
if (receipts == null || receipts.Length == 0) {
Console.WriteLine ("Uri {0} not found in {1}",
uri, index_name);
return;
}
IndexerRemovedReceipt r = receipts [0] as IndexerRemovedReceipt;
if (r == null || r.NumRemoved == 0) {
Console.WriteLine ("Uri {0} not found in {1}",
uri, index_name);
return;
}
Console.WriteLine ("Uri {0} deleted", uri);
}
virtual protected void PreFlushHook(IndexerRequest flushed_request)
{
}
private IndexerReceipt [] FlushGeneratedIndexables_Unlocked(IndexerRequest request)
{
int num_indexed = 0;
ArrayList receipt_queue;
receipt_queue = new ArrayList();
if (text_cache != null)
{
text_cache.BeginTransaction();
}
IndexWriter primary_writer, secondary_writer;
primary_writer = new IndexWriter(PrimaryStore, IndexingAnalyzer, false);
secondary_writer = null;
IndexerAddedReceipt r;
Log.Debug("Continuing indexing generated indexables from {0} indexables", deferred_indexables.Count);
// Access using index so that we can add more deferred_indexable at the front
// deferred_indexables are added at the front and fetched from the front like a stack
while (deferred_indexables.Count > 0)
{
DeferredInfo di = (DeferredInfo)deferred_indexables [0];
if (di.Indexable.LocalState ["HasNextIndexable"] != null)
{
// Finally, good to index
// Should we do a sanity check ? deferred_indexables [0] =?= di
deferred_indexables.RemoveAt(0);
Document persistent_prop_doc = null;
if (di.PersistentPropDocs != null)
{
persistent_prop_doc = (Document)di.PersistentPropDocs [di.Indexable.Uri];
}
if (di.Indexable.DisplayUri != di.Indexable.ContentUri)
{
Log.Debug("+{0} ({1}) [deferred]", di.Indexable.DisplayUri, di.Indexable.ContentUri);
}
else
{
Log.Debug("+{0} [deferred]", di.Indexable.DisplayUri);
}
AddDocumentToIndex(di.Indexable, persistent_prop_doc, primary_writer, ref secondary_writer);
// Add the receipt if the indexable was submitted and not generated
if (di.Indexable.LocalState ["GeneratedIndexable"] == null)
{
r = new IndexerAddedReceipt(di.Indexable.Id);
r.FilterName = di.Filter.GetType().ToString();
r.FilterVersion = di.Filter.Version;
receipt_queue.Add(r);
}
// Cleanup, and text cache maintenance.
di.Cleanup();
if (disable_textcache && text_cache != null)
{
text_cache.Delete(di.Indexable.Uri);
}
num_indexed++;
continue;
}
Log.Debug("Processing deferred indexable from {0}", di.Indexable.DisplayUri);
bool next = false;
while (!next && !Shutdown.ShutdownRequested && num_indexed <= RequestFlushThreshold)
{
Indexable generated_indexable = null;
bool next_indexable = false;
try {
next_indexable = di.Filter.GenerateNextIndexable(out generated_indexable);
} catch (Exception e) {
Log.Error(e, "Error while generating next indexable from {0}", di.Indexable.DisplayUri);
}
if (!next_indexable)
{
// Mark it for indexing and leave it in the stack
di.Indexable.LocalState ["HasNextIndexable"] = false;
next = true;
break;
}
if (generated_indexable == null)
{
continue;
}
Log.Debug("Adding generated indexable {0}", generated_indexable.DisplayUri);
// Mark this indexable
generated_indexable.LocalState ["GeneratedIndexable"] = true;
// IndexerGenerated indexables have a common parenturi, which has been used before
// to remove all docs from the lucene index with that parenturi. So, now we can safely
// go ahead and just add the new information.
r = AddIndexableToIndex(generated_indexable, primary_writer, ref secondary_writer, di.PersistentPropDocs);
// But do not add r to the receipt queue, since this was generated
if (r != null) // null receipt is returned if generated_indexable is deferred
{
num_indexed++;
}
}
if (Shutdown.ShutdownRequested || num_indexed > RequestFlushThreshold)
{
break;
}
}
if (text_cache != null)
{
text_cache.CommitTransaction();
}
if (Shutdown.ShutdownRequested)
{
foreach (DeferredInfo di in deferred_indexables)
{
di.Cleanup();
}
deferred_indexables.Clear();
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
return(null);
}
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
// Send a single IndexerIndexablesReceipt if there were deferred indexables
if (deferred_indexables.Count > 0)
{
Log.Debug("{0} more indexable-generating indexable remainding to index; asking daemon to schedule their indexing.", deferred_indexables.Count);
IndexerIndexablesReceipt paused_receipt = new IndexerIndexablesReceipt();
receipt_queue.Add(paused_receipt);
}
IndexerReceipt [] receipt_array;
receipt_array = new IndexerReceipt [receipt_queue.Count];
for (int i = 0; i < receipt_queue.Count; ++i)
{
receipt_array [i] = (IndexerReceipt)receipt_queue [i];
}
return(receipt_array);
}
private IndexerReceipt [] Flush_Unlocked(IndexerRequest request)
{
ArrayList receipt_queue;
receipt_queue = new ArrayList();
IndexReader primary_reader, secondary_reader;
primary_reader = IndexReader.Open(PrimaryStore);
secondary_reader = IndexReader.Open(SecondaryStore);
// Step #1: Make our first pass over the list of
// indexables that make up our request. For each add
// or property change in the request, get the Lucene
// documents so we can move forward any persistent
// properties (for adds) or all old properties (for
// property changes).
//
// Then, for each add or remove in the request,
// delete the associated documents from the index.
// Note that we previously cached added documents so
// that we can move persistent properties forward.
// parent_child_old_props is double-nested hashtable (depth-2 tree)
// indexed by the parent uri, it stores another hashtable indexed by the (parent+child documents)
// FIXME: 2-level hashtable is a waste for any non-child document.
// Replace this by a better data structure.
Hashtable parent_child_old_props = UriFu.NewHashtable();
TermDocs term_docs = secondary_reader.TermDocs();
int delete_count = 0;
IEnumerable request_indexables = request.Indexables;
foreach (Indexable indexable in request_indexables)
{
string uri_str = UriFu.UriToEscapedString(indexable.Uri);
Term term;
// Store the necessary properties from old documents for re-addition
if (indexable.Type == IndexableType.Add ||
indexable.Type == IndexableType.PropertyChange)
{
term = new Term("Uri", uri_str);
term_docs.Seek(term);
Hashtable this_parent_child_props = null;
if (term_docs.Next())
{
this_parent_child_props = UriFu.NewHashtable();
this_parent_child_props [indexable.Uri] = secondary_reader.Document(term_docs.Doc());
parent_child_old_props [indexable.Uri] = this_parent_child_props;
}
term = new Term("ParentUri", uri_str);
term_docs.Seek(term);
while (term_docs.Next())
{
Document doc = secondary_reader.Document(term_docs.Doc());
string child_uri_str = doc.Get("Uri");
Uri child_uri = UriFu.EscapedStringToUri(child_uri_str);
// Any valid lucene document *should* have a Uri, so no need to check for null
// Store the child documents too, to save persistent-properties
// of child documents
this_parent_child_props [child_uri] = doc;
}
}
// Now remove (non-remove indexables will be re-added in next block)
Logger.Log.Debug("-{0}", indexable.DisplayUri);
int num_delete = 0;
term = new Term("Uri", uri_str);
// For property changes, only secondary index is modified
secondary_reader.DeleteDocuments(term);
// Now remove from everywhere else (if asked to remove or if asked to add, in which case
// we first remove and then add)
// So we also need to remove child documents
if (indexable.Type != IndexableType.PropertyChange)
{
num_delete = primary_reader.DeleteDocuments(term);
// When we delete an indexable, also delete any children.
// FIXME: Shouldn't we also delete any children of children, etc.?
term = new Term("ParentUri", uri_str);
num_delete += primary_reader.DeleteDocuments(term);
secondary_reader.DeleteDocuments(term);
}
// If this is a strict removal (and not a deletion that
// we are doing in anticipation of adding something back),
// queue up a removed receipt.
if (indexable.Type == IndexableType.Remove)
{
IndexerRemovedReceipt r;
r = new IndexerRemovedReceipt(indexable.Id);
r.NumRemoved = num_delete;
receipt_queue.Add(r);
}
delete_count += num_delete;
}
term_docs.Close();
if (HaveItemCount)
{
AdjustItemCount(-delete_count);
}
else
{
SetItemCount(primary_reader);
}
// We are now done with the readers, so we close them.
// And also free them. Somehow not freeing them is preventing them from
// GCed at all.
primary_reader.Close();
primary_reader = null;
secondary_reader.Close();
secondary_reader = null;
// FIXME: If we crash at exactly this point, we are in
// trouble. Items will have been dropped from the index
// without the proper replacements being added. We can
// hopefully fix this when we move to Lucene 2.1.
// Step #2: Make another pass across our list of indexables
// and write out any new documents.
if (text_cache != null)
{
text_cache.BeginTransaction();
}
IndexWriter primary_writer, secondary_writer;
// FIXME: Lock obtain time-out can happen here; if that happens,
// an exception will be thrown and this method will break in the middle
// leaving IndexWriters unclosed! Same for any Lucene.Net-index modification
// methods.
primary_writer = new IndexWriter(PrimaryStore, IndexingAnalyzer, false);
secondary_writer = null;
foreach (Indexable indexable in request_indexables)
{
// If shutdown has been started, break here
// FIXME: Some more processing will continue, a lot of them
// concerning receipts, but the daemon will anyway ignore receipts
// now, what is the fastest way to stop from here ?
if (Shutdown.ShutdownRequested)
{
Log.Debug("Shutdown initiated. Breaking while flushing indexables.");
break;
}
// Receipts for removes were generated in the
// previous block. Now we just have to remove
// items from the text cache.
if (indexable.Type == IndexableType.Remove)
{
if (text_cache != null)
{
text_cache.Delete(indexable.Uri);
}
continue;
}
IndexerAddedReceipt r;
Hashtable prop_change_docs = (Hashtable)parent_child_old_props [indexable.Uri];
if (indexable.Type == IndexableType.PropertyChange)
{
Logger.Log.Debug("+{0} (props only)", indexable.DisplayUri);
r = new IndexerAddedReceipt(indexable.Id);
r.PropertyChangesOnly = true;
receipt_queue.Add(r);
Document doc;
if (prop_change_docs == null)
{
doc = null;
}
else
{
doc = (Document)prop_change_docs [indexable.Uri];
}
Document new_doc;
new_doc = RewriteDocument(doc, indexable);
// Write out the new document...
if (secondary_writer == null)
{
secondary_writer = new IndexWriter(SecondaryStore, IndexingAnalyzer, false);
}
secondary_writer.AddDocument(new_doc);
// Get child property change indexables...
ArrayList prop_change_indexables;
prop_change_indexables = GetChildPropertyChange(prop_change_docs, indexable);
// and store them; no need to delete them first, since they were already removed from the index
if (prop_change_indexables == null)
{
continue;
}
foreach (Indexable prop_change_indexable in prop_change_indexables)
{
Log.Debug("+{0} (props only, generated indexable)", prop_change_indexable.Uri);
doc = (Document)prop_change_docs [prop_change_indexable.Uri];
new_doc = RewriteDocument(doc, prop_change_indexable);
secondary_writer.AddDocument(new_doc);
}
continue; // ...and proceed to the next Indexable
}
// If we reach this point we know we are dealing with an IndexableType.Add
if (indexable.Type != IndexableType.Add)
{
throw new Exception("When I said it was an IndexableType.Add, I meant it!");
}
r = AddIndexableToIndex(indexable, primary_writer, ref secondary_writer, prop_change_docs);
if (r != null)
{
receipt_queue.Add(r);
}
}
if (text_cache != null)
{
text_cache.CommitTransaction();
}
if (Shutdown.ShutdownRequested)
{
foreach (DeferredInfo di in deferred_indexables)
{
di.Cleanup();
}
deferred_indexables.Clear();
foreach (Indexable indexable in request_indexables)
{
indexable.Cleanup();
}
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
return(null);
}
if (request.OptimizeIndex)
{
Stopwatch watch = new Stopwatch();
Logger.Log.Debug("Optimizing {0}", IndexName);
watch.Start();
primary_writer.Optimize();
if (secondary_writer == null)
{
secondary_writer = new IndexWriter(SecondaryStore, IndexingAnalyzer, false);
}
secondary_writer.Optimize();
watch.Stop();
Logger.Log.Debug("{0} optimized in {1}", IndexName, watch);
}
// Step #4. Close our writers and return the events to
// indicate what has happened.
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
// Send a single IndexerIndexablesReceipt if there were deferred indexables
if (deferred_indexables.Count > 0)
{
Log.Debug("{0} indexables generated more indexables; asking daemon to schedule their indexing.", deferred_indexables.Count);
IndexerIndexablesReceipt r = new IndexerIndexablesReceipt();
receipt_queue.Add(r);
}
IndexerReceipt [] receipt_array;
receipt_array = new IndexerReceipt [receipt_queue.Count];
for (int i = 0; i < receipt_queue.Count; ++i)
{
receipt_array [i] = (IndexerReceipt)receipt_queue [i];
}
return(receipt_array);
}
// This is mostly a copy of LuceneQueryable.Flush + FSQ.PostAddHooks/PostRemoveHook
static bool FlushIndexer (IIndexer indexer)
{
IndexerRequest flushed_request;
if (pending_request.IsEmpty)
return false;
flushed_request = pending_request;
pending_request = new IndexerRequest ();
IndexerReceipt [] receipts;
receipts = indexer.Flush (flushed_request);
// Flush will return null if it encounters a shutdown during flushing
if (receipts == null)
return false;
fa_store.BeginTransaction ();
bool indexer_indexable_receipt = false;
foreach (IndexerReceipt raw_r in receipts) {
if (raw_r is IndexerAddedReceipt) {
// Update the file attributes
IndexerAddedReceipt r = (IndexerAddedReceipt) raw_r;
Indexable indexable = flushed_request.RetrieveRequestIndexable (r);
if (indexable == null) {
Logger.Log.Debug ("Should not happen! Previously requested indexable with id #{0} has eloped!", r.Id);
continue;
}
// We don't need to write out any file attributes for
// children.
if (indexable.ParentUri != null)
continue;
string path = indexable.Uri.LocalPath;
FileAttributes attr;
attr = fa_store.ReadOrCreate (path);
attr.LastWriteTime = indexable.Timestamp;
attr.FilterName = r.FilterName;
attr.FilterVersion = r.FilterVersion;
fa_store.Write (attr);
} else if (raw_r is IndexerRemovedReceipt) {
// Update the file attributes
IndexerRemovedReceipt r = (IndexerRemovedReceipt) raw_r;
Indexable indexable = flushed_request.RetrieveRequestIndexable (r);
if (indexable == null) { // Should never happen
Log.Warn ("Unable to match indexable-remove #{0} to any request!", r.Id);
continue;
}
string path = indexable.Uri.LocalPath;
Logger.Log.Debug ("Removing: '{0}'", path);
fa_store.Drop (path);
} else if (raw_r is IndexerIndexablesReceipt) {
indexer_indexable_receipt = true;
}
}
pending_request.DeferredIndexables = flushed_request.DeferredIndexables;
// Reschedule if some indexable generated more indexables
if (indexer_indexable_receipt) {
pending_request.ContinueIndexing = true;
return true;
}
fa_store.CommitTransaction ();
return false;
}
static void DoIndexing ()
{
int count_dirs = 0;
int count_files = 0;
Indexable indexable;
pending_request = new IndexerRequest ();
Queue modified_directories = new Queue ();
while (pending_directories.Count > 0) {
DirectoryInfo dir = (DirectoryInfo) pending_directories.Dequeue ();
AddToRequest (DirectoryToIndexable (dir, modified_directories));
try {
if (arg_recursive)
foreach (DirectoryInfo subdir in DirectoryWalker.GetDirectoryInfos (dir))
if (!Ignore (subdir)
&& !FileSystem.IsSpecialFile (subdir.FullName))
pending_directories.Enqueue (subdir);
foreach (FileInfo file in DirectoryWalker.GetFileInfos (dir))
if (!Ignore (file)
&& !FileSystem.IsSpecialFile (file.FullName)) {
AddToRequest (FileToIndexable (file));
count_files ++;
}
} catch (DirectoryNotFoundException) {}
if (Shutdown.ShutdownRequested)
break;
count_dirs++;
}
Logger.Log.Debug ("Scanned {0} files and directories in {1} directories", count_dirs + count_files, count_dirs);
if (Shutdown.ShutdownRequested) {
backing_fa_store.Flush ();
return;
}
// Time to remove deleted directories from the index and attributes store
while (modified_directories.Count > 0) {
DirectoryInfo subdir = (DirectoryInfo) modified_directories.Dequeue ();
Logger.Log.Debug ("Checking {0} for deleted files and directories", subdir.FullName);
// Get a list of all documents from lucene index with ParentDirUriPropKey set as that of subdir
ICollection all_dirent = GetAllItemsInDirectory (subdir);
foreach (Dirent info in all_dirent) {
// check if the item exists
if ((! info.IsDirectory && File.Exists (info.FullName)) ||
(info.IsDirectory && Directory.Exists (info.FullName)))
continue;
if (info.IsDirectory)
// Recursively remove deleted subdirectories
modified_directories.Enqueue (new DirectoryInfo (info.FullName));
// remove
Uri uri = PathToUri (info.FullName);
indexable = new Indexable (IndexableType.Remove, uri);
AddToRequest (indexable);
}
}
bool reschedule = false;
// Call Flush until our request is empty. We have to do this in a loop
// because Flush happens in a batch size and some indexables might generate more indexables
while (reschedule || pending_request.Count > 0) {
if (Shutdown.ShutdownRequested)
break;
reschedule = FlushIndexer (driver);
}
backing_fa_store.Flush ();
if (Shutdown.ShutdownRequested)
return;
Logger.Log.Debug ("Optimizing index");
driver.OptimizeNow ();
}
virtual protected void PostFlushHook (IndexerRequest flushed_request,
IndexerReceipt [] receipts)
{ }
static void DoIndexing()
{
int count_dirs = 0;
int count_files = 0;
Indexable indexable;
pending_request = new IndexerRequest();
Queue modified_directories = new Queue();
while (pending_directories.Count > 0)
{
DirectoryInfo dir = (DirectoryInfo)pending_directories.Dequeue();
AddToRequest(DirectoryToIndexable(dir, modified_directories));
try {
if (arg_recursive)
{
foreach (DirectoryInfo subdir in DirectoryWalker.GetDirectoryInfos(dir))
{
if (!Ignore(subdir) &&
!FileSystem.IsSpecialFile(subdir.FullName))
{
pending_directories.Enqueue(subdir);
}
}
}
foreach (FileInfo file in DirectoryWalker.GetFileInfos(dir))
{
if (!Ignore(file) &&
!FileSystem.IsSpecialFile(file.FullName))
{
AddToRequest(FileToIndexable(file));
count_files++;
}
}
} catch (DirectoryNotFoundException) {}
if (Shutdown.ShutdownRequested)
{
break;
}
count_dirs++;
}
Logger.Log.Debug("Scanned {0} files and directories in {1} directories", count_dirs + count_files, count_dirs);
if (Shutdown.ShutdownRequested)
{
backing_fa_store.Flush();
return;
}
// Time to remove deleted directories from the index and attributes store
while (modified_directories.Count > 0)
{
DirectoryInfo subdir = (DirectoryInfo)modified_directories.Dequeue();
Logger.Log.Debug("Checking {0} for deleted files and directories", subdir.FullName);
// Get a list of all documents from lucene index with ParentDirUriPropKey set as that of subdir
ICollection all_dirent = GetAllItemsInDirectory(subdir);
foreach (Dirent info in all_dirent)
{
// check if the item exists
if ((!info.IsDirectory && File.Exists(info.FullName)) ||
(info.IsDirectory && Directory.Exists(info.FullName)))
{
continue;
}
if (info.IsDirectory)
{
// Recursively remove deleted subdirectories
modified_directories.Enqueue(new DirectoryInfo(info.FullName));
}
// remove
Uri uri = PathToUri(info.FullName);
indexable = new Indexable(IndexableType.Remove, uri);
AddToRequest(indexable);
}
}
bool reschedule = false;
// Call Flush until our request is empty. We have to do this in a loop
// because Flush happens in a batch size and some indexables might generate more indexables
while (reschedule || pending_request.Count > 0)
{
if (Shutdown.ShutdownRequested)
{
break;
}
reschedule = FlushIndexer(driver);
}
backing_fa_store.Flush();
if (Shutdown.ShutdownRequested)
{
return;
}
Logger.Log.Debug("Optimizing index");
driver.OptimizeNow();
}
private IndexerReceipt [] FlushGeneratedIndexables_Unlocked (IndexerRequest request)
{
int num_indexed = 0;
ArrayList receipt_queue;
receipt_queue = new ArrayList ();
if (text_cache != null)
text_cache.BeginTransaction ();
IndexWriter primary_writer, secondary_writer;
primary_writer = new IndexWriter (PrimaryStore, IndexingAnalyzer, false);
secondary_writer = null;
IndexerAddedReceipt r;
Log.Debug ("Continuing indexing generated indexables from {0} indexables", deferred_indexables.Count);
// Access using index so that we can add more deferred_indexable at the front
// deferred_indexables are added at the front and fetched from the front like a stack
while (deferred_indexables.Count > 0) {
DeferredInfo di = (DeferredInfo) deferred_indexables [0];
if (di.Indexable.LocalState ["HasNextIndexable"] != null) {
// Finally, good to index
// Should we do a sanity check ? deferred_indexables [0] =?= di
deferred_indexables.RemoveAt (0);
Document persistent_prop_doc = null;
if (di.PersistentPropDocs != null)
persistent_prop_doc = (Document) di.PersistentPropDocs [di.Indexable.Uri];
if (di.Indexable.DisplayUri != di.Indexable.ContentUri)
Log.Debug ("+{0} ({1}) [deferred]", di.Indexable.DisplayUri, di.Indexable.ContentUri);
else
Log.Debug ("+{0} [deferred]", di.Indexable.DisplayUri);
AddDocumentToIndex (di.Indexable, persistent_prop_doc, primary_writer, ref secondary_writer);
// Add the receipt if the indexable was submitted and not generated
if (di.Indexable.LocalState ["GeneratedIndexable"] == null) {
r = new IndexerAddedReceipt (di.Indexable.Id);
r.FilterName = di.Filter.GetType ().ToString ();
r.FilterVersion = di.Filter.Version;
receipt_queue.Add (r);
}
// Cleanup, and text cache maintenance.
di.Cleanup ();
if (disable_textcache && text_cache != null)
text_cache.Delete (di.Indexable.Uri);
num_indexed ++;
continue;
}
Log.Debug ("Processing deferred indexable from {0}", di.Indexable.DisplayUri);
bool next = false;
while (! next && ! Shutdown.ShutdownRequested && num_indexed <= RequestFlushThreshold) {
Indexable generated_indexable = null;
bool next_indexable = false;
try {
next_indexable = di.Filter.GenerateNextIndexable (out generated_indexable);
} catch (Exception e) {
Log.Error (e, "Error while generating next indexable from {0}", di.Indexable.DisplayUri);
}
if (! next_indexable) {
// Mark it for indexing and leave it in the stack
di.Indexable.LocalState ["HasNextIndexable"] = false;
next = true;
break;
}
if (generated_indexable == null)
continue;
Log.Debug ("Adding generated indexable {0}", generated_indexable.DisplayUri);
// Mark this indexable
generated_indexable.LocalState ["GeneratedIndexable"] = true;
// IndexerGenerated indexables have a common parenturi, which has been used before
// to remove all docs from the lucene index with that parenturi. So, now we can safely
// go ahead and just add the new information.
r = AddIndexableToIndex (generated_indexable, primary_writer, ref secondary_writer, di.PersistentPropDocs);
// But do not add r to the receipt queue, since this was generated
if (r != null) // null receipt is returned if generated_indexable is deferred
num_indexed ++;
}
if (Shutdown.ShutdownRequested || num_indexed > RequestFlushThreshold)
break;
}
if (text_cache != null)
text_cache.CommitTransaction ();
if (Shutdown.ShutdownRequested) {
foreach (DeferredInfo di in deferred_indexables)
di.Cleanup ();
deferred_indexables.Clear ();
primary_writer.Close ();
if (secondary_writer != null)
secondary_writer.Close ();
return null;
}
primary_writer.Close ();
if (secondary_writer != null)
secondary_writer.Close ();
// Send a single IndexerIndexablesReceipt if there were deferred indexables
if (deferred_indexables.Count > 0) {
Log.Debug ("{0} more indexable-generating indexable remainding to index; asking daemon to schedule their indexing.", deferred_indexables.Count);
IndexerIndexablesReceipt paused_receipt = new IndexerIndexablesReceipt ();
receipt_queue.Add (paused_receipt);
}
IndexerReceipt [] receipt_array;
receipt_array = new IndexerReceipt [receipt_queue.Count];
for (int i = 0; i < receipt_queue.Count; ++i)
receipt_array [i] = (IndexerReceipt) receipt_queue [i];
return receipt_array;
}
private IndexerReceipt [] Flush_Unlocked (IndexerRequest request)
{
ArrayList receipt_queue;
receipt_queue = new ArrayList ();
IndexReader primary_reader, secondary_reader;
primary_reader = IndexReader.Open (PrimaryStore);
secondary_reader = IndexReader.Open (SecondaryStore);
// Step #1: Make our first pass over the list of
// indexables that make up our request. For each add
// or property change in the request, get the Lucene
// documents so we can move forward any persistent
// properties (for adds) or all old properties (for
// property changes).
//
// Then, for each add or remove in the request,
// delete the associated documents from the index.
// Note that we previously cached added documents so
// that we can move persistent properties forward.
// parent_child_old_props is double-nested hashtable (depth-2 tree)
// indexed by the parent uri, it stores another hashtable indexed by the (parent+child documents)
// FIXME: 2-level hashtable is a waste for any non-child document.
// Replace this by a better data structure.
Hashtable parent_child_old_props = UriFu.NewHashtable ();
TermDocs term_docs = secondary_reader.TermDocs ();
int delete_count = 0;
IEnumerable request_indexables = request.Indexables;
foreach (Indexable indexable in request_indexables) {
string uri_str = UriFu.UriToEscapedString (indexable.Uri);
Term term;
// Store the necessary properties from old documents for re-addition
if (indexable.Type == IndexableType.Add ||
indexable.Type == IndexableType.PropertyChange) {
term = new Term ("Uri", uri_str);
term_docs.Seek (term);
Hashtable this_parent_child_props = null;
if (term_docs.Next ()) {
this_parent_child_props = UriFu.NewHashtable ();
this_parent_child_props [indexable.Uri] = secondary_reader.Document (term_docs.Doc ());
parent_child_old_props [indexable.Uri] = this_parent_child_props;
}
term = new Term ("ParentUri", uri_str);
term_docs.Seek (term);
while (term_docs.Next ()) {
Document doc = secondary_reader.Document (term_docs.Doc ());
string child_uri_str = doc.Get ("Uri");
Uri child_uri = UriFu.EscapedStringToUri (child_uri_str);
// Any valid lucene document *should* have a Uri, so no need to check for null
// Store the child documents too, to save persistent-properties
// of child documents
this_parent_child_props [child_uri] = doc;
}
}
// Now remove (non-remove indexables will be re-added in next block)
Logger.Log.Debug ("-{0}", indexable.DisplayUri);
int num_delete = 0;
term = new Term ("Uri", uri_str);
// For property changes, only secondary index is modified
secondary_reader.DeleteDocuments (term);
// Now remove from everywhere else (if asked to remove or if asked to add, in which case
// we first remove and then add)
// So we also need to remove child documents
if (indexable.Type != IndexableType.PropertyChange) {
num_delete = primary_reader.DeleteDocuments (term);
// When we delete an indexable, also delete any children.
// FIXME: Shouldn't we also delete any children of children, etc.?
term = new Term ("ParentUri", uri_str);
num_delete += primary_reader.DeleteDocuments (term);
secondary_reader.DeleteDocuments (term);
}
// If this is a strict removal (and not a deletion that
// we are doing in anticipation of adding something back),
// queue up a removed receipt.
if (indexable.Type == IndexableType.Remove) {
IndexerRemovedReceipt r;
r = new IndexerRemovedReceipt (indexable.Id);
r.NumRemoved = num_delete;
receipt_queue.Add (r);
}
delete_count += num_delete;
}
term_docs.Close ();
if (HaveItemCount)
AdjustItemCount (-delete_count);
else
SetItemCount (primary_reader);
// We are now done with the readers, so we close them.
// And also free them. Somehow not freeing them is preventing them from
// GCed at all.
primary_reader.Close ();
primary_reader = null;
secondary_reader.Close ();
secondary_reader = null;
// FIXME: If we crash at exactly this point, we are in
// trouble. Items will have been dropped from the index
// without the proper replacements being added. We can
// hopefully fix this when we move to Lucene 2.1.
// Step #2: Make another pass across our list of indexables
// and write out any new documents.
if (text_cache != null)
text_cache.BeginTransaction ();
IndexWriter primary_writer, secondary_writer;
// FIXME: Lock obtain time-out can happen here; if that happens,
// an exception will be thrown and this method will break in the middle
// leaving IndexWriters unclosed! Same for any Lucene.Net-index modification
// methods.
primary_writer = new IndexWriter (PrimaryStore, IndexingAnalyzer, false);
secondary_writer = null;
foreach (Indexable indexable in request_indexables) {
// If shutdown has been started, break here
// FIXME: Some more processing will continue, a lot of them
// concerning receipts, but the daemon will anyway ignore receipts
// now, what is the fastest way to stop from here ?
if (Shutdown.ShutdownRequested) {
Log.Debug ("Shutdown initiated. Breaking while flushing indexables.");
break;
}
// Receipts for removes were generated in the
// previous block. Now we just have to remove
// items from the text cache.
if (indexable.Type == IndexableType.Remove) {
if (text_cache != null)
text_cache.Delete (indexable.Uri);
continue;
}
IndexerAddedReceipt r;
Hashtable prop_change_docs = (Hashtable) parent_child_old_props [indexable.Uri];
if (indexable.Type == IndexableType.PropertyChange) {
Logger.Log.Debug ("+{0} (props only)", indexable.DisplayUri);
r = new IndexerAddedReceipt (indexable.Id);
r.PropertyChangesOnly = true;
receipt_queue.Add (r);
Document doc;
if (prop_change_docs == null)
doc = null;
else
doc = (Document) prop_change_docs [indexable.Uri];
Document new_doc;
new_doc = RewriteDocument (doc, indexable);
// Write out the new document...
if (secondary_writer == null)
secondary_writer = new IndexWriter (SecondaryStore, IndexingAnalyzer, false);
secondary_writer.AddDocument (new_doc);
// Get child property change indexables...
ArrayList prop_change_indexables;
prop_change_indexables = GetChildPropertyChange (prop_change_docs, indexable);
// and store them; no need to delete them first, since they were already removed from the index
if (prop_change_indexables == null)
continue;
foreach (Indexable prop_change_indexable in prop_change_indexables) {
Log.Debug ("+{0} (props only, generated indexable)", prop_change_indexable.Uri);
doc = (Document) prop_change_docs [prop_change_indexable.Uri];
new_doc = RewriteDocument (doc, prop_change_indexable);
secondary_writer.AddDocument (new_doc);
}
continue; // ...and proceed to the next Indexable
}
// If we reach this point we know we are dealing with an IndexableType.Add
if (indexable.Type != IndexableType.Add)
throw new Exception ("When I said it was an IndexableType.Add, I meant it!");
r = AddIndexableToIndex (indexable, primary_writer, ref secondary_writer, prop_change_docs);
if (r != null)
receipt_queue.Add (r);
}
if (text_cache != null)
text_cache.CommitTransaction ();
if (Shutdown.ShutdownRequested) {
foreach (DeferredInfo di in deferred_indexables)
di.Cleanup ();
deferred_indexables.Clear ();
foreach (Indexable indexable in request_indexables)
indexable.Cleanup ();
primary_writer.Close ();
if (secondary_writer != null)
secondary_writer.Close ();
return null;
}
if (request.OptimizeIndex) {
Stopwatch watch = new Stopwatch ();
Logger.Log.Debug ("Optimizing {0}", IndexName);
watch.Start ();
primary_writer.Optimize ();
if (secondary_writer == null)
secondary_writer = new IndexWriter (SecondaryStore, IndexingAnalyzer, false);
secondary_writer.Optimize ();
watch.Stop ();
Logger.Log.Debug ("{0} optimized in {1}", IndexName, watch);
}
// Step #4. Close our writers and return the events to
// indicate what has happened.
primary_writer.Close ();
if (secondary_writer != null)
secondary_writer.Close ();
// Send a single IndexerIndexablesReceipt if there were deferred indexables
if (deferred_indexables.Count > 0) {
Log.Debug ("{0} indexables generated more indexables; asking daemon to schedule their indexing.", deferred_indexables.Count);
IndexerIndexablesReceipt r = new IndexerIndexablesReceipt ();
receipt_queue.Add (r);
}
IndexerReceipt [] receipt_array;
receipt_array = new IndexerReceipt [receipt_queue.Count];
for (int i = 0; i < receipt_queue.Count; ++i)
receipt_array [i] = (IndexerReceipt) receipt_queue [i];
return receipt_array;
}
////////////////////////////////////////////////////////////////
//
// Implementation of the IIndexer interface
//
public IndexerReceipt [] Flush (IndexerRequest request)
{
// This is just to keep a big block of code from being
// indented an extra eight spaces.
lock (flush_lock) {
if (request.ContinueIndexing)
return FlushGeneratedIndexables_Unlocked (request);
else
return Flush_Unlocked (request);
}
}
protected void Flush (bool continue_only)
{
IndexerRequest flushed_request;
if (continue_only) {
// if the request is merely to signal indexhelper to continue indexing,
// then sent a fake indexerrequest but then use the previous request to retrieve
// deferred indexables
flushed_request = new IndexerRequest ();
flushed_request.ContinueIndexing = true;
// Do not pass this through PreFlushHook since this is a fake request
} else {
lock (request_lock) {
if (pending_request.IsEmpty)
return;
flushed_request = pending_request;
pending_request = new IndexerRequest ();
// We hold the request_lock when calling PreFlushHook, so
// that no other requests can come in until it exits.
PreFlushHook (flushed_request);
}
}
IndexerReceipt [] receipts;
receipts = indexer.Flush (flushed_request);
PostFlushHook (flushed_request, receipts);
if (continue_only) {
flushed_request = pending_request;
}
// Silently return if we get a null back. This is probably
// a bad thing to do. If IndexHelper is shutdown because of
// memory blowup or is crashed, then null is returned. Silently
// returning means ignoring the indexables in the IndexHelper's
// queue (which could be more than what was sent in the last request,
// since there could be some deferred-indexables too).
if (receipts == null)
return;
// Nothing happened (except maybe an optimize, which does not
// generate a receipt). Also do nothing.
if (receipts.Length == 0)
return;
// Update the cached count of items in the driver
// FIXME: Verify that this still works after all the deferred-indexable fu
driver.SetItemCount (indexer.GetItemCount ());
// Something happened, so schedule an optimize just in case.
ScheduleOptimize ();
if (fa_store != null)
fa_store.BeginTransaction ();
ArrayList added_uris = new ArrayList ();
ArrayList removed_uris = new ArrayList ();
bool indexer_indexable_receipt = false;
for (int i = 0; i < receipts.Length; ++i) {
if (receipts [i] is IndexerAddedReceipt) {
IndexerAddedReceipt r;
r = (IndexerAddedReceipt) receipts [i];
Indexable indexable = flushed_request.RetrieveRequestIndexable (r);
if (indexable == null) {
Log.Debug ("Should not happen! Previously requested indexable with id #{0} has eloped!", r.Id);
continue;
}
// Add the Uri to the list for our change data
// *before* doing any post-processing.
// This ensures that we have internal uris when
// we are remapping.
added_uris.Add (indexable.Uri);
// Call the appropriate hook
Uri notification_uri = indexable.Uri;
try {
// Map from internal->external Uris in the PostAddHook
notification_uri = PostAddHook (indexable, r);
} catch (Exception ex) {
Logger.Log.Warn (ex, "Caught exception in PostAddHook '{0}' '{1}' '{2}'",
indexable.Uri, r.FilterName, r.FilterVersion);
}
// Every added Uri also needs to be listed as removed,
// to avoid duplicate hits in the query. Since the
// removed Uris need to be external Uris, we add them
// to the list *after* post-processing.
removed_uris.Add (notification_uri);
} else if (receipts [i] is IndexerRemovedReceipt) {
IndexerRemovedReceipt r;
r = (IndexerRemovedReceipt) receipts [i];
Indexable indexable = flushed_request.RetrieveRequestIndexable (r);
if (indexable == null) { // Should never happen
Log.Warn ("Unable to match indexable-remove #{0} to any request!", r.Id);
continue;
}
// Call the appropriate hook
Uri notification_uri = indexable.Uri;
try {
notification_uri = PostRemoveHook (indexable, r.NumRemoved);
} catch (Exception ex) {
Logger.Log.Warn (ex, "Caught exception in PostRemoveHook '{0}'",
indexable.Uri);
}
// If nothing was removed, no need for change notification
if (r.NumRemoved <= 0)
continue;
// Add the removed Uri to the list for our
// change data. This will be an external Uri
// when we are remapping.
removed_uris.Add (notification_uri);
} else if (receipts [i] is IndexerIndexablesReceipt) {
indexer_indexable_receipt = true;
}
}
if (! continue_only) {
lock (request_lock) {
pending_request.DeferredIndexables = flushed_request.DeferredIndexables;
}
}
if (indexer_indexable_receipt) {
Log.Debug ("Indexing of indexer generated indexables is paused. Scheduling job to continue.");
// Create a task asking the indexer to continue indexing
Scheduler.Task task;
task = Scheduler.TaskFromHook (new Scheduler.TaskHook (ContinueIndexerIndexableIndexing));
// Schedule it so that it is the immediate next task to be scheduled
task.Priority = Scheduler.Priority.Immediate;
task.SubPriority = 100;
task.Source = this;
task.Tag = "Continue indexing generated indexables from " + IndexName;
ThisScheduler.Add (task);
}
if (fa_store != null)
fa_store.CommitTransaction ();
// Propagate the change notification to any open queries.
if (added_uris.Count > 0 || removed_uris.Count > 0) {
ChangeData change_data;
change_data = new ChangeData ();
change_data.AddedUris = added_uris;
change_data.RemovedUris = removed_uris;
QueryDriver.QueryableChanged (this, change_data);
}
}
virtual protected void PostFlushHook(IndexerRequest flushed_request,
IndexerReceipt [] receipts)
{
}
// This is mostly a copy of LuceneQueryable.Flush + FSQ.PostAddHooks/PostRemoveHook
static bool FlushIndexer(IIndexer indexer)
{
IndexerRequest flushed_request;
if (pending_request.IsEmpty)
{
return(false);
}
flushed_request = pending_request;
pending_request = new IndexerRequest();
IndexerReceipt [] receipts;
receipts = indexer.Flush(flushed_request);
// Flush will return null if it encounters a shutdown during flushing
if (receipts == null)
{
return(false);
}
fa_store.BeginTransaction();
bool indexer_indexable_receipt = false;
foreach (IndexerReceipt raw_r in receipts)
{
if (raw_r is IndexerAddedReceipt)
{
// Update the file attributes
IndexerAddedReceipt r = (IndexerAddedReceipt)raw_r;
Indexable indexable = flushed_request.RetrieveRequestIndexable(r);
if (indexable == null)
{
Logger.Log.Debug("Should not happen! Previously requested indexable with id #{0} has eloped!", r.Id);
continue;
}
// We don't need to write out any file attributes for
// children.
if (indexable.ParentUri != null)
{
continue;
}
string path = indexable.Uri.LocalPath;
FileAttributes attr;
attr = fa_store.ReadOrCreate(path);
attr.LastWriteTime = indexable.Timestamp;
attr.FilterName = r.FilterName;
attr.FilterVersion = r.FilterVersion;
fa_store.Write(attr);
}
else if (raw_r is IndexerRemovedReceipt)
{
// Update the file attributes
IndexerRemovedReceipt r = (IndexerRemovedReceipt)raw_r;
Indexable indexable = flushed_request.RetrieveRequestIndexable(r);
if (indexable == null) // Should never happen
{
Log.Warn("Unable to match indexable-remove #{0} to any request!", r.Id);
continue;
}
string path = indexable.Uri.LocalPath;
Logger.Log.Debug("Removing: '{0}'", path);
fa_store.Drop(path);
}
else if (raw_r is IndexerIndexablesReceipt)
{
indexer_indexable_receipt = true;
}
}
pending_request.DeferredIndexables = flushed_request.DeferredIndexables;
// Reschedule if some indexable generated more indexables
if (indexer_indexable_receipt)
{
pending_request.ContinueIndexing = true;
return(true);
}
fa_store.CommitTransaction();
return(false);
}
protected void Flush(bool continue_only)
{
IndexerRequest flushed_request;
if (continue_only)
{
// if the request is merely to signal indexhelper to continue indexing,
// then sent a fake indexerrequest but then use the previous request to retrieve
// deferred indexables
flushed_request = new IndexerRequest();
flushed_request.ContinueIndexing = true;
// Do not pass this through PreFlushHook since this is a fake request
}
else
{
lock (request_lock) {
if (pending_request.IsEmpty)
{
return;
}
flushed_request = pending_request;
pending_request = new IndexerRequest();
// We hold the request_lock when calling PreFlushHook, so
// that no other requests can come in until it exits.
PreFlushHook(flushed_request);
}
}
IndexerReceipt [] receipts;
receipts = indexer.Flush(flushed_request);
PostFlushHook(flushed_request, receipts);
if (continue_only)
{
flushed_request = pending_request;
}
// Silently return if we get a null back. This is probably
// a bad thing to do. If IndexHelper is shutdown because of
// memory blowup or is crashed, then null is returned. Silently
// returning means ignoring the indexables in the IndexHelper's
// queue (which could be more than what was sent in the last request,
// since there could be some deferred-indexables too).
if (receipts == null)
{
return;
}
// Nothing happened (except maybe an optimize, which does not
// generate a receipt). Also do nothing.
if (receipts.Length == 0)
{
return;
}
// Update the cached count of items in the driver
// FIXME: Verify that this still works after all the deferred-indexable fu
driver.SetItemCount(indexer.GetItemCount());
// Something happened, so schedule an optimize just in case.
ScheduleOptimize();
if (fa_store != null)
{
fa_store.BeginTransaction();
}
ArrayList added_uris = new ArrayList();
ArrayList removed_uris = new ArrayList();
bool indexer_indexable_receipt = false;
for (int i = 0; i < receipts.Length; ++i)
{
if (receipts [i] is IndexerAddedReceipt)
{
IndexerAddedReceipt r;
r = (IndexerAddedReceipt)receipts [i];
Indexable indexable = flushed_request.RetrieveRequestIndexable(r);
if (indexable == null)
{
Log.Debug("Should not happen! Previously requested indexable with id #{0} has eloped!", r.Id);
continue;
}
// Add the Uri to the list for our change data
// *before* doing any post-processing.
// This ensures that we have internal uris when
// we are remapping.
added_uris.Add(indexable.Uri);
// Call the appropriate hook
Uri notification_uri = indexable.Uri;
try {
// Map from internal->external Uris in the PostAddHook
notification_uri = PostAddHook(indexable, r);
} catch (Exception ex) {
Logger.Log.Warn(ex, "Caught exception in PostAddHook '{0}' '{1}' '{2}'",
indexable.Uri, r.FilterName, r.FilterVersion);
}
// Every added Uri also needs to be listed as removed,
// to avoid duplicate hits in the query. Since the
// removed Uris need to be external Uris, we add them
// to the list *after* post-processing.
removed_uris.Add(notification_uri);
}
else if (receipts [i] is IndexerRemovedReceipt)
{
IndexerRemovedReceipt r;
r = (IndexerRemovedReceipt)receipts [i];
Indexable indexable = flushed_request.RetrieveRequestIndexable(r);
if (indexable == null) // Should never happen
{
Log.Warn("Unable to match indexable-remove #{0} to any request!", r.Id);
continue;
}
// Call the appropriate hook
Uri notification_uri = indexable.Uri;
try {
notification_uri = PostRemoveHook(indexable, r.NumRemoved);
} catch (Exception ex) {
Logger.Log.Warn(ex, "Caught exception in PostRemoveHook '{0}'",
indexable.Uri);
}
// If nothing was removed, no need for change notification
if (r.NumRemoved <= 0)
{
continue;
}
// Add the removed Uri to the list for our
// change data. This will be an external Uri
// when we are remapping.
removed_uris.Add(notification_uri);
}
else if (receipts [i] is IndexerIndexablesReceipt)
{
indexer_indexable_receipt = true;
}
}
if (!continue_only)
{
lock (request_lock) {
pending_request.DeferredIndexables = flushed_request.DeferredIndexables;
}
}
if (indexer_indexable_receipt)
{
Log.Debug("Indexing of indexer generated indexables is paused. Scheduling job to continue.");
// Create a task asking the indexer to continue indexing
Scheduler.Task task;
task = Scheduler.TaskFromHook(new Scheduler.TaskHook(ContinueIndexerIndexableIndexing));
// Schedule it so that it is the immediate next task to be scheduled
task.Priority = Scheduler.Priority.Immediate;
task.SubPriority = 100;
task.Source = this;
task.Tag = "Continue indexing generated indexables from " + IndexName;
ThisScheduler.Add(task);
}
if (fa_store != null)
{
fa_store.CommitTransaction();
}
// Propagate the change notification to any open queries.
if (added_uris.Count > 0 || removed_uris.Count > 0)
{
ChangeData change_data;
change_data = new ChangeData();
change_data.AddedUris = added_uris;
change_data.RemovedUris = removed_uris;
QueryDriver.QueryableChanged(this, change_data);
}
}
virtual protected void PreFlushHook (IndexerRequest flushed_request)
{ }