override protected Uri GetChildUri(FileSystemObject child)
{
string s = Path.Combine(UriFu.UriToEscapedString(this.Uri), child.Name);
//Console.WriteLine ("Asked to combine {0} and {1} = {2}", UriFu.UriToEscapedString (this.Uri), child.Name, s);
return(UriFu.EscapedStringToUri(s));
}
override public void AddToStream(Stream stream, EventTracker tracker)
{
if (ChildCount > 1)
{
throw new Exception("Bzip2 file " + Uri + " has " + ChildCount + " children");
}
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
UnclosableStream unclosable;
unclosable = new UnclosableStream(stream);
BZip2OutputStream bz2_out;
bz2_out = new BZip2OutputStream(unclosable);
MemoryStream memory;
memory = new MemoryStream();
// There should just be one child
foreach (FileObject file in Children)
{
file.AddToStream(memory, tracker);
}
bz2_out.Write(memory.ToArray(), 0, (int)memory.Length);
memory.Close();
bz2_out.Close();
}
////////////////////////////////////////////////////////////////
public bool HasUri(Uri uri)
{
IndexReader primary_reader;
primary_reader = LuceneCommon.GetReader(PrimaryStore);
Term term;
term = new Term("Uri", UriFu.UriToEscapedString(uri));
TermDocs term_docs;
term_docs = primary_reader.TermDocs();
term_docs.Seek(term);
bool has_uri = false;
if (term_docs.Next())
{
has_uri = true;
}
term_docs.Close();
LuceneCommon.ReleaseReader(primary_reader);
return(has_uri);
}
///////////////////////////////////////////////////////////////////////
override public void AddOnDisk(EventTracker tracker)
{
string full_name;
full_name = FullName;
if (full_name == null)
{
throw new Exception("Attempt to instantiate something other than a real file: " + Uri);
}
if (is_root)
{
// Root directories must already exist.
if (!Directory.Exists(full_name))
{
throw new Exception("Missing root directory " + full_name);
}
}
else
{
Directory.CreateDirectory(full_name);
timestamp = Directory.GetLastWriteTimeUtc(full_name);
}
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
// Recursively add the children
foreach (FileSystemObject fso in children.Values)
{
fso.AddOnDisk(tracker);
}
}
static public bool CheckQuery(Query query, FileSystemObject root)
{
// Find the set of objects that we expect to match the query,
// based on our knowledge of the current state of the tree.
ICollection matching_fsos;
matching_fsos = root.RecursiveQuery(query);
// Query the daemon and get the actual list of hits.
Hashtable matching_hits;
matching_hits = QueryFu.GetHits(query);
bool success;
success = true;
foreach (FileSystemObject fso in matching_fsos)
{
string uri = UriFu.UriToEscapedString(fso.Uri);
if (matching_hits.Contains(uri))
{
matching_hits.Remove(uri);
}
else
{
Log.Failure("Hit missing from beagled query results: {0}", uri);
success = false;
}
}
foreach (Hit hit in matching_hits.Values)
{
Log.Failure("Unexpected extra hit in beagled query results: {0}", hit.Uri);
Log.Failure(" Properties:");
foreach (Property prop in hit.Properties)
{
Log.Failure(" {0} = {1}", prop.Key, prop.Value);
}
success = false;
}
return(success);
}
override public void AddToStream(Stream stream, EventTracker tracker)
{
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
// We can't just use a StreamWriter here, since that
// will close the underlying stream when it gets
// disposed.
UnclosableStream unclosable = new UnclosableStream(stream);
StreamWriter writer = new StreamWriter(unclosable);
foreach (string str in body)
{
writer.WriteLine(str);
}
writer.Close();
}
////////////////////////////////////////////////////////////////
public NameInfo GetNameInfoById(Guid id)
{
Uri uri;
uri = GuidFu.ToUri(id);
IndexReader reader;
reader = LuceneCommon.GetReader(SecondaryStore);
TermDocs term_docs;
term_docs = reader.TermDocs();
Term term = new Term("Uri", UriFu.UriToEscapedString(uri));
term_docs.Seek(term);
int match_id = -1;
if (term_docs.Next())
{
match_id = term_docs.Doc();
}
term_docs.Close();
NameInfo info = null;
if (match_id != -1)
{
Document doc;
doc = reader.Document(match_id, fields_nameinfo);
info = DocumentToNameInfo(doc);
}
LuceneCommon.ReleaseReader(reader);
return(info);
}
override public void AddToStream(Stream stream, EventTracker tracker)
{
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
UnclosableStream unclosable;
unclosable = new UnclosableStream(stream);
ZipOutputStream zip_out;
zip_out = new ZipOutputStream(unclosable);
foreach (FileSystemObject fso in Children)
{
WriteObjectToZip(zip_out, fso, tracker);
}
zip_out.Finish();
zip_out.Close();
}
override public void AddToStream(Stream stream, EventTracker tracker)
{
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
UnclosableStream unclosable;
unclosable = new UnclosableStream(stream);
TarOutputStream tar_out;
tar_out = new TarOutputStream(unclosable);
foreach (FileSystemObject fso in Children)
{
WriteObjectToTar(tar_out, fso, tracker);
}
// This calls close on the underlying stream,
// which is why we wrapped the stream in an
// UnclosableStream.
tar_out.Close();
}
static int DumpOneIndex_Metadata(string index_name, ArrayList uris, bool show_properties)
{
LuceneQueryingDriver driver;
driver = new LuceneQueryingDriver(index_name, -1, true);
Hashtable all_hits_by_uri = null;
ArrayList all_hits = null;
if (uris.Count == 0 || index_name == "FileSystemIndex")
{
all_hits_by_uri = driver.GetAllHitsByUri();
all_hits = new ArrayList(all_hits_by_uri.Values);
}
// A hard-wired hack
if (index_name == "FileSystemIndex")
{
foreach (Hit hit in all_hits)
{
string internal_uri;
if (hit [Property.IsChildPropKey] == "true")
{
string path = RemapUriToPath(all_hits_by_uri, hit);
internal_uri = UriFu.UriToEscapedString(hit.ParentUri);
hit.ParentUri = UriFu.PathToFileUri(path);
hit.Uri = UriFu.AddFragment(UriFu.PathToFileUri(path),
hit.Uri.Fragment,
true);
}
else
{
internal_uri = UriFu.UriToEscapedString(hit.Uri);
hit.Uri = UriFu.PathToFileUri(RemapUriToPath(all_hits_by_uri, hit));
hit.AddProperty(Property.NewUnsearched("beagrep:InternalUri", internal_uri));
}
}
}
ArrayList matching_hits;
if (uris.Count == 0)
{
matching_hits = all_hits;
}
else
{
matching_hits = new ArrayList(driver.GetHitsForUris(RemapUris(driver, uris)));
if (index_name == "FileSystemIndex")
{
for (int i = 0; i < matching_hits.Count; i++)
{
Hit hit = (Hit)matching_hits [i];
Hit mapped_hit = (Hit)all_hits_by_uri [hit.Uri];
matching_hits [i] = mapped_hit;
}
}
}
matching_hits.Sort(new HitByUriComparer());
foreach (Hit hit in matching_hits)
{
if (!show_properties)
{
Console.WriteLine("{0}: {1}", index_name, hit.Uri);
continue;
}
Console.WriteLine(" Index: {0}", index_name);
Console.WriteLine(" Uri: {0}", hit.Uri);
if (hit.ParentUri != null)
{
Console.WriteLine("Parent: {0}", hit.ParentUri);
}
Console.WriteLine(" MimeT: {0}", hit.MimeType);
Console.WriteLine(" Type: {0}", hit.Type);
Console.WriteLine("Source: {0}", hit.Source);
ArrayList props;
props = new ArrayList(hit.Properties);
props.Sort();
foreach (Property prop in props)
{
char [] legend = new char [4];
legend [0] = prop.IsMutable ? 'm' : ' ';
legend [1] = prop.IsSearched ? 's' : ' ';
legend [2] = prop.IsPersistent ? 'p' : ' ';
legend [3] = prop.Type == PropertyType.Text ? 't' : ' ';
Console.WriteLine(" Prop: [{0}] {1} = '{2}'", new String(legend), prop.Key, prop.Value);
}
Console.WriteLine();
}
return(matching_hits.Count);
}
private IndexerReceipt [] Flush_Unlocked(IndexerRequest request)
{
ArrayList receipt_queue;
receipt_queue = new ArrayList();
IndexReader primary_reader, secondary_reader;
primary_reader = IndexReader.Open(PrimaryStore);
secondary_reader = IndexReader.Open(SecondaryStore);
// Step #1: Make our first pass over the list of
// indexables that make up our request. For each add
// or property change in the request, get the Lucene
// documents so we can move forward any persistent
// properties (for adds) or all old properties (for
// property changes).
//
// Then, for each add or remove in the request,
// delete the associated documents from the index.
// Note that we previously cached added documents so
// that we can move persistent properties forward.
// parent_child_old_props is double-nested hashtable (depth-2 tree)
// indexed by the parent uri, it stores another hashtable indexed by the (parent+child documents)
// FIXME: 2-level hashtable is a waste for any non-child document.
// Replace this by a better data structure.
Hashtable parent_child_old_props = UriFu.NewHashtable();
TermDocs term_docs = secondary_reader.TermDocs();
int delete_count = 0;
IEnumerable request_indexables = request.Indexables;
foreach (Indexable indexable in request_indexables)
{
string uri_str = UriFu.UriToEscapedString(indexable.Uri);
Term term;
// Store the necessary properties from old documents for re-addition
if (indexable.Type == IndexableType.Add ||
indexable.Type == IndexableType.PropertyChange)
{
term = new Term("Uri", uri_str);
term_docs.Seek(term);
Hashtable this_parent_child_props = null;
if (term_docs.Next())
{
this_parent_child_props = UriFu.NewHashtable();
this_parent_child_props [indexable.Uri] = secondary_reader.Document(term_docs.Doc());
parent_child_old_props [indexable.Uri] = this_parent_child_props;
}
term = new Term("ParentUri", uri_str);
term_docs.Seek(term);
while (term_docs.Next())
{
Document doc = secondary_reader.Document(term_docs.Doc());
string child_uri_str = doc.Get("Uri");
Uri child_uri = UriFu.EscapedStringToUri(child_uri_str);
// Any valid lucene document *should* have a Uri, so no need to check for null
// Store the child documents too, to save persistent-properties
// of child documents
this_parent_child_props [child_uri] = doc;
}
}
// Now remove (non-remove indexables will be re-added in next block)
Logger.Log.Debug("-{0}", indexable.DisplayUri);
int num_delete = 0;
term = new Term("Uri", uri_str);
// For property changes, only secondary index is modified
secondary_reader.DeleteDocuments(term);
// Now remove from everywhere else (if asked to remove or if asked to add, in which case
// we first remove and then add)
// So we also need to remove child documents
if (indexable.Type != IndexableType.PropertyChange)
{
num_delete = primary_reader.DeleteDocuments(term);
// When we delete an indexable, also delete any children.
// FIXME: Shouldn't we also delete any children of children, etc.?
term = new Term("ParentUri", uri_str);
num_delete += primary_reader.DeleteDocuments(term);
secondary_reader.DeleteDocuments(term);
}
// If this is a strict removal (and not a deletion that
// we are doing in anticipation of adding something back),
// queue up a removed receipt.
if (indexable.Type == IndexableType.Remove)
{
IndexerRemovedReceipt r;
r = new IndexerRemovedReceipt(indexable.Id);
r.NumRemoved = num_delete;
receipt_queue.Add(r);
}
delete_count += num_delete;
}
term_docs.Close();
if (HaveItemCount)
{
AdjustItemCount(-delete_count);
}
else
{
SetItemCount(primary_reader);
}
// We are now done with the readers, so we close them.
// And also free them. Somehow not freeing them is preventing them from
// GCed at all.
primary_reader.Close();
primary_reader = null;
secondary_reader.Close();
secondary_reader = null;
// FIXME: If we crash at exactly this point, we are in
// trouble. Items will have been dropped from the index
// without the proper replacements being added. We can
// hopefully fix this when we move to Lucene 2.1.
// Step #2: Make another pass across our list of indexables
// and write out any new documents.
if (text_cache != null)
{
text_cache.BeginTransaction();
}
IndexWriter primary_writer, secondary_writer;
// FIXME: Lock obtain time-out can happen here; if that happens,
// an exception will be thrown and this method will break in the middle
// leaving IndexWriters unclosed! Same for any Lucene.Net-index modification
// methods.
primary_writer = new IndexWriter(PrimaryStore, IndexingAnalyzer, false);
secondary_writer = null;
foreach (Indexable indexable in request_indexables)
{
// If shutdown has been started, break here
// FIXME: Some more processing will continue, a lot of them
// concerning receipts, but the daemon will anyway ignore receipts
// now, what is the fastest way to stop from here ?
if (Shutdown.ShutdownRequested)
{
Log.Debug("Shutdown initiated. Breaking while flushing indexables.");
break;
}
// Receipts for removes were generated in the
// previous block. Now we just have to remove
// items from the text cache.
if (indexable.Type == IndexableType.Remove)
{
if (text_cache != null)
{
text_cache.Delete(indexable.Uri);
}
continue;
}
IndexerAddedReceipt r;
Hashtable prop_change_docs = (Hashtable)parent_child_old_props [indexable.Uri];
if (indexable.Type == IndexableType.PropertyChange)
{
Logger.Log.Debug("+{0} (props only)", indexable.DisplayUri);
r = new IndexerAddedReceipt(indexable.Id);
r.PropertyChangesOnly = true;
receipt_queue.Add(r);
Document doc;
if (prop_change_docs == null)
{
doc = null;
}
else
{
doc = (Document)prop_change_docs [indexable.Uri];
}
Document new_doc;
new_doc = RewriteDocument(doc, indexable);
// Write out the new document...
if (secondary_writer == null)
{
secondary_writer = new IndexWriter(SecondaryStore, IndexingAnalyzer, false);
}
secondary_writer.AddDocument(new_doc);
// Get child property change indexables...
ArrayList prop_change_indexables;
prop_change_indexables = GetChildPropertyChange(prop_change_docs, indexable);
// and store them; no need to delete them first, since they were already removed from the index
if (prop_change_indexables == null)
{
continue;
}
foreach (Indexable prop_change_indexable in prop_change_indexables)
{
Log.Debug("+{0} (props only, generated indexable)", prop_change_indexable.Uri);
doc = (Document)prop_change_docs [prop_change_indexable.Uri];
new_doc = RewriteDocument(doc, prop_change_indexable);
secondary_writer.AddDocument(new_doc);
}
continue; // ...and proceed to the next Indexable
}
// If we reach this point we know we are dealing with an IndexableType.Add
if (indexable.Type != IndexableType.Add)
{
throw new Exception("When I said it was an IndexableType.Add, I meant it!");
}
r = AddIndexableToIndex(indexable, primary_writer, ref secondary_writer, prop_change_docs);
if (r != null)
{
receipt_queue.Add(r);
}
}
if (text_cache != null)
{
text_cache.CommitTransaction();
}
if (Shutdown.ShutdownRequested)
{
foreach (DeferredInfo di in deferred_indexables)
{
di.Cleanup();
}
deferred_indexables.Clear();
foreach (Indexable indexable in request_indexables)
{
indexable.Cleanup();
}
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
return(null);
}
if (request.OptimizeIndex)
{
Stopwatch watch = new Stopwatch();
Logger.Log.Debug("Optimizing {0}", IndexName);
watch.Start();
primary_writer.Optimize();
if (secondary_writer == null)
{
secondary_writer = new IndexWriter(SecondaryStore, IndexingAnalyzer, false);
}
secondary_writer.Optimize();
watch.Stop();
Logger.Log.Debug("{0} optimized in {1}", IndexName, watch);
}
// Step #4. Close our writers and return the events to
// indicate what has happened.
primary_writer.Close();
if (secondary_writer != null)
{
secondary_writer.Close();
}
// Send a single IndexerIndexablesReceipt if there were deferred indexables
if (deferred_indexables.Count > 0)
{
Log.Debug("{0} indexables generated more indexables; asking daemon to schedule their indexing.", deferred_indexables.Count);
IndexerIndexablesReceipt r = new IndexerIndexablesReceipt();
receipt_queue.Add(r);
}
IndexerReceipt [] receipt_array;
receipt_array = new IndexerReceipt [receipt_queue.Count];
for (int i = 0; i < receipt_queue.Count; ++i)
{
receipt_array [i] = (IndexerReceipt)receipt_queue [i];
}
return(receipt_array);
}
private static string UriToString(Uri uri)
{
return(UriFu.UriToEscapedString(uri).Replace("'", "''"));
}
////////////////////////////////////////////////////////////
public void AddUri(Uri uri)
{
AddUri(UriFu.UriToEscapedString(uri));
}
public void OpenFromUri (System.Uri uri)
{
OpenFromUri (UriFu.UriToEscapedString (uri));
}
public override void Open()
{
base.OpenFromUri(UriFu.UriToEscapedString(Hit.Uri));
}