override public void AddToStream(Stream stream, EventTracker tracker)
{
if (ChildCount > 1)
{
throw new Exception("Bzip2 file " + Uri + " has " + ChildCount + " children");
}
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
UnclosableStream unclosable;
unclosable = new UnclosableStream(stream);
BZip2OutputStream bz2_out;
bz2_out = new BZip2OutputStream(unclosable);
MemoryStream memory;
memory = new MemoryStream();
// There should just be one child
foreach (FileObject file in Children)
{
file.AddToStream(memory, tracker);
}
bz2_out.Write(memory.ToArray(), 0, (int)memory.Length);
memory.Close();
bz2_out.Close();
}
public Indexable GetNextIndexable()
{
if (initial_scan)
{
return(null);
}
Uri uri = new Uri(String.Format("kabc:///{0}", current_uid));
Indexable indexable = new Indexable(uri);
indexable.ParentUri = UriFu.PathToFileUri(kabc_file);
indexable.MimeType = ICalParser.KabcMimeType;
indexable.HitType = "Contact";
indexable.Timestamp = current_dt;
indexable.AddProperty(Property.NewKeyword("fixme:client", "KAddressBook"));
// Add uid as a keyword field for convenience
indexable.AddProperty(Property.NewUnsearched("fixme:uid", current_uid));
// FIXME: Comment this Debug statement after the backend stabilizes
//Log.Debug ("Creating {0} from:[{1}]", uri, string_builder.ToString ());
StringReader string_reader = new StringReader(string_builder.ToString());
indexable.SetTextReader(string_reader);
return(indexable);
}
override public ISnippetReader GetSnippet(string[] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
if (text_cache == null)
{
return(null);
}
Uri uri;
if (mount_dir == null || hit ["beagrep:RemovableUri"] == null)
{
uri = hit.Uri;
}
else
{
uri = UriFu.EscapedStringToUri(hit ["beagrep:RemovableUri"]);
}
// Look up the hit in our local text cache.
// Need to handle self-cached removable:/// uris
bool self_cache = true;
TextReader reader = text_cache.GetReader(uri, ref self_cache);
if (self_cache)
{
reader = new StreamReader(hit.Uri.LocalPath);
}
else if (reader == null)
{
return(null);
}
return(SnippetFu.GetSnippet(query_terms, reader, full_text, ctx_length, snp_length));
}
// Since some parent properties maybe stored in child properties
// as parent: property, any property change should be propagated
// to all its children as well.
private ArrayList GetChildPropertyChange(Hashtable children_docs,
Indexable parent)
{
// FIXME FIXME FIXME: Post-Child-Indexable-Fix
if (children_docs == null)
{
return(null);
}
Uri parent_uri = parent.Uri;
ArrayList child_indexable_list = new ArrayList();
foreach (Uri uri in children_docs.Keys)
{
// FIXME: Currently, children_docs has both the parent and children docs
if (UriFu.Equals(uri, parent_uri))
{
continue;
}
Indexable child_indexable;
child_indexable = new Indexable(IndexableType.PropertyChange, uri);
Log.Debug("Creating property change child indexable for {1} (parent {0})", parent.Uri, uri);
// This is where the child_indexables will have new properties from parent
child_indexable.SetChildOf(parent);
child_indexable_list.Add(child_indexable);
}
return(child_indexable_list);
}
/////////////////////////////////////////////////
private Indexable NoteToIndexable(FileInfo file, Note note)
{
Indexable indexable = new Indexable(note.Uri);
indexable.ContentUri = UriFu.PathToFileUri(file.FullName);
indexable.Timestamp = note.timestamp;
indexable.HitType = "Note";
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty(Property.New("dc:title", note.subject));
indexable.AddProperty(Property.NewUnsearched("fixme:application", "tomboy"));
// FIXME: tagging is disabled in Tomboy-0.8.x and is planned for 0.10.0
foreach (string s in note.tags)
{
indexable.AddProperty(Property.New("note:tag", s));
}
// We remember the note's text so that we can stuff it in
// the TextCache later.
note_text_cache [note.Uri] = note.text;
StringReader reader = new StringReader(note.text);
indexable.SetTextReader(reader);
return(indexable);
}
//////////////////////////////////////////////////
// Convert matches to Hits
private Hit PathToHit(string path)
{
// Check if hidden
if (path.IndexOf("/.") != -1)
{
return(null);
}
Hit hit = new Hit();
hit.Uri = UriFu.PathToFileUri(path);
hit.Timestamp = File.GetLastWriteTimeUtc(path);
hit.AddProperty(Property.NewUnsearched("beagle:HitType", "File"));
// Prevent any mimetype matching
hit.AddProperty(Property.NewUnsearched("beagle:MimeType", "beagle/x-locate-result"));
hit.AddProperty(Property.NewUnsearched("beagle:Source", "Locate"));
// Use a generic enough filetype to hint there is _no_ special properties
// for this hit
hit.AddProperty(Property.NewUnsearched("beagle:FileType", "document"));
hit.Score = 1.0;
foreach (Property std_prop in Property.StandardFileProperties(Path.GetFileName(path), true))
{
hit.AddProperty(std_prop);
}
hit.AddProperty(Property.NewUnsearched(
Property.ParentDirUriPropKey,
UriFu.PathToFileUri(Path.GetDirectoryName(path))));
return(hit);
}
private Indexable BookmarkToIndexable()
{
Indexable indexable = new Indexable(new Uri(current_bookmark_id));
indexable.ParentUri = UriFu.PathToFileUri(bookmark_file);
indexable.HitType = "Bookmark";
indexable.Timestamp = current_dt;
indexable.NoContent = true;
indexable.AddProperty(Property.New("dc:title", current_bookmark.Title));
indexable.AddProperty(Property.New("dc:identifier", current_bookmark.Href));
indexable.AddProperty(Property.NewUnsearched("fixme:icon", current_bookmark.Icon));
if (current_bookmark.Info != null &&
current_bookmark.Info.Metadata != null &&
current_bookmark.Info.Metadata.NumVisited != 0)
{
indexable.AddProperty(Property.NewUnsearched("fixme:visit_count", current_bookmark.Info.Metadata.NumVisited));
}
foreach (string folder in folder_stack)
{
indexable.AddProperty(Property.New("fixme:folder", folder));
}
return(indexable);
}
override protected Uri GetChildUri(FileSystemObject child)
{
string s = Path.Combine(UriFu.UriToEscapedString(this.Uri), child.Name);
//Console.WriteLine ("Asked to combine {0} and {1} = {2}", UriFu.UriToEscapedString (this.Uri), child.Name, s);
return(UriFu.EscapedStringToUri(s));
}
private Indexable current_itemToIndexable()
{
// sanity check
if (current_item == null)
{
return(null);
}
//Log.Debug ("Indexing " + channel_link + ":" + current_item.Link);
Indexable indexable = new Indexable(new Uri(String.Format("feed:{0};item={1}", channel_link, current_item.Link)));
indexable.ParentUri = UriFu.PathToFileUri(feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
string RFC822 = "ddd, dd MMM yyyy HH:mm:ss zzz";
DateTime date = DateTime.ParseExact(current_item.PubDate, RFC822, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal);
indexable.Timestamp = date;
// replace property names with Dublin Core names
indexable.AddProperty(Property.New("dc:title", current_item.Title));
indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Link));
indexable.AddProperty(Property.NewKeyword("dc:source", channel_link));
indexable.AddProperty(Property.New("dc:publisher", channel_title));
StringReader reader = new StringReader(current_item.Description);
indexable.SetTextReader(reader);
return(indexable);
}
///////////////////////////////////////////////////////////////////////
override public void AddOnDisk(EventTracker tracker)
{
string full_name;
full_name = FullName;
if (full_name == null)
{
throw new Exception("Attempt to instantiate something other than a real file: " + Uri);
}
if (is_root)
{
// Root directories must already exist.
if (!Directory.Exists(full_name))
{
throw new Exception("Missing root directory " + full_name);
}
}
else
{
Directory.CreateDirectory(full_name);
timestamp = Directory.GetLastWriteTimeUtc(full_name);
}
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
// Recursively add the children
foreach (FileSystemObject fso in children.Values)
{
fso.AddOnDisk(tracker);
}
}
////////////////////////////////////////////////////////////////
public bool HasUri(Uri uri)
{
IndexReader primary_reader;
primary_reader = LuceneCommon.GetReader(PrimaryStore);
Term term;
term = new Term("Uri", UriFu.UriToEscapedString(uri));
TermDocs term_docs;
term_docs = primary_reader.TermDocs();
term_docs.Seek(term);
bool has_uri = false;
if (term_docs.Next())
{
has_uri = true;
}
term_docs.Close();
LuceneCommon.ReleaseReader(primary_reader);
return(has_uri);
}
static public ICollection CreateFiltersFromPath(string path)
{
string guessed_mime_type = XdgMime.GetMimeType(path);
string extension = Path.GetExtension(path);
return(CreateFilters(UriFu.PathToFileUri(path), extension, guessed_mime_type));
}
static public ICollection CreateFiltersFromIndexable(Indexable indexable)
{
string path = indexable.ContentUri.LocalPath;
string extension = Path.GetExtension(path);
string mime_type = indexable.MimeType;
return(CreateFilters(UriFu.PathToFileUri(path), extension, mime_type));
}
public EvolutionDataServerQueryable() : base("EvolutionDataServerIndex", INDEX_VERSION)
{
indexable_text_cache = UriFu.NewHashtable();
photo_dir = Path.Combine(Driver.TopDirectory, "Photos");
System.IO.Directory.CreateDirectory(photo_dir);
generator = new SchedulingIndexableGenerator(this, "Evolution Data Server");
}
public Indexable GetNextIndexable()
{
DateTime line_dt = DateTime.MinValue;
while (log_line_as_sb != null)
{
//Log.Debug ("Checking line from {0}:[{1}]", log_file, log_line_as_sb);
bool in_session = AppendLogText(log_line_as_sb, out line_dt);
if (!in_session)
{
break;
}
prev_line_offset = reader.Position;
log_line_as_sb = reader.ReadLineAsStringBuilder();
}
// Check if there is new data to index
if (data_sb.Length == 0)
{
session_begin_time = line_dt;
return(null);
}
Uri uri = new Uri(String.Format("konversation://{0}@dumb/{1}", session_begin_offset, log_file));
Log.Debug("Creating indexable {0}", uri);
Indexable indexable = new Indexable(uri);
indexable.ParentUri = UriFu.PathToFileUri(log_file);
indexable.Timestamp = session_begin_time;
indexable.HitType = "IMLog";
indexable.CacheContent = false;
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_begin_offset", session_begin_offset));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_end_offset", prev_line_offset));
indexable.AddProperty(Beagle.Property.NewDate("fixme:starttime", session_begin_time));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:client", "Konversation"));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:protocol", "IRC"));
AddChannelInformation(indexable);
foreach (string speaker in speakers.Keys)
{
indexable.AddProperty(Beagle.Property.NewUnstored("fixme:speaker", speaker));
}
StringReader data_reader = new StringReader(data_sb.ToString());
indexable.SetTextReader(data_reader);
// update session begin time to the date of the current line which is not in this session
session_begin_time = line_dt;
return(indexable);
}
private void RemoveFeedFile(string file)
{
Log.Debug("Removing Akregator feedfile:" + file);
Uri uri = UriFu.PathToFileUri(file);
Scheduler.Task task = NewRemoveTask(uri);
task.Priority = Scheduler.Priority.Immediate;
task.SubPriority = 0;
ThisScheduler.Add(task);
}
/**
* deleting mbox means deleting all the mails which were in this mbox
* we use the idea of parent-uri
* while creating indexables, we set the parent uri to be the uri of the mbox file
* so to delete all mails in the mbox we just delete all documents whose parent uri
* is the uri of the mbox file
*/
public void RemoveMbox(string file)
{
Logger.Log.Debug("Removing mbox:" + file);
Uri uri = UriFu.PathToFileUri(file);
Scheduler.Task task = queryable.NewRemoveTask(uri);
task.Priority = Scheduler.Priority.Immediate;
task.SubPriority = 0;
queryable.ThisScheduler.Add(task);
}
/////////////////////////////////////////////////
private static Indexable NoteToIndexable(FileInfo file)
{
Indexable indexable = new Indexable(UriFu.PathToFileUri(file.FullName));
indexable.Timestamp = file.LastWriteTimeUtc;
indexable.HitType = "Note";
indexable.MimeType = "x-beagle/x-labyrinth-note";
indexable.AddProperty(Property.NewUnsearched("fixme:application", "labyrinth"));
return(indexable);
}
/////////////////////////////////////////////////////////
static void ExecuteRemove(string arg)
{
LuceneDriver driver = new LuceneDriver(index_dir);
if (arg.IndexOf("://") != -1)
{
Uri uri = new Uri(arg);
ICollection hits = driver.DoQueryByUri(uri);
if (hits == null || hits.Count == 0)
{
Console.WriteLine("Uri not found in the index: {0}", uri);
Environment.Exit(1);
}
driver.Remove(uri);
driver.Flush();
Console.WriteLine("Successfully removed Uri: {0}", uri);
}
else
{
IndexSearcher searcher = new IndexSearcher(driver.Store);
BooleanQuery query = new BooleanQuery();
Term term = new Term("prop:k:Tag", arg); // Argh
TermQuery term_query = new TermQuery(term);
query.Add(term_query, false, false);
Hits hits = searcher.Search(query);
int n_hits = hits.Length();
string uri;
for (int i = 0; i < n_hits; ++i)
{
Document doc = hits.Doc(i);
uri = doc.Get("Uri");
if (uri == null)
{
continue;
}
driver.Remove(UriFu.UriStringToUri(uri));
}
driver.Flush();
Console.WriteLine("Successfully removed {0} items with tag: {1}", n_hits, arg);
}
}
/////////////////////////////////////////////////
private static Indexable ImLogToIndexable(string filename)
{
Uri uri = UriFu.PathToFileUri(filename);
Indexable indexable = new Indexable(uri);
indexable.ContentUri = uri;
indexable.Timestamp = File.GetLastWriteTimeUtc(filename);
indexable.MimeType = "beagle/x-empathy-log";
indexable.HitType = "IMLog";
indexable.CacheContent = true;
return(indexable);
}
static Uri PathToUri(string fullpath)
{
if (!arg_removable)
{
return(UriFu.PathToFileUri(fullpath));
}
fullpath = PathInIndex(fullpath);
return(new Uri(String.Format("removable{0}{1}{2}",
Uri.SchemeDelimiter,
volume_label,
StringFu.HexEscape(fullpath)), true));
}
private Indexable current_itemToIndexable()
{
Indexable indexable;
try {
indexable = new Indexable(new Uri(String.Format("{0};item={1}", feed_source, current_item.Source)));
} catch (System.UriFormatException) {
indexable = new Indexable(new Uri(String.Format("liferea://dummy?{0};item={1}", feed_source, current_item.Source)));
}
indexable.ParentUri = UriFu.PathToFileUri(feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
DateTime date = DateTimeUtil.UnixToDateTimeUtc(0);
date = date.AddSeconds(current_item.Timestamp);
indexable.Timestamp = date;
// cleaning up the property names as far as possible
// this way querying for specific field is possible
// following DC element names wherever applicable
indexable.AddProperty(Property.New("dc:title", current_item.Title));
Attribute[] attribs = current_item.Attribs.AttribArray;
if (attribs != null)
{
foreach (Attribute attrib in attribs)
{
if (attrib.Name != "author")
{
continue;
}
indexable.AddProperty(Property.New("dc:creator", attrib.Value));
}
}
indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Source));
indexable.AddProperty(Property.NewKeyword("dc:source", feed_source));
indexable.AddProperty(Property.New("dc:publisher", publisher));
if (File.Exists(icon_file))
{
indexable.AddProperty(Property.NewUnsearched("fixme:cachedimg", icon_file));
}
StringReader reader = new StringReader(current_item.Description);
indexable.SetTextReader(reader);
return(indexable);
}
static private Indexable MemberNodeToIndexable(XmlNode node, Uri base_uri, string parentName)
{
char memberType = MemberTypeToChar(node.SelectSingleNode("MemberType").InnerText);
StringBuilder memberFullName = new StringBuilder();
memberFullName.Append(memberType + ":" + parentName);
if (memberType != 'C')
{
memberFullName.Append("." + node.Attributes["MemberName"].Value);
}
if (memberType == 'C' || memberType == 'M' || memberType == 'E')
{
memberFullName.Append("(");
bool inside = false;
foreach (XmlNode parameter in node.SelectNodes("Parameters/Parameter"))
{
if (!inside)
{
inside = true;
}
else
{
memberFullName.Append(",");
}
memberFullName.Append(parameter.Attributes["Type"].Value);
}
memberFullName.Append(")");
}
Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, memberFullName.ToString(), false));
indexable.MimeType = "text/html";
indexable.HitType = "MonodocEntry";
indexable.AddProperty(Property.New("dc:title", memberFullName.ToString()));
indexable.AddProperty(Property.New("fixme:name", memberFullName.ToString()));
indexable.AddProperty(Property.NewUnsearched("fixme:type", node.SelectSingleNode("MemberType").InnerText.ToLower()));
StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml);
indexable.SetTextReader(reader);
return(indexable);
}
public void Add(Indexable indexable)
{
if (indexable == null)
{
return;
}
if (indexables_by_uri == null)
{
indexables_by_uri = UriFu.NewHashtable();
}
Indexable prior;
prior = indexables_by_uri [indexable.Uri] as Indexable;
if (prior != null)
{
switch (indexable.Type)
{
case IndexableType.Add:
case IndexableType.Remove:
// Clobber the prior indexable.
indexable.Id = prior.Id;
indexables [prior.Id] = indexable;
indexables_by_uri [indexable.Uri] = indexable;
break;
case IndexableType.PropertyChange:
if (prior.Type != IndexableType.Remove)
{
// Merge with the prior indexable.
prior.Merge(indexable);
}
break;
}
}
else
{
lock (indexable_id_lock) {
indexable.Id = indexable_id;
indexable_id++;
}
indexables [indexable.Id] = indexable;
indexables_by_uri [indexable.Uri] = indexable;
}
}
static private Indexable TypeNodeToIndexable(XmlNode node, Uri base_uri)
{
string fragment = "T:" + node.Attributes ["FullName"].Value;
Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, fragment, false));
indexable.MimeType = "text/html";
indexable.HitType = "MonodocEntry";
indexable.AddProperty(Property.New("dc:title", "T:" + node.Attributes["FullName"].Value));
indexable.AddProperty(Property.NewUnsearched("fixme:name", "T:" + node.Attributes["FullName"].Value));
indexable.AddProperty(Property.NewUnsearched("fixme:type", "type"));
StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml);
indexable.SetTextReader(reader);
return(indexable);
}
static public bool CheckQuery(Query query, FileSystemObject root)
{
// Find the set of objects that we expect to match the query,
// based on our knowledge of the current state of the tree.
ICollection matching_fsos;
matching_fsos = root.RecursiveQuery(query);
// Query the daemon and get the actual list of hits.
Hashtable matching_hits;
matching_hits = QueryFu.GetHits(query);
bool success;
success = true;
foreach (FileSystemObject fso in matching_fsos)
{
string uri = UriFu.UriToEscapedString(fso.Uri);
if (matching_hits.Contains(uri))
{
matching_hits.Remove(uri);
}
else
{
Log.Failure("Hit missing from beagled query results: {0}", uri);
success = false;
}
}
foreach (Hit hit in matching_hits.Values)
{
Log.Failure("Unexpected extra hit in beagled query results: {0}", hit.Uri);
Log.Failure(" Properties:");
foreach (Property prop in hit.Properties)
{
Log.Failure(" {0} = {1}", prop.Key, prop.Value);
}
success = false;
}
return(success);
}
/**
* Create an indexable from a maildir message
*/
public Indexable MaildirMessageToIndexable(string filename, bool crawl)
{
//Logger.Log.Debug ("+ indexing maildir mail:" + filename);
String folder = GetFolderMaildir(filename);
Uri file_uri = UriFu.PathToFileUri(filename);
Indexable indexable = new Indexable(file_uri);
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.FlushBufferCache = crawl;
indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail"));
indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name));
indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder));
indexable.ContentUri = file_uri;
return(indexable);
}
private bool HitIsValidRemovable(Hit hit)
{
if (mount_dir == null || hit.Uri.Scheme != "removable")
{
return(true);
}
hit.AddProperty(Beagrep.Property.NewKeyword("beagrep:RemovableUri",
hit.EscapedUri));
string path = hit.Uri.LocalPath;
path = path.Substring(1); // Remove initial '/'
path = Path.Combine(mount_dir, path);
hit.Uri = UriFu.PathToFileUri(path);
hit.AddProperty(Beagrep.Property.NewKeyword("fixme:mount_dir", mount_dir));
return(true);
}
override public void AddToStream(Stream stream, EventTracker tracker)
{
if (tracker != null)
{
tracker.ExpectingAdded(UriFu.UriToEscapedString(this.Uri));
}
// We can't just use a StreamWriter here, since that
// will close the underlying stream when it gets
// disposed.
UnclosableStream unclosable = new UnclosableStream(stream);
StreamWriter writer = new StreamWriter(unclosable);
foreach (string str in body)
{
writer.WriteLine(str);
}
writer.Close();
}
//////////////////////////
private static Uri TextReaderToTempFileUri(TextReader reader)
{
if (reader == null)
{
return(null);
}
string filename = Path.GetTempFileName();
FileStream fileStream = File.OpenWrite(filename);
// When we dump the contents of an indexable into a file, we
// expect to use it again soon.
FileAdvise.PreLoad(fileStream);
// Make sure the temporary file is only readable by the owner.
// FIXME: There is probably a race here. Could some malicious program
// do something to the file between creation and the chmod?
Mono.Unix.Native.Syscall.chmod(filename, Mono.Unix.Native.FilePermissions.S_IRUSR);
BufferedStream bufferedStream = new BufferedStream(fileStream);
StreamWriter writer = new StreamWriter(bufferedStream);
char [] buffer;
buffer = GetCharBuffer();
int read;
do
{
read = reader.Read(buffer, 0, buffer.Length);
if (read > 0)
{
writer.Write(buffer, 0, read);
}
} while (read > 0);
writer.Close();
return(UriFu.PathToFileUri(filename));
}
