public Indexable GetNextIndexable()
{
if (initial_scan)
{
return(null);
}
Uri uri = new Uri(String.Format("kabc:///{0}", current_uid));
Indexable indexable = new Indexable(uri);
indexable.ParentUri = UriFu.PathToFileUri(kabc_file);
indexable.MimeType = ICalParser.KabcMimeType;
indexable.HitType = "Contact";
indexable.Timestamp = current_dt;
indexable.AddProperty(Property.NewKeyword("fixme:client", "KAddressBook"));
// Add uid as a keyword field for convenience
indexable.AddProperty(Property.NewUnsearched("fixme:uid", current_uid));
// FIXME: Comment this Debug statement after the backend stabilizes
//Log.Debug ("Creating {0} from:[{1}]", uri, string_builder.ToString ());
StringReader string_reader = new StringReader(string_builder.ToString());
indexable.SetTextReader(string_reader);
return(indexable);
}
//////////////////////////////////////////////////
// Convert matches to Hits
private Hit PathToHit(string path)
{
// Check if hidden
if (path.IndexOf("/.") != -1)
{
return(null);
}
Hit hit = new Hit();
hit.Uri = UriFu.PathToFileUri(path);
hit.Timestamp = File.GetLastWriteTimeUtc(path);
hit.AddProperty(Property.NewUnsearched("beagle:HitType", "File"));
// Prevent any mimetype matching
hit.AddProperty(Property.NewUnsearched("beagle:MimeType", "beagle/x-locate-result"));
hit.AddProperty(Property.NewUnsearched("beagle:Source", "Locate"));
// Use a generic enough filetype to hint there is _no_ special properties
// for this hit
hit.AddProperty(Property.NewUnsearched("beagle:FileType", "document"));
hit.Score = 1.0;
foreach (Property std_prop in Property.StandardFileProperties(Path.GetFileName(path), true))
{
hit.AddProperty(std_prop);
}
hit.AddProperty(Property.NewUnsearched(
Property.ParentDirUriPropKey,
UriFu.PathToFileUri(Path.GetDirectoryName(path))));
return(hit);
}
static public ICollection CreateFiltersFromPath(string path)
{
string guessed_mime_type = XdgMime.GetMimeType(path);
string extension = Path.GetExtension(path);
return(CreateFilters(UriFu.PathToFileUri(path), extension, guessed_mime_type));
}
private Indexable BookmarkToIndexable()
{
Indexable indexable = new Indexable(new Uri(current_bookmark_id));
indexable.ParentUri = UriFu.PathToFileUri(bookmark_file);
indexable.HitType = "Bookmark";
indexable.Timestamp = current_dt;
indexable.NoContent = true;
indexable.AddProperty(Property.New("dc:title", current_bookmark.Title));
indexable.AddProperty(Property.New("dc:identifier", current_bookmark.Href));
indexable.AddProperty(Property.NewUnsearched("fixme:icon", current_bookmark.Icon));
if (current_bookmark.Info != null &&
current_bookmark.Info.Metadata != null &&
current_bookmark.Info.Metadata.NumVisited != 0)
{
indexable.AddProperty(Property.NewUnsearched("fixme:visit_count", current_bookmark.Info.Metadata.NumVisited));
}
foreach (string folder in folder_stack)
{
indexable.AddProperty(Property.New("fixme:folder", folder));
}
return(indexable);
}
/////////////////////////////////////////////////
private Indexable NoteToIndexable(FileInfo file, Note note)
{
Indexable indexable = new Indexable(note.Uri);
indexable.ContentUri = UriFu.PathToFileUri(file.FullName);
indexable.Timestamp = note.timestamp;
indexable.HitType = "Note";
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty(Property.New("dc:title", note.subject));
indexable.AddProperty(Property.NewUnsearched("fixme:application", "tomboy"));
// FIXME: tagging is disabled in Tomboy-0.8.x and is planned for 0.10.0
foreach (string s in note.tags)
{
indexable.AddProperty(Property.New("note:tag", s));
}
// We remember the note's text so that we can stuff it in
// the TextCache later.
note_text_cache [note.Uri] = note.text;
StringReader reader = new StringReader(note.text);
indexable.SetTextReader(reader);
return(indexable);
}
private Indexable current_itemToIndexable()
{
// sanity check
if (current_item == null)
{
return(null);
}
//Log.Debug ("Indexing " + channel_link + ":" + current_item.Link);
Indexable indexable = new Indexable(new Uri(String.Format("feed:{0};item={1}", channel_link, current_item.Link)));
indexable.ParentUri = UriFu.PathToFileUri(feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
string RFC822 = "ddd, dd MMM yyyy HH:mm:ss zzz";
DateTime date = DateTime.ParseExact(current_item.PubDate, RFC822, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal);
indexable.Timestamp = date;
// replace property names with Dublin Core names
indexable.AddProperty(Property.New("dc:title", current_item.Title));
indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Link));
indexable.AddProperty(Property.NewKeyword("dc:source", channel_link));
indexable.AddProperty(Property.New("dc:publisher", channel_title));
StringReader reader = new StringReader(current_item.Description);
indexable.SetTextReader(reader);
return(indexable);
}
static public ICollection CreateFiltersFromIndexable(Indexable indexable)
{
string path = indexable.ContentUri.LocalPath;
string extension = Path.GetExtension(path);
string mime_type = indexable.MimeType;
return(CreateFilters(UriFu.PathToFileUri(path), extension, mime_type));
}
public Indexable GetNextIndexable()
{
DateTime line_dt = DateTime.MinValue;
while (log_line_as_sb != null)
{
//Log.Debug ("Checking line from {0}:[{1}]", log_file, log_line_as_sb);
bool in_session = AppendLogText(log_line_as_sb, out line_dt);
if (!in_session)
{
break;
}
prev_line_offset = reader.Position;
log_line_as_sb = reader.ReadLineAsStringBuilder();
}
// Check if there is new data to index
if (data_sb.Length == 0)
{
session_begin_time = line_dt;
return(null);
}
Uri uri = new Uri(String.Format("konversation://{0}@dumb/{1}", session_begin_offset, log_file));
Log.Debug("Creating indexable {0}", uri);
Indexable indexable = new Indexable(uri);
indexable.ParentUri = UriFu.PathToFileUri(log_file);
indexable.Timestamp = session_begin_time;
indexable.HitType = "IMLog";
indexable.CacheContent = false;
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_begin_offset", session_begin_offset));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_end_offset", prev_line_offset));
indexable.AddProperty(Beagle.Property.NewDate("fixme:starttime", session_begin_time));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:client", "Konversation"));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:protocol", "IRC"));
AddChannelInformation(indexable);
foreach (string speaker in speakers.Keys)
{
indexable.AddProperty(Beagle.Property.NewUnstored("fixme:speaker", speaker));
}
StringReader data_reader = new StringReader(data_sb.ToString());
indexable.SetTextReader(data_reader);
// update session begin time to the date of the current line which is not in this session
session_begin_time = line_dt;
return(indexable);
}
/**
* deleting mbox means deleting all the mails which were in this mbox
* we use the idea of parent-uri
* while creating indexables, we set the parent uri to be the uri of the mbox file
* so to delete all mails in the mbox we just delete all documents whose parent uri
* is the uri of the mbox file
*/
public void RemoveMbox(string file)
{
Logger.Log.Debug("Removing mbox:" + file);
Uri uri = UriFu.PathToFileUri(file);
Scheduler.Task task = queryable.NewRemoveTask(uri);
task.Priority = Scheduler.Priority.Immediate;
task.SubPriority = 0;
queryable.ThisScheduler.Add(task);
}
private void RemoveFeedFile(string file)
{
Log.Debug("Removing Akregator feedfile:" + file);
Uri uri = UriFu.PathToFileUri(file);
Scheduler.Task task = NewRemoveTask(uri);
task.Priority = Scheduler.Priority.Immediate;
task.SubPriority = 0;
ThisScheduler.Add(task);
}
/////////////////////////////////////////////////
private static Indexable NoteToIndexable(FileInfo file)
{
Indexable indexable = new Indexable(UriFu.PathToFileUri(file.FullName));
indexable.Timestamp = file.LastWriteTimeUtc;
indexable.HitType = "Note";
indexable.MimeType = "x-beagle/x-labyrinth-note";
indexable.AddProperty(Property.NewUnsearched("fixme:application", "labyrinth"));
return(indexable);
}
/////////////////////////////////////////////////
private static Indexable ImLogToIndexable(string filename)
{
Uri uri = UriFu.PathToFileUri(filename);
Indexable indexable = new Indexable(uri);
indexable.ContentUri = uri;
indexable.Timestamp = File.GetLastWriteTimeUtc(filename);
indexable.MimeType = "beagle/x-empathy-log";
indexable.HitType = "IMLog";
indexable.CacheContent = true;
return(indexable);
}
static Uri PathToUri(string fullpath)
{
if (!arg_removable)
{
return(UriFu.PathToFileUri(fullpath));
}
fullpath = PathInIndex(fullpath);
return(new Uri(String.Format("removable{0}{1}{2}",
Uri.SchemeDelimiter,
volume_label,
StringFu.HexEscape(fullpath)), true));
}
private Indexable current_itemToIndexable()
{
Indexable indexable;
try {
indexable = new Indexable(new Uri(String.Format("{0};item={1}", feed_source, current_item.Source)));
} catch (System.UriFormatException) {
indexable = new Indexable(new Uri(String.Format("liferea://dummy?{0};item={1}", feed_source, current_item.Source)));
}
indexable.ParentUri = UriFu.PathToFileUri(feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
DateTime date = DateTimeUtil.UnixToDateTimeUtc(0);
date = date.AddSeconds(current_item.Timestamp);
indexable.Timestamp = date;
// cleaning up the property names as far as possible
// this way querying for specific field is possible
// following DC element names wherever applicable
indexable.AddProperty(Property.New("dc:title", current_item.Title));
Attribute[] attribs = current_item.Attribs.AttribArray;
if (attribs != null)
{
foreach (Attribute attrib in attribs)
{
if (attrib.Name != "author")
{
continue;
}
indexable.AddProperty(Property.New("dc:creator", attrib.Value));
}
}
indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Source));
indexable.AddProperty(Property.NewKeyword("dc:source", feed_source));
indexable.AddProperty(Property.New("dc:publisher", publisher));
if (File.Exists(icon_file))
{
indexable.AddProperty(Property.NewUnsearched("fixme:cachedimg", icon_file));
}
StringReader reader = new StringReader(current_item.Description);
indexable.SetTextReader(reader);
return(indexable);
}
/**
* Create an indexable from a maildir message
*/
public Indexable MaildirMessageToIndexable(string filename, bool crawl)
{
//Logger.Log.Debug ("+ indexing maildir mail:" + filename);
String folder = GetFolderMaildir(filename);
Uri file_uri = UriFu.PathToFileUri(filename);
Indexable indexable = new Indexable(file_uri);
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.FlushBufferCache = crawl;
indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail"));
indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name));
indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder));
indexable.ContentUri = file_uri;
return(indexable);
}
private bool HitIsValidRemovable(Hit hit)
{
if (mount_dir == null || hit.Uri.Scheme != "removable")
{
return(true);
}
hit.AddProperty(Beagrep.Property.NewKeyword("beagrep:RemovableUri",
hit.EscapedUri));
string path = hit.Uri.LocalPath;
path = path.Substring(1); // Remove initial '/'
path = Path.Combine(mount_dir, path);
hit.Uri = UriFu.PathToFileUri(path);
hit.AddProperty(Beagrep.Property.NewKeyword("fixme:mount_dir", mount_dir));
return(true);
}
//////////////////////////
private static Uri TextReaderToTempFileUri(TextReader reader)
{
if (reader == null)
{
return(null);
}
string filename = Path.GetTempFileName();
FileStream fileStream = File.OpenWrite(filename);
// When we dump the contents of an indexable into a file, we
// expect to use it again soon.
FileAdvise.PreLoad(fileStream);
// Make sure the temporary file is only readable by the owner.
// FIXME: There is probably a race here. Could some malicious program
// do something to the file between creation and the chmod?
Mono.Unix.Native.Syscall.chmod(filename, Mono.Unix.Native.FilePermissions.S_IRUSR);
BufferedStream bufferedStream = new BufferedStream(fileStream);
StreamWriter writer = new StreamWriter(bufferedStream);
char [] buffer;
buffer = GetCharBuffer();
int read;
do
{
read = reader.Read(buffer, 0, buffer.Length);
if (read > 0)
{
writer.Write(buffer, 0, read);
}
} while (read > 0);
writer.Close();
return(UriFu.PathToFileUri(filename));
}
/////////////////////////////////////////////////
private Indexable PidginLogToIndexable(string filename)
{
FileInfo info = new FileInfo(filename);
Uri uri = UriFu.PathToFileUri(filename);
Indexable indexable = new Indexable(uri);
indexable.ContentUri = uri;
indexable.Timestamp = info.LastWriteTimeUtc;
indexable.MimeType = "beagle/x-pidgin-log";
indexable.HitType = "IMLog";
indexable.CacheContent = false;
ImBuddy buddy = queryable.ImBuddyListReader.Search(info.Directory.Name);
if (buddy != null && !String.IsNullOrEmpty(buddy.Alias))
{
indexable.AddProperty(Property.NewKeyword("fixme:alias", buddy.Alias));
}
return(indexable);
}
private Indexable MailToIndexable(TB.Mail mail)
{
Indexable indexable;
GMime.Message message = mail.Message;
FullIndex = mail.GetBool("FullIndex"); // Make sure this is up to date
string mailbox = (MailboxName != null ? MailboxName : (string)mail.GetString("mailbox"));
indexable = NewIndexable(mail.Uri, message.Date.ToUniversalTime(), "MailMessage");
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty(Property.NewKeyword("fixme:folder", mailbox));
indexable.SetBinaryStream(message.Stream);
if (mail.GetBool("FullIndex"))
{
indexable.ContentUri = UriFu.PathToFileUri(Thunderbird.GetFullyIndexableFile(DbFile));
}
message.Dispose();
return(indexable);
}
static Indexable FileToIndexable(FileInfo file)
{
if (!file.Exists)
{
return(null);
}
if (fa_store.IsUpToDateAndFiltered(PathInIndex(file.FullName),
FileSystem.GetLastWriteTimeUtc(file.FullName)))
{
return(null);
}
// Create the indexable and add the standard properties we
// use in the FileSystemQueryable.
Uri uri = PathToUri(file.FullName);
Indexable indexable = new Indexable(uri);
indexable.Timestamp = file.LastWriteTimeUtc;
indexable.FlushBufferCache = true;
indexable.AddProperty(Property.NewUnsearched("fixme:filesize", file.Length));
FSQ.AddStandardPropertiesToIndexable(indexable, file.Name, Guid.Empty, false);
// Store directory name in the index
string dirname = file.DirectoryName;
indexable.AddProperty(Property.NewUnsearched(Property.ParentDirUriPropKey, PathToUri(dirname)));
if (arg_removable)
{
indexable.AddProperty(Property.NewKeyword("beagle:RemovableVolume", volume_label));
indexable.ContentUri = UriFu.PathToFileUri(file.FullName);
}
return(indexable);
}
static void Main(String[] args)
{
string uriStr = null;
string title = null;
string sourcefile = null;
bool deletesourcefile = false;
if (args.Length == 0 || Array.IndexOf(args, "--help") > -1)
{
PrintUsage();
Environment.Exit(1);
}
for (int i = 0; i < args.Length; i++)
{
switch (args [i])
{
case "--url":
case "--title":
case "--sourcefile":
if (i + 1 >= args.Length ||
args [i + 1].StartsWith("--"))
{
PrintUsage();
Environment.Exit(1);
}
break;
}
switch (args [i])
{
case "--url":
uriStr = args [++i];
break;
case "--title":
title = args [++i];
break;
case "--sourcefile":
sourcefile = args [++i];
break;
case "--deletesourcefile":
deletesourcefile = true;
break;
case "--help":
PrintUsage();
return;
case "--version":
VersionFu.PrintVersion();
return;
}
}
if (uriStr == null)
{
Logger.Log.Error("URI not specified!\n");
PrintUsage();
Environment.Exit(1);
}
Uri uri = new Uri(uriStr, true);
if (uri.Scheme == Uri.UriSchemeHttps)
{
// For security/privacy reasons, we don't index any
// SSL-encrypted pages.
Logger.Log.Error("Indexing secure https:// URIs is not secure!");
Environment.Exit(1);
}
// We don't index file: Uris. Silently exit.
if (uri.IsFile)
{
return;
}
// We *definitely* don't index mailto: Uris. Silently exit.
if (uri.Scheme == Uri.UriSchemeMailto)
{
return;
}
Indexable indexable;
indexable = new Indexable(uri);
indexable.HitType = "WebHistory";
indexable.MimeType = "text/html";
indexable.Timestamp = DateTime.Now;
if (title != null)
{
indexable.AddProperty(Property.New("dc:title", title));
}
if (sourcefile != null)
{
if (!File.Exists(sourcefile))
{
Logger.Log.Error("sourcefile '{0}' does not exist!", sourcefile);
Environment.Exit(1);
}
indexable.ContentUri = UriFu.PathToFileUri(sourcefile);
indexable.DeleteContent = deletesourcefile;
}
else
{
Stream stdin = Console.OpenStandardInput();
if (stdin == null)
{
Logger.Log.Error("No sourcefile specified, and no standard input!\n");
PrintUsage();
Environment.Exit(1);
}
indexable.SetTextReader(new StreamReader(stdin));
}
IndexingServiceRequest req = new IndexingServiceRequest();
req.Add(indexable);
try {
Logger.Log.Info("Indexing");
Logger.Log.Debug("SendAsync");
req.SendAsync();
Logger.Log.Debug("Close");
req.Close();
Logger.Log.Debug("Done");
} catch (Exception e) {
Logger.Log.Error("Indexing failed: {0}", e);
// Still clean up after ourselves, even if we couldn't
// index the content.
if (deletesourcefile)
{
File.Delete(sourcefile);
}
Environment.Exit(1);
}
}
public Indexable GetNextIndexable()
{
string line;
string_builder.Length = 0;
DateTime dt = DateTime.MinValue;
string uid = null;
// Keep reading till "END:EVENT" or "END:VTODO"
while ((line = reader.ReadLine()) != null)
{
//UID:libkcal-1467827482.768
//LAST-MODIFIED:20061015T085606Z
if (line == "END:VEVENT" || line == "END:VTODO")
{
break;
}
else if (line.StartsWith("UID:"))
{
uid = line.Substring(4);
}
else if (line.StartsWith("LAST-MODIFIED:"))
{
string dt_string = line.Substring(14);
dt_string = dt_string.Replace("Z", "+00:00");
dt = DateTime.ParseExact(
dt_string,
fmts,
DateTimeFormatInfo.InvariantInfo,
DateTimeStyles.AdjustToUniversal);
}
else
{
string_builder.Append(line);
string_builder.Append('\n');
}
}
if (line == null)
{
reader.Close();
return(null);
}
// Bad entry
if (string_builder.Length == 0 ||
uid == null ||
dt == DateTime.MinValue)
{
return(null);
}
// Mark entry with uid as seen ('undeleted')
deleted_entries [uid] = false;
lock (last_modified_table) {
if (last_modified_table.Contains(uid))
{
DateTime old_dt = (DateTime)last_modified_table [uid];
// FIXME: Returning null for more than 179 times will cause trouble
if (dt == old_dt)
{
return(null);
}
else
{
//Log.Debug ("Updating last_mod_date [{0}] = {1}", uid, dt);
last_modified_table [uid] = dt;
}
}
else
{
//Log.Debug ("Adding last_mod_date [{0}] = {1}", uid, dt);
last_modified_table [uid] = dt;
}
}
if (initial_scan)
{
return(null);
}
Uri uri = new Uri(String.Format("korganizer:///{0}", uid));
Indexable indexable = new Indexable(uri);
indexable.ParentUri = UriFu.PathToFileUri(korganizer_file);
if (line == "END:VEVENT")
{
indexable.MimeType = ICalParser.KOrganizerEventMimeType;
indexable.HitType = "Calendar";
}
else
{
indexable.MimeType = ICalParser.KOrganizerTodoMimeType;
indexable.HitType = "Task";
}
indexable.Timestamp = dt;
// Add uid as a keyword field for convenience
indexable.AddProperty(Property.NewUnsearched("fixme:uid", uid));
// FIXME: Comment this Debug statement after the backend stabilizes
//Log.Debug ("Creating {0} from:[{1}]", uri, string_builder.ToString ());
StringReader string_reader = new StringReader(string_builder.ToString());
indexable.SetTextReader(string_reader);
return(indexable);
}
public Indexable GetXmpQueryable(string path, Guid id, DirectoryModel parent)
{
Log.Debug("Asked to create xmp indexable for ({0}) {1}", GuidFu.ToShortString(id), path);
// Should be at least 6 characters /<...>.xmp
if (path.Length < 6)
{
return(null);
}
string basefile_path = Path.ChangeExtension(path, null);
// Ignore xmp files by itself
// FIXME: To support indexing independent xmp files will require even greater trouble
if (!File.Exists(basefile_path))
{
return(null);
}
XmpFile xmp_file = null;
try {
xmp_file = new XmpFile(path);
} catch {
Log.Warn("Cannot create xmpfile from {0}", path);
return(null);
}
// Try to get the correct uid for the basefile
// First we need to see if basefile is already scheduled (yet to be dispatched)
Uri basefile_uri = null;
Indexable base_indexable;
if (uid_manager.HasNewId(basefile_path))
{
// Since uid_manager has a new id for this basefile, so basefile is already scheduled
// Get basefile uid from there
Guid basefile_id = uid_manager.GetNewId(basefile_path);
basefile_uri = GuidFu.ToUri(basefile_id);
Log.Debug("{0} is already scheduled with uri {1}", basefile_path, basefile_uri);
}
else
{
// Basefile is not scheduled in the current batch
string basefile_name = Path.GetFileName(basefile_path);
// Try to schedule it for addition
base_indexable = queryable.GetCrawlingFileIndexable(parent, basefile_name);
if (base_indexable == null)
{
// GetCrawlingFileIndexable returns null if file does not need to be indexed
// So basefile is up-to-date
// Need to figure out id from uid manager
Guid basefile_id = uid_manager.GetIdByNameAndParentId(basefile_name, parent.UniqueId);
basefile_uri = GuidFu.ToUri(basefile_id);
Log.Debug("{0} is not scheduled and need not be, uri is {1}", basefile_path, basefile_uri);
}
else
{
Log.Debug("Need to index {0}", basefile_path);
// basefile needs to be indexed
// FIXME: Move the task business out of handler and into FSQ.cs
Scheduler.Task task;
task = queryable.NewAddTask(base_indexable);
// FIXME: What is the correct priority ?
// If should have similar priority to the one that this xmp-indexable will be a part of
task.Priority = Scheduler.Priority.Immediate;
queryable.ThisScheduler.Add(task);
// Get the basefile uri from the indexable
basefile_uri = base_indexable.Uri;
}
}
Log.Debug("Adding xmp-indexable for {0} (basefile uri {1}) with uid {2}",
path,
basefile_uri,
GuidFu.ToShortString(id));
Indexable indexable = new Indexable(IndexableType.PropertyChange, basefile_uri);
// Set the timestamp of the indexable as the timestamp of the basefile
// It could have also been skipped, the original Indexable.Add would anyway have it
indexable.Timestamp = File.GetLastWriteTimeUtc(basefile_path);
indexable.DisplayUri = UriFu.PathToFileUri(path);
// If the file was somehow deleted before this point, bail out.
if (!FileSystem.ExistsByDateTime(indexable.Timestamp))
{
xmp_file.Close();
return(null);
}
// Save some local states for PostAddHook, namely, path to the xmp file, path to basefile and generated uid
indexable.LocalState ["XmpFilePath"] = path;
indexable.LocalState ["BaseFilePath"] = basefile_path;
indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString(id);
// FIXME: Should also delete previous xmp properties!
foreach (Property p in xmp_file.Properties)
{
p.IsMutable = true;
indexable.AddProperty(p);
}
xmp_file.Close();
return(indexable);
}
/**
* Create an indexable from an mbox message
* Most of the code here is from Evo backend
*/
public Indexable MessageToIndexable(string file_name, System.Uri uri, GMime.Message message, string folder_name)
{
//Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name);
Indexable indexable = new Indexable(uri);
// set parent uri to the filename so that when an mbox file
// is deleted, all the messages in that file can be deleted
indexable.ParentUri = UriFu.PathToFileUri(file_name);
indexable.Timestamp = message.Date.ToUniversalTime();
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail"));
indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name));
indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder_name));
GMime.InternetAddressList addrs;
if (folder_name == Queryable.SentMailFolderName)
{
addrs = message.GetRecipients(GMime.RecipientType.To);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewKeyword("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
}
if (folder_name == Queryable.SentMailFolderName)
{
addrs = message.GetRecipients(GMime.RecipientType.Cc);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewKeyword("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
}
if (folder_name != Queryable.SentMailFolderName)
{
addrs = GMime.InternetAddressList.Parse(message.Sender);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewKeyword("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose();
}
if (folder_name == Queryable.SentMailFolderName)
{
indexable.AddProperty(Property.NewFlag("fixme:isSent"));
}
else
{
string kmail_msg_sent = message.GetHeader("X-KMail-Link-Type");
if (kmail_msg_sent == "reply")
{
indexable.AddProperty(Property.NewFlag("fixme:isSent"));
}
}
// no need to store date again, use the issent flag to determine if the date is sentdate or not
#if false
if (folder_name == Queryable.SentMailFolderName)
{
indexable.AddProperty(Property.NewDate("fixme:sentdate", message.Date.ToUniversalTime()));
}
else
{
indexable.AddProperty(Property.NewDate("fixme:received", message.Date.ToUniversalTime()));
}
#endif
indexable.SetBinaryStream(message.Stream);
return(indexable);
}
static void Main(string [] args)
{
Mode mode = Mode.Uris;
bool show_counts = true;
ArrayList index_dirs = new ArrayList();
ArrayList index_names = new ArrayList();
ArrayList uris = new ArrayList();
foreach (string arg in args)
{
switch (arg)
{
case "--help":
PrintUsage();
Environment.Exit(0);
break;
case "--uris":
mode = Mode.Uris;
break;
case "--properties":
case "--props":
mode = Mode.Properties;
break;
case "--term-frequencies":
case "--term-freqs":
mode = Mode.TermFrequencies;
break;
case "--hide-counts":
show_counts = false;
break;
case "--show-counts":
show_counts = false;
break;
case "--fields":
mode = Mode.Fields;
break;
default:
if (arg.StartsWith("--indexdir="))
{
index_dirs.Add(arg.Remove(0, 11));
}
else if (arg.StartsWith("--index="))
{
index_names.Add(arg.Remove(0, 8));
}
else
{
Uri uri;
try {
uri = UriFu.UserUritoEscapedUri(arg);
} catch (UriFormatException) {
uri = UriFu.PathToFileUri(arg);
}
uris.Add(uri);
}
break;
}
}
if (uris.Count > 0 && (mode == Mode.TermFrequencies || mode == Mode.Fields))
{
Console.WriteLine("ERROR: --term-frequencies and --fields do not make sense with files or URIs.");
Environment.Exit(1);
}
ArrayList indexes = new ArrayList();
// If no --index or --indexdir options, get all the default indexes.
if (index_dirs.Count == 0 && index_names.Count == 0)
{
foreach (DirectoryInfo subdir in DirectoryWalker.GetDirectoryInfos(PathFinder.IndexDir))
{
indexes.Add(new IndexInfo(subdir.Name));
}
}
else
{
foreach (string name in index_names)
{
DirectoryInfo info = new DirectoryInfo(Path.Combine(PathFinder.IndexDir, name));
if (!info.Exists)
{
Console.WriteLine("ERROR: No index named '{0}'", name);
Environment.Exit(1);
}
indexes.Add(new IndexInfo(info.Name));
}
foreach (string dir in index_dirs)
{
indexes.Add(new IndexInfo(dir));
}
}
indexes.Sort();
if (mode == Mode.Uris || mode == Mode.Properties)
{
DumpIndexInformation(indexes, uris, mode == Mode.Properties, show_counts);
}
else if (mode == Mode.TermFrequencies)
{
DumpIndexTermFreqs(indexes);
}
else if (mode == Mode.Fields)
{
DumpIndexFields(indexes);
}
}
static int DumpOneIndex_Metadata(string index_name, ArrayList uris, bool show_properties)
{
LuceneQueryingDriver driver;
driver = new LuceneQueryingDriver(index_name, -1, true);
Hashtable all_hits_by_uri = null;
ArrayList all_hits = null;
if (uris.Count == 0 || index_name == "FileSystemIndex")
{
all_hits_by_uri = driver.GetAllHitsByUri();
all_hits = new ArrayList(all_hits_by_uri.Values);
}
// A hard-wired hack
if (index_name == "FileSystemIndex")
{
foreach (Hit hit in all_hits)
{
string internal_uri;
if (hit [Property.IsChildPropKey] == "true")
{
string path = RemapUriToPath(all_hits_by_uri, hit);
internal_uri = UriFu.UriToEscapedString(hit.ParentUri);
hit.ParentUri = UriFu.PathToFileUri(path);
hit.Uri = UriFu.AddFragment(UriFu.PathToFileUri(path),
hit.Uri.Fragment,
true);
}
else
{
internal_uri = UriFu.UriToEscapedString(hit.Uri);
hit.Uri = UriFu.PathToFileUri(RemapUriToPath(all_hits_by_uri, hit));
hit.AddProperty(Property.NewUnsearched("beagrep:InternalUri", internal_uri));
}
}
}
ArrayList matching_hits;
if (uris.Count == 0)
{
matching_hits = all_hits;
}
else
{
matching_hits = new ArrayList(driver.GetHitsForUris(RemapUris(driver, uris)));
if (index_name == "FileSystemIndex")
{
for (int i = 0; i < matching_hits.Count; i++)
{
Hit hit = (Hit)matching_hits [i];
Hit mapped_hit = (Hit)all_hits_by_uri [hit.Uri];
matching_hits [i] = mapped_hit;
}
}
}
matching_hits.Sort(new HitByUriComparer());
foreach (Hit hit in matching_hits)
{
if (!show_properties)
{
Console.WriteLine("{0}: {1}", index_name, hit.Uri);
continue;
}
Console.WriteLine(" Index: {0}", index_name);
Console.WriteLine(" Uri: {0}", hit.Uri);
if (hit.ParentUri != null)
{
Console.WriteLine("Parent: {0}", hit.ParentUri);
}
Console.WriteLine(" MimeT: {0}", hit.MimeType);
Console.WriteLine(" Type: {0}", hit.Type);
Console.WriteLine("Source: {0}", hit.Source);
ArrayList props;
props = new ArrayList(hit.Properties);
props.Sort();
foreach (Property prop in props)
{
char [] legend = new char [4];
legend [0] = prop.IsMutable ? 'm' : ' ';
legend [1] = prop.IsSearched ? 's' : ' ';
legend [2] = prop.IsPersistent ? 'p' : ' ';
legend [3] = prop.Type == PropertyType.Text ? 't' : ' ';
Console.WriteLine(" Prop: [{0}] {1} = '{2}'", new String(legend), prop.Key, prop.Value);
}
Console.WriteLine();
}
return(matching_hits.Count);
}
private void IndexLauncher(FileInfo file, Scheduler.Priority priority)
{
if ((!file.Exists) ||
(this.FileAttributesStore.IsUpToDate(file.FullName)))
{
return;
}
/* Check to see if file is a launcher */
if (Beagle.Util.VFS.Mime.GetMimeType(file.FullName) != "application/x-desktop")
{
return;
}
StreamReader reader;
try {
reader = new StreamReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read));
} catch (Exception e) {
log.Warn("Could not open '{0}': {1}", file.FullName, e.Message);
return;
}
if (reader.ReadLine() != "[Desktop Entry]")
{
reader.Close();
return;
}
/* I'm convinced it is a launcher */
Indexable indexable = new Indexable(UriFu.PathToFileUri(file.FullName));
indexable.Timestamp = file.LastWriteTime;
indexable.Type = "Launcher";
indexable.MimeType = "application/x-desktop";
// desktop files must have a name
bool have_name = false;
String line;
while ((line = reader.ReadLine()) != null)
{
string [] sline = line.Split('=');
if (sline.Length != 2)
{
continue;
}
// FIXME: We shouldnt really search fields that are in other locales than the current should we?
if (sline [0].Equals("Icon") || sline [0].Equals("Exec"))
{
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:" + sline[0], sline[1]));
}
else if (sline [0].StartsWith("Name"))
{
if (sline [0] == "Name")
{
have_name = true;
}
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:" + sline[0], sline[1]));
}
else if (sline[0].StartsWith("Comment"))
{
indexable.AddProperty(Beagle.Property.New("fixme:" + sline[0], sline[1]));
}
}
reader.Close();
if (have_name)
{
Scheduler.Task task = NewAddTask(indexable);
task.Priority = priority;
ThisScheduler.Add(task);
}
}
static Uri PathToUri(string fullpath)
{
return(UriFu.PathToFileUri(fullpath));
}
private Indexable CamelMessageToIndexable(B_U_Camel.MessageInfo messageInfo, string msg_file)
{
// Don't index messages flagged as junk
if (messageInfo.IsJunk)
{
return(null);
}
// Many properties will be set by the filter when
// processing the cached data, if it's there. So
// don't set a number of properties in that case.
bool have_content = File.Exists(msg_file);
Uri uri = CamelMessageUri(messageInfo);
Indexable indexable = new Indexable(uri);
indexable.Timestamp = messageInfo.SentDate;
indexable.MimeType = "message/rfc822";
indexable.HitType = "MailMessage";
indexable.AddProperty(Property.NewUnsearched("fixme:account", this.imap_name));
indexable.AddProperty(Property.NewUnsearched("fixme:folder", this.folder_name));
indexable.AddProperty(Property.NewUnsearched("fixme:client", "evolution"));
if (!have_content)
{
indexable.AddProperty(Property.New("dc:title", GMime.Utils.HeaderDecodePhrase(messageInfo.subject)));
indexable.AddProperty(Property.NewDate("fixme:date", messageInfo.SentDate));
}
GMime.InternetAddressList addrs;
addrs = GMime.InternetAddressList.Parse(messageInfo.to);
foreach (GMime.InternetAddress ia in addrs)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
if (!have_content)
{
indexable.AddProperty(Property.NewUnsearched("fixme:to", ia.ToString(false)));
if (ia is GMime.InternetAddressMailbox)
{
indexable.AddProperty(Property.New("fixme:to_address", mailbox.Address));
}
indexable.AddProperty(Property.New("fixme:to_name", ia.Name));
}
if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox)
{
indexable.AddProperty(Property.NewUnsearched("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
addrs = GMime.InternetAddressList.Parse(messageInfo.cc);
foreach (GMime.InternetAddress ia in addrs)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
if (!have_content)
{
indexable.AddProperty(Property.NewUnsearched("fixme:cc", ia.ToString(false)));
if (ia is GMime.InternetAddressMailbox)
{
indexable.AddProperty(Property.New("fixme:cc_address", mailbox.Address));
}
indexable.AddProperty(Property.New("fixme:cc_name", ia.Name));
}
if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox)
{
indexable.AddProperty(Property.NewUnsearched("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
addrs = GMime.InternetAddressList.Parse(messageInfo.from);
foreach (GMime.InternetAddress ia in addrs)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
if (!have_content)
{
indexable.AddProperty(Property.NewUnsearched("fixme:from", ia.ToString(false)));
if (ia is GMime.InternetAddressMailbox)
{
indexable.AddProperty(Property.New("fixme:from_address", mailbox.Address));
}
indexable.AddProperty(Property.New("fixme:from_name", ia.Name));
}
if (this.folder_name != "Sent" && ia is GMime.InternetAddressMailbox)
{
indexable.AddProperty(Property.NewUnsearched("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose();
indexable.AddProperty(Property.NewKeyword("fixme:mlist", messageInfo.mlist));
Property flag_prop = Property.NewUnsearched("fixme:flags", messageInfo.flags);
flag_prop.IsMutable = true;
indexable.AddProperty(flag_prop);
if (this.folder_name == "Sent")
{
indexable.AddProperty(Property.NewFlag("fixme:isSent"));
}
if (messageInfo.IsAnswered)
{
indexable.AddProperty(Property.NewFlag("fixme:isAnswered"));
}
if (messageInfo.IsDeleted)
{
indexable.AddProperty(Property.NewFlag("fixme:isDeleted"));
}
if (messageInfo.IsDraft)
{
indexable.AddProperty(Property.NewFlag("fixme:isDraft"));
}
if (messageInfo.IsFlagged)
{
indexable.AddProperty(Property.NewFlag("fixme:isFlagged"));
}
if (messageInfo.IsSeen)
{
indexable.AddProperty(Property.NewFlag("fixme:isSeen"));
}
if (messageInfo.HasAttachments && !have_content)
{
indexable.AddProperty(Property.NewFlag("fixme:hasAttachments"));
}
if (messageInfo.IsAnsweredAll)
{
indexable.AddProperty(Property.NewFlag("fixme:isAnsweredAll"));
}
if (have_content)
{
indexable.ContentUri = UriFu.PathToFileUri(msg_file);
}
else
{
indexable.NoContent = true;
}
return(indexable);
}
static int Main(string[] args)
{
SystemInformation.SetProcessName ("beagle-extract-content");
if (args.Length < 1 || Array.IndexOf (args, "--help") != -1) {
PrintUsage ();
return 0;
}
if (Array.IndexOf (args, "--debug") == -1)
Log.Disable ();
if (Array.IndexOf (args, "--version") != -1) {
VersionFu.PrintVersion ();
return 0;
}
if (Array.IndexOf (args, "--tokenize") != -1)
tokenize = true;
if (Array.IndexOf (args, "--analyze") != -1)
analyze = true;
if (Array.IndexOf (args, "--show-generated") != -1 || Array.IndexOf (args, "--show-children") != -1)
show_generated = true;
StreamWriter writer = null;
string outfile = null;
foreach (string arg in args) {
// mime-type option
if (arg.StartsWith ("--mimetype=")) {
mime_type = arg.Substring (11);
continue;
// output file option
// we need this in case the output contains different encoding
// printing to Console might not always display properly
} else if (arg.StartsWith ("--outfile=")) {
outfile = arg.Substring (10);
Console.WriteLine ("Redirecting output to " + outfile);
FileStream f = new FileStream (outfile, FileMode.Create);
writer = new StreamWriter (f, System.Text.Encoding.UTF8);
continue;
} else if (arg.StartsWith ("--")) // option, skip it
continue;
Uri uri = UriFu.PathToFileUri (arg);
Indexable indexable = new Indexable (uri);
if (mime_type != null)
indexable.MimeType = mime_type;
try {
if (writer != null) {
Console.SetOut (writer);
}
Display (indexable);
if (writer != null) {
writer.Flush ();
}
if (outfile != null) {
StreamWriter standardOutput = new StreamWriter(Console.OpenStandardOutput());
standardOutput.AutoFlush = true;
Console.SetOut(standardOutput);
}
} catch (Exception e) {
Console.WriteLine ("Unable to filter {0}: {1}", uri, e.Message);
return -1;
}
// Super Lame Hack: gtk-sharp up to 2.10 requires a main loop
// to dispose of any managed wrappers around GObjects. Since
// we don't have one, we'll process all the pending items in
// a loop here. This is particularly an issue with maildirs,
// because we need the loop to clean up after GMime. Without
// it, GMime's streams are never completely unref'd, the
// file descriptors aren't closed, and we run out and crash.
while (GLib.MainContext.Pending ())
GLib.MainContext.Iteration ();
}
if (writer != null)
writer.Close ();
return 0;
}