/////////////////////////////////////////////////
private Indexable NoteToIndexable(FileInfo file, Note note)
{
Indexable indexable = new Indexable(note.Uri);
indexable.ContentUri = UriFu.PathToFileUri(file.FullName);
indexable.Timestamp = note.timestamp;
indexable.HitType = "Note";
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty(Property.New("dc:title", note.subject));
indexable.AddProperty(Property.NewUnsearched("fixme:application", "tomboy"));
// FIXME: tagging is disabled in Tomboy-0.8.x and is planned for 0.10.0
foreach (string s in note.tags)
{
indexable.AddProperty(Property.New("note:tag", s));
}
// We remember the note's text so that we can stuff it in
// the TextCache later.
note_text_cache [note.Uri] = note.text;
StringReader reader = new StringReader(note.text);
indexable.SetTextReader(reader);
return(indexable);
}
private Indexable BookmarkToIndexable()
{
Indexable indexable = new Indexable(new Uri(current_bookmark_id));
indexable.ParentUri = UriFu.PathToFileUri(bookmark_file);
indexable.HitType = "Bookmark";
indexable.Timestamp = current_dt;
indexable.NoContent = true;
indexable.AddProperty(Property.New("dc:title", current_bookmark.Title));
indexable.AddProperty(Property.New("dc:identifier", current_bookmark.Href));
indexable.AddProperty(Property.NewUnsearched("fixme:icon", current_bookmark.Icon));
if (current_bookmark.Info != null &&
current_bookmark.Info.Metadata != null &&
current_bookmark.Info.Metadata.NumVisited != 0)
{
indexable.AddProperty(Property.NewUnsearched("fixme:visit_count", current_bookmark.Info.Metadata.NumVisited));
}
foreach (string folder in folder_stack)
{
indexable.AddProperty(Property.New("fixme:folder", folder));
}
return(indexable);
}
private Indexable current_itemToIndexable()
{
// sanity check
if (current_item == null)
{
return(null);
}
//Log.Debug ("Indexing " + channel_link + ":" + current_item.Link);
Indexable indexable = new Indexable(new Uri(String.Format("feed:{0};item={1}", channel_link, current_item.Link)));
indexable.ParentUri = UriFu.PathToFileUri(feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
string RFC822 = "ddd, dd MMM yyyy HH:mm:ss zzz";
DateTime date = DateTime.ParseExact(current_item.PubDate, RFC822, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal);
indexable.Timestamp = date;
// replace property names with Dublin Core names
indexable.AddProperty(Property.New("dc:title", current_item.Title));
indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Link));
indexable.AddProperty(Property.NewKeyword("dc:source", channel_link));
indexable.AddProperty(Property.New("dc:publisher", channel_title));
StringReader reader = new StringReader(current_item.Description);
indexable.SetTextReader(reader);
return(indexable);
}
public Indexable GetNextIndexable()
{
if (initial_scan)
{
return(null);
}
Uri uri = new Uri(String.Format("kabc:///{0}", current_uid));
Indexable indexable = new Indexable(uri);
indexable.ParentUri = UriFu.PathToFileUri(kabc_file);
indexable.MimeType = ICalParser.KabcMimeType;
indexable.HitType = "Contact";
indexable.Timestamp = current_dt;
indexable.AddProperty(Property.NewKeyword("fixme:client", "KAddressBook"));
// Add uid as a keyword field for convenience
indexable.AddProperty(Property.NewUnsearched("fixme:uid", current_uid));
// FIXME: Comment this Debug statement after the backend stabilizes
//Log.Debug ("Creating {0} from:[{1}]", uri, string_builder.ToString ());
StringReader string_reader = new StringReader(string_builder.ToString());
indexable.SetTextReader(string_reader);
return(indexable);
}
private Indexable OperaRowToIndexable(OperaHistory.Row row)
{
// It's unsafe to index secure content since it may contain sensitive data
if (row.Address.Scheme == Uri.UriSchemeHttps)
{
return(null);
}
Indexable indexable = new Indexable(row.Address);
indexable.HitType = "WebHistory";
indexable.MimeType = row.MimeType;
indexable.Timestamp = row.LastVisited;
indexable.AddProperty(Beagle.Property.New("fixme:host", row.Address.Host));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:size", row.Length));
// hint for the filter about the charset
indexable.AddProperty(Property.NewUnsearched(StringFu.UnindexedNamespace + "charset", row.Encoding.ToString()));
if (row.Compression == "gzip")
{
indexable.SetBinaryStream(new GZipInputStream(File.OpenRead(Path.Combine(cache_dir, row.LocalFileName))));
}
else
{
indexable.ContentUri = new Uri(Path.Combine(cache_dir, row.LocalFileName));
}
indexer.AttributeStore.AttachLastWriteTime(Path.Combine(cache_dir, row.LocalFileName), DateTime.UtcNow);
return(indexable);
}
static Indexable FileToIndexable(FileInfo file)
{
if (!file.Exists)
{
return(null);
}
if (fa_store.IsUpToDateAndFiltered(PathInIndex(file.FullName),
FileSystem.GetLastWriteTimeUtc(file.FullName)))
{
return(null);
}
// Create the indexable and add the standard properties we
// use in the FileSystemQueryable.
Uri uri = PathToUri(file.FullName);
Indexable indexable = new Indexable(uri);
indexable.Timestamp = file.LastWriteTimeUtc;
indexable.FlushBufferCache = true;
indexable.AddProperty(Property.NewUnsearched("fixme:filesize", file.Length));
FSQ.AddStandardPropertiesToIndexable(indexable, file.Name, Guid.Empty, false);
// Store directory name in the index
string dirname = file.DirectoryName;
indexable.AddProperty(Property.NewUnsearched(Property.ParentDirUriPropKey, PathToUri(dirname)));
return(indexable);
}
private Indexable MemoToIndexable(CalComponent cc)
{
Indexable indexable = new Indexable(GetComponentUri(cc));
indexable.Timestamp = cc.Dtstart;
indexable.HitType = "Note";
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty(Property.NewUnsearched("fixme:application", "evolution"));
indexable.AddProperty(Property.New("dc:title", cc.Summary));
// We remember the note's text so that we can stuff it in
// the TextCache later.
// This is here form compability with Tomboy notes.
foreach (string description in cc.Descriptions)
{
queryable.IndexableTextCache [indexable.Uri] = description;
StringReader reader = new StringReader(description);
indexable.SetTextReader(reader);
}
return(indexable);
}
static Indexable DirectoryToIndexable(DirectoryInfo dir, Queue modified_directories)
{
if (!dir.Exists)
{
return(null);
}
// Check if the directory information is stored in attributes store
// And if the mtime of the directory is same as that in the attributes store
FileAttributes attr = fa_store.Read(PathInIndex(dir.FullName));
// If the directory exists in the fa store, then it is already indexed.
if (attr != null)
{
// If we don't care about deleted content then we are fine.
// If the attributes are up-to-date, then we are fine too.
if (!arg_delete || FileAttributesStore.IsUpToDate(attr, FileSystem.GetLastWriteTimeUtc(dir.FullName)))
{
return(null);
}
// But the last write time needs to be uptodate to support enable-deletion,
// so we actually index the directories, even if --disable-directories
// is set.
modified_directories.Enqueue(dir);
}
// Create the indexable and add the standard properties we
// use in the FileSystemQueryable.
Uri uri = PathToUri(dir.FullName);
Indexable indexable = new Indexable(uri);
indexable.MimeType = "inode/directory";
indexable.NoContent = true;
indexable.Timestamp = dir.LastWriteTimeUtc;
// Store the directory information in the index anyway, but if --disable-directories
// was passed, then do not store the names and other standard properties
// used during searching
if (!arg_disable_directories)
{
FSQ.AddStandardPropertiesToIndexable(indexable, dir.Name, Guid.Empty, false);
}
// Add directory name property
string dirname = dir.Parent.FullName;
indexable.AddProperty(Property.NewUnsearched(Property.ParentDirUriPropKey, PathToUri(dirname)));
indexable.AddProperty(Property.NewBool(Property.IsDirectoryPropKey, true));
if (arg_removable)
{
indexable.AddProperty(Property.NewKeyword("beagle:removable", volume_label));
}
return(indexable);
}
private void AddChannelInformation(Indexable indexable)
{
// FIXME: Parse identity information from konversation .config file
//AddProperty (Beagle.Property.NewUnsearched ("fixme:identity", log.Identity));
// Get server name, channel name from the filename and add it here
indexable.AddProperty(Beagle.Property.NewKeyword("fixme:server", server_name));
indexable.AddProperty(Beagle.Property.NewKeyword("fixme:speakingto", speaking_to));
}
private Indexable ToAddRssIndexable(XmlDocument document)
{
string encoding_str = null;
StringReader reader = null;
if (ToBool(GetText(document, "HasOffline")))
{
try {
// RSS does not use OfflineSize but MessageSize instead (for some reason...)
int offset = Convert.ToInt32(GetText(document, "MessageOffset")),
size = Convert.ToInt32(GetText(document, "MessageSize"));
reader = GetRssBody(GetText(document, "FolderFile"), offset, size, out encoding_str);
} catch (Exception e) {
Logger.Log.Debug(e, "Failed to parse RSS body");
}
}
Indexable indexable = new Indexable(GenerateUniqueUri(document));
indexable.HitType = "FeedItem";
indexable.MimeType = "text/html";
indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc(Convert.ToInt64(GetText(document, "Date")));
indexable.CacheContent = true;
indexable.FlushBufferCache = true;
indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird"));
indexable.AddProperty(Property.NewKeyword("fixme:folder", GetText(document, "Folder")));
indexable.AddProperty(Property.NewUnsearched("ParentUri", GetText(document, "FolderFile")));
indexable.AddProperty(Property.NewUnsearched("fixme:uri", GetText(document, "Uri")));
indexable.AddProperty(Property.NewKeyword("dc:identifier", ExtractUrl(GetText(document, "MessageId"))));
indexable.AddProperty(Property.NewKeyword("dc:source", GetText(document, "FeedURL")));
indexable.AddProperty(Property.New("dc:publisher", Mime.HeaderDecodePhrase(GetText(document, "Author"))));
// The title will be added by the filter. In case we add it twice we will just get
// an empty tile in the search tool (a bug maybe?).
if (reader != null)
{
// If we got an encoding, make sure we use that
if (!String.IsNullOrEmpty(encoding_str))
{
indexable.AddProperty(Property.New(
String.Format("{0}encoding", StringFu.UnindexedNamespace), encoding_str));
}
indexable.SetTextReader(reader);
}
else
{
indexable.AddProperty(Property.New("dc:title", Mime.HeaderDecodePhrase(GetText(document, "Subject"))));
}
return(indexable);
}
static private Indexable MemberNodeToIndexable(XmlNode node, Uri base_uri, string parentName)
{
char memberType = MemberTypeToChar(node.SelectSingleNode("MemberType").InnerText);
StringBuilder memberFullName = new StringBuilder();
memberFullName.Append(memberType + ":" + parentName);
if (memberType != 'C')
{
memberFullName.Append("." + node.Attributes["MemberName"].Value);
}
if (memberType == 'C' || memberType == 'M' || memberType == 'E')
{
memberFullName.Append("(");
bool inside = false;
foreach (XmlNode parameter in node.SelectNodes("Parameters/Parameter"))
{
if (!inside)
{
inside = true;
}
else
{
memberFullName.Append(",");
}
memberFullName.Append(parameter.Attributes["Type"].Value);
}
memberFullName.Append(")");
}
Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, memberFullName.ToString(), false));
indexable.MimeType = "text/html";
indexable.HitType = "MonodocEntry";
indexable.AddProperty(Property.New("dc:title", memberFullName.ToString()));
indexable.AddProperty(Property.New("fixme:name", memberFullName.ToString()));
indexable.AddProperty(Property.NewUnsearched("fixme:type", node.SelectSingleNode("MemberType").InnerText.ToLower()));
StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml);
indexable.SetTextReader(reader);
return(indexable);
}
protected virtual Indexable NewIndexable(Uri uri, DateTime timestamp, string hit_type)
{
Indexable indexable;
indexable = new Indexable(uri);
indexable.HitType = hit_type;
indexable.Timestamp = timestamp;
indexable.AddProperty(Property.NewKeyword("fixme:account", account.Server));
indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird"));
indexable.AddProperty(Property.NewUnsearched("fixme:fullyIndexed", full_index));
indexable.AddProperty(Property.NewUnsearched("fixme:file", RelativePath));
indexable.AddProperty(Property.NewDate("fixme:indexDateTime", DateTime.UtcNow));
return(indexable);
}
/////////////////////////////////////////////////////////////////
static void AddToRequest(Indexable indexable)
{
if (indexable == null)
{
return;
}
// Disable filtering and only index file attributes
if (arg_disable_filtering)
{
indexable.Filtering = IndexableFiltering.Never;
}
// Tag the item for easy identification (for say, removal)
if (arg_tag != null)
{
indexable.AddProperty(Property.NewUnsearched("Tag", arg_tag));
}
indexable.Source = arg_source;
pending_request.Add(indexable);
bool reschedule = false;
do
{
if (Shutdown.ShutdownRequested)
{
break;
}
if (!reschedule && pending_request.Count < BATCH_SIZE)
{
break;
}
if (reschedule)
{
Logger.Log.Debug("Continuing indexing indexer generated indexables");
}
else
{
Logger.Log.Debug("Flushing driver, {0} items in queue", pending_request.Count);
}
reschedule = FlushIndexer(driver);
// Super Lame Hack: gtk-sharp up to 2.10 requires a main loop
// to dispose of any managed wrappers around GObjects. Since
// we don't have one, we'll process all the pending items in
// a loop here. This is particularly an issue with maildirs,
// because we need the loop to clean up after GMime. Without
// it, GMime's streams are never completely unref'd, the
// file descriptors aren't closed, and we run out and crash.
while (GLib.MainContext.Pending())
{
GLib.MainContext.Iteration();
}
} while (reschedule);
}
private Indexable TodoToIndexable(CalComponent cc)
{
Indexable indexable = new Indexable(GetComponentUri(cc));
indexable.Timestamp = cc.Dtstart;
indexable.HitType = "Task";
indexable.AddProperty(Property.NewUnsearched("fixme:source_uid", this.source.Uid));
indexable.AddProperty(Property.NewUnsearched("fixme:uid", cc.Uid));
indexable.AddProperty(Property.NewDate("fixme:starttime", cc.Dtstart.ToUniversalTime()));
if (cc.Dtend != DateTime.MinValue)
{
indexable.AddProperty(Property.NewDate("fixme:endtime", cc.Dtend.ToUniversalTime()));
}
foreach (string description in cc.Descriptions)
{
indexable.AddProperty(Property.New("fixme:description", description));
}
indexable.AddProperty(Property.New("fixme:summary", cc.Summary));
foreach (string category in cc.Categories)
{
indexable.AddProperty(Property.NewUnsearched("fixme:category", category));
}
return(indexable);
}
static private Indexable TypeNodeToIndexable(XmlNode node, Uri base_uri)
{
string fragment = "T:" + node.Attributes ["FullName"].Value;
Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, fragment, false));
indexable.MimeType = "text/html";
indexable.HitType = "MonodocEntry";
indexable.AddProperty(Property.New("dc:title", "T:" + node.Attributes["FullName"].Value));
indexable.AddProperty(Property.NewUnsearched("fixme:name", "T:" + node.Attributes["FullName"].Value));
indexable.AddProperty(Property.NewUnsearched("fixme:type", "type"));
StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml);
indexable.SetTextReader(reader);
return(indexable);
}
public static void SendUpdate(IBrowsableItem item)
{
Indexable indexable = new Indexable(item.DefaultVersionUri);
indexable.Type = IndexableType.PropertyChange;
Beagle.Property prop;
// Clear the existing tags
prop = Beagle.Property.NewKeyword("fspot:Tag", "");
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
prop = Beagle.Property.NewKeyword("image:Tag", "");
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
foreach (Tag t in item.Tags)
{
prop = Beagle.Property.NewKeyword("fspot:Tag", t.Name);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
prop = Beagle.Property.NewKeyword("image:Tag", t.Name);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
}
prop = Beagle.Property.New("fspot:Description", item.Description);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
// Create a message to send to the daemon with this information.
// The source tells it what index the existing "/home/joe/test.txt" document lives.
IndexingServiceRequest req = new IndexingServiceRequest();
req.Keepalive = false;
req.Source = "Files";
req.Add(indexable);
req.SendAsync();
}
private Indexable GetNewXmpIndexable(ref Indexable indexable,
string path,
Guid id,
DirectoryModel parent)
{
// In non-crawl mode, check if a corresponding xmp file is present and not already scheduled and index it.
// If file.xmp and file are rapidly written/updated (in that order), this does the right thing.
// If file and file.xmp are rapidly written/updated (in that order), either
// - file.xmp is present during FileToIndexable(file): in which case xmp properties are
// added to file; and when file.xmp is indexed, it will replace the xmp properties
// - file.xmp is not present during FileToIndexable(file): when the xmp file is later indexed
// it will add the xmp properties
// since the uid file will still be in the uid-cache, correct uid will be used for xmp prop-change indexable
string possible_xmp_file_path = string.Concat(path, ".xmp");
if (!File.Exists(possible_xmp_file_path))
{
return(null);
}
Guid xmp_id = queryable.RegisterFile(parent, (Path.GetFileName(possible_xmp_file_path)));
if (xmp_id == Guid.Empty)
{
return(null);
}
XmpFile xmp_file = null;
try {
xmp_file = new XmpFile(possible_xmp_file_path);
} catch {
uid_manager.ForgetNewId(possible_xmp_file_path);
return(null);
}
// FIXME: Should also delete previous xmp properties!
foreach (Property p in xmp_file.Properties)
{
p.IsMutable = true;
indexable.AddProperty(p);
}
xmp_file.Close();
// Also need to save some local states for PostAddHook,
// namely, path to the xmp file, path to basefile and generated uid
indexable.LocalState ["XmpFilePath"] = possible_xmp_file_path;
indexable.LocalState ["BaseFilePath"] = path;
indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString(xmp_id);
if (Debug)
{
Log.Debug("Adding properties from {0}({2}) to {1}({3})", possible_xmp_file_path, path, GuidFu.ToShortString(xmp_id), GuidFu.ToShortString(id));
}
return(null);
}
// FIXME: This need some more info
private Indexable NntpMessageToIndexable(TB.NntpMessage message)
{
Indexable indexable;
indexable = new Indexable(message.Uri);
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.Timestamp = DateTime.Parse(message.GetString("date")).ToUniversalTime();
indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird"));
indexable.AddProperty(Property.NewUnsearched("fixme:fullyIndexed", message.GetBool("FullIndex")));
indexable.AddProperty(Property.NewDate("fixme:indexDateTime", DateTime.UtcNow));
string subject = GMime.Utils.HeaderDecodePhrase(message.GetString("subject"));
indexable.AddProperty(Property.New("dc:title", subject));
return(indexable);
}
private Indexable EventToIndexable(CalComponent cc)
{
Indexable indexable = new Indexable(GetComponentUri(cc));
indexable.Timestamp = cc.Dtstart;
indexable.HitType = "Calendar";
indexable.AddProperty(Property.NewUnsearched("fixme:source_uid", this.source.Uid));
indexable.AddProperty(Property.NewUnsearched("fixme:uid", cc.Uid));
indexable.AddProperty(Property.NewDate("fixme:starttime", cc.Dtstart.ToUniversalTime()));
if (cc.Dtend != DateTime.MinValue)
{
indexable.AddProperty(Property.NewDate("fixme:endtime", cc.Dtend.ToUniversalTime()));
}
foreach (CalComponentAttendee attendee in cc.Attendees)
{
indexable.AddProperty(Property.New("fixme:attendee", attendee.value));
}
foreach (string comment in cc.Comments)
{
indexable.AddProperty(Property.New("fixme:comment", comment));
}
foreach (string description in cc.Descriptions)
{
indexable.AddProperty(Property.New("fixme:description", description));
}
indexable.AddProperty(Property.New("fixme:summary", cc.Summary));
foreach (string category in cc.Categories)
{
indexable.AddProperty(Property.NewUnsearched("fixme:category", category));
}
indexable.AddProperty(Property.New("fixme:location", cc.Location));
return(indexable);
}
/////////////////////////////////////////////////
private static Indexable NoteToIndexable(FileInfo file)
{
Indexable indexable = new Indexable(UriFu.PathToFileUri(file.FullName));
indexable.Timestamp = file.LastWriteTimeUtc;
indexable.HitType = "Note";
indexable.MimeType = "x-beagle/x-labyrinth-note";
indexable.AddProperty(Property.NewUnsearched("fixme:application", "labyrinth"));
return(indexable);
}
/**
* Create an indexable from a maildir message
*/
public Indexable MaildirMessageToIndexable(string filename, bool crawl)
{
//Logger.Log.Debug ("+ indexing maildir mail:" + filename);
String folder = GetFolderMaildir(filename);
Uri file_uri = UriFu.PathToFileUri(filename);
Indexable indexable = new Indexable(file_uri);
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.FlushBufferCache = crawl;
indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail"));
indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name));
indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder));
indexable.ContentUri = file_uri;
return(indexable);
}
private Indexable ToAddMailMessageIndexable(XmlDocument document)
{
GMime.Message message = null;
// Check if the entire message is available
if (ToBool(GetText(document, "HasOffline")))
{
// We must make sure we don't get an exception here since we can fallback to
// other information
try {
int offset = Convert.ToInt32(GetText(document, "MessageOffset")),
size = Convert.ToInt32(GetText(document, "OfflineSize"));
message = GetGMimeMessage(GetText(document, "FolderFile"), offset, size);
} catch (Exception e) {
Logger.Log.Debug(e, "Failed to parse GMime message");
}
}
if (message == null)
{
message = GetStubMessage(document);
}
Indexable indexable = new Indexable(GenerateUniqueUri(document));
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc(Convert.ToInt64(GetText(document, "Date")));
indexable.CacheContent = true;
indexable.FlushBufferCache = true;
indexable.SetBinaryStream(message.Stream);
indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird"));
indexable.AddProperty(Property.NewKeyword("fixme:folder", GetText(document, "Folder")));
indexable.AddProperty(Property.NewUnsearched("ParentUri", GetText(document, "FolderFile")));
indexable.AddProperty(Property.NewUnsearched("fixme:uri", GetText(document, "Uri")));
message.Dispose();
return(indexable);
}
/////////////////////////////////////////////////
public Indexable GetIndexable(NautilusTools.NautilusMetadata nm)
{
Indexable indexable = new Indexable(nm.Uri);
indexable.Type = IndexableType.PropertyChange;
Property prop;
// Reset the notes property.
if (nm.Notes == null)
{
nm.Notes = String.Empty;
}
prop = Property.New("nautilus:notes", nm.Notes);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
foreach (string emblem in nm.Emblems)
{
prop = Property.NewKeyword("nautilus:emblem", emblem);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
}
// We add an empty keyword so that the property is reset
if (nm.Emblems.Count == 0)
{
prop = Property.NewKeyword("nautilus:emblem", String.Empty);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty(prop);
}
return(indexable);
}
public Indexable GetNextIndexable()
{
DateTime line_dt = DateTime.MinValue;
while (log_line_as_sb != null)
{
//Log.Debug ("Checking line from {0}:[{1}]", log_file, log_line_as_sb);
bool in_session = AppendLogText(log_line_as_sb, out line_dt);
if (!in_session)
{
break;
}
prev_line_offset = reader.Position;
log_line_as_sb = reader.ReadLineAsStringBuilder();
}
// Check if there is new data to index
if (data_sb.Length == 0)
{
session_begin_time = line_dt;
return(null);
}
Uri uri = new Uri(String.Format("konversation://{0}@dumb/{1}", session_begin_offset, log_file));
Log.Debug("Creating indexable {0}", uri);
Indexable indexable = new Indexable(uri);
indexable.ParentUri = UriFu.PathToFileUri(log_file);
indexable.Timestamp = session_begin_time;
indexable.HitType = "IMLog";
indexable.CacheContent = false;
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_begin_offset", session_begin_offset));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_end_offset", prev_line_offset));
indexable.AddProperty(Beagle.Property.NewDate("fixme:starttime", session_begin_time));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:client", "Konversation"));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:protocol", "IRC"));
AddChannelInformation(indexable);
foreach (string speaker in speakers.Keys)
{
indexable.AddProperty(Beagle.Property.NewUnstored("fixme:speaker", speaker));
}
StringReader data_reader = new StringReader(data_sb.ToString());
indexable.SetTextReader(data_reader);
// update session begin time to the date of the current line which is not in this session
session_begin_time = line_dt;
return(indexable);
}
protected override Indexable MessageToIndexable(Message message)
{
Uri uri = new Uri(String.Format("feed:///{0};id={1}", message.Path, message.Id));
Indexable indexable = new Indexable(uri);
indexable.MimeType = "text/html";
indexable.Type = "FeedItem";
indexable.AddProperty(Property.New("fixme:client", "mozilla"));
indexable.AddProperty(Property.New("dc:title", message.Subject));
indexable.AddProperty(Property.New("fixme:author", message.From));
//indexable.AddProperty(Property.NewDate ("fixme:published", item.PubDate));
indexable.AddProperty(Property.NewUnsearched("fixme:itemuri", message.Headers ["Content-Base"]));
indexable.AddProperty(Property.New("fixme:offset", message.Offset));
StringReader reader = new StringReader(message.Body);
indexable.SetTextReader(reader);
return(indexable);
}
Indexable TypeNodeToIndexable(XmlNode node, FileInfo file)
{
Indexable indexable = new Indexable(
new Uri("monodoc:///" + file + ";item=T:" + node.Attributes["FullName"].Value));
indexable.MimeType = "application/monodoc";
indexable.Type = "Monodoc";
indexable.AddProperty(Property.NewUnsearched("fixme:type", "type"));
indexable.AddProperty(Property.NewUnsearched("fixme:name", "T:" + node.Attributes["FullName"].Value));
string splitname = String.Join(" ",
StringFu.FuzzySplit(node.Attributes["FullName"].Value.ToString()));
indexable.AddProperty(Property.NewKeyword("fixme:splitname", splitname));
// Should we add other stuff here? Implemented interfaces etc?
StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml);
indexable.SetTextReader(reader);
return(indexable);
}
private Indexable current_itemToIndexable()
{
Indexable indexable;
try {
indexable = new Indexable(new Uri(String.Format("{0};item={1}", feed_source, current_item.Source)));
} catch (System.UriFormatException) {
indexable = new Indexable(new Uri(String.Format("liferea://dummy?{0};item={1}", feed_source, current_item.Source)));
}
indexable.ParentUri = UriFu.PathToFileUri(feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
DateTime date = DateTimeUtil.UnixToDateTimeUtc(0);
date = date.AddSeconds(current_item.Timestamp);
indexable.Timestamp = date;
// cleaning up the property names as far as possible
// this way querying for specific field is possible
// following DC element names wherever applicable
indexable.AddProperty(Property.New("dc:title", current_item.Title));
Attribute[] attribs = current_item.Attribs.AttribArray;
if (attribs != null)
{
foreach (Attribute attrib in attribs)
{
if (attrib.Name != "author")
{
continue;
}
indexable.AddProperty(Property.New("dc:creator", attrib.Value));
}
}
indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Source));
indexable.AddProperty(Property.NewKeyword("dc:source", feed_source));
indexable.AddProperty(Property.New("dc:publisher", publisher));
if (File.Exists(icon_file))
{
indexable.AddProperty(Property.NewUnsearched("fixme:cachedimg", icon_file));
}
StringReader reader = new StringReader(current_item.Description);
indexable.SetTextReader(reader);
return(indexable);
}
/////////////////////////////////////////////////
private Indexable PidginLogToIndexable(string filename)
{
FileInfo info = new FileInfo(filename);
Uri uri = UriFu.PathToFileUri(filename);
Indexable indexable = new Indexable(uri);
indexable.ContentUri = uri;
indexable.Timestamp = info.LastWriteTimeUtc;
indexable.MimeType = "beagle/x-pidgin-log";
indexable.HitType = "IMLog";
indexable.CacheContent = false;
ImBuddy buddy = queryable.ImBuddyListReader.Search(info.Directory.Name);
if (buddy != null && !String.IsNullOrEmpty(buddy.Alias))
{
indexable.AddProperty(Property.NewKeyword("fixme:alias", buddy.Alias));
}
return(indexable);
}
protected override Indexable MessageToIndexable (Message message)
{
Uri uri = new Uri (String.Format ("email:///{0};id={1}", message.Path, message.Id));
Indexable indexable = new Indexable (uri);
indexable.Type = "MailMessage";
indexable.AddProperty (Property.New ("fixme:client", "mozilla"));
indexable.AddProperty (Property.New ("dc:title", message.Subject));
indexable.AddProperty (Property.New ("fixme:subject", message.Subject));
indexable.AddProperty (Property.New ("fixme:to", message.To));
indexable.AddProperty (Property.New ("fixme:from", message.From));
indexable.AddProperty (Property.New ("fixme:offset", message.Offset));
StringReader reader = new StringReader (message.Body);
indexable.SetTextReader (reader);
return indexable;
}
protected override Indexable MessageToIndexable(Message message)
{
Uri uri = new Uri(String.Format("email:///{0};id={1}", message.Path, message.Id));
Indexable indexable = new Indexable(uri);
indexable.Type = "MailMessage";
indexable.AddProperty(Property.New("fixme:client", "mozilla"));
indexable.AddProperty(Property.New("dc:title", message.Subject));
indexable.AddProperty(Property.New("fixme:subject", message.Subject));
indexable.AddProperty(Property.New("fixme:to", message.To));
indexable.AddProperty(Property.New("fixme:from", message.From));
indexable.AddProperty(Property.New("fixme:offset", message.Offset));
StringReader reader = new StringReader(message.Body);
indexable.SetTextReader(reader);
return(indexable);
}
override protected void DoPullProperties ()
{
Stopwatch watch = new Stopwatch ();
watch.Start ();
while (reader.Read ()) {
switch (reader.NodeType) {
case XmlNodeType.Element:
if (reader.Name.StartsWith ("sect") || reader.Name.StartsWith ("chapter")) {
string id = reader.GetAttribute ("id");
if (id != null && id != String.Empty) {
DocbookEntry entry = new DocbookEntry ();
entry.Id = id;
entry.Depth = reader.Depth;
string language = reader.GetAttribute ("lang");
if (language != null && language != String.Empty)
entry.Language = language;
entries_stack.Push (entry);
}
} else if (reader.Name == "article" || reader.Name == "book") {
string language = reader.GetAttribute ("lang");
if (language != null && language != String.Empty)
base_language = language;
} else if (reader.Name == "title") {
reader.Read (); // Go to the text node
if (entries_stack.Count == 0 && base_title == null) {
// This is probably the book title
base_title = reader.Value;
} else if (entries_stack.Count > 0) {
DocbookEntry entry = (DocbookEntry) entries_stack.Peek ();
if (entry.Title == null)
entry.Title = reader.Value;
}
} else if (reader.Name == "keyword") {
reader.Read (); // read the text node
AddProperty (Property.NewKeyword ("dc:subject", reader.Value));
}
break;
case XmlNodeType.Text:
// Append text to the child indexable
if (entries_stack.Count > 0)
((DocbookEntry) entries_stack.Peek ()).Content.Append (reader.Value);
// Append text to the main indexable
else
AppendWord (reader.Value);
break;
case XmlNodeType.EndElement:
if (entries_stack.Count > 0 &&
((DocbookEntry) entries_stack.Peek ()).Depth == reader.Depth) {
DocbookEntry entry, parent_entry = null;
entry = (DocbookEntry) entries_stack.Pop ();
if (entries_stack.Count > 0)
parent_entry = (DocbookEntry) entries_stack.Peek ();
Indexable indexable;
indexable = new Indexable (UriFu.AddFragment (Indexable.Uri, entry.Id, false));
indexable.HitType = "DocbookEntry";
indexable.MimeType = "text/x-docbook-entry";
indexable.AddProperty (Property.NewKeyword ("beagle:FileType", "documentation"));
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty (Property.NewUnsearched ("fixme:id", entry.Id));
indexable.AddProperty (Property.New ("dc:title", entry.Title));
// Add the docbook book title
indexable.AddProperty (Property.NewUnsearched ("fixme:base_title", base_title));
// Add the child language (or docbook language if none is specified)
if (entry.Language != null)
indexable.AddProperty (Property.NewUnsearched ("fixme:language", entry.Language));
else if (base_language != null)
indexable.AddProperty (Property.NewUnsearched ("fixme:language", base_language));
// Add any parent (as in docbook parent entry, not beagle) data if we have it
if (parent_entry != null) {
indexable.AddProperty (Property.NewUnsearched ("fixme:parent_id", parent_entry.Id));
indexable.AddProperty (Property.NewUnsearched ("fixme:parent_title", parent_entry.Title));
}
StringReader content_reader = new StringReader (entry.Content.ToString ());
indexable.SetTextReader (content_reader);
indexable.SetChildOf (this.Indexable);
AddIndexable (indexable);
}
break;
}
}
// Add the common properties to the top-level
// file item such as Title, Language etc.
AddProperty (Property.New ("dc:title", base_title));
AddProperty (Property.NewUnsearched ("fixme:language", base_language));
watch.Stop ();
// If we've successfully crawled the file but haven't
// found any indexables, we shouldn't consider it
// successfull at all (unless we have a title, which
// means that it's actually a docbook file, just without
// sections.
if (! HasGeneratedIndexable && base_title == null) {
Log.Error ("Probably not a docbook. Ignoring {0}!", base_path);
Error ();
return;
}
Logger.Log.Debug ("Parsed docbook file in {0}", watch);
Finished ();
}
static void Main(String[] args)
{
string uriStr = null;
string title = null;
string sourcefile = null;
bool deletesourcefile = false;
if (args.Length == 0 || Array.IndexOf(args, "--help") > -1)
{
PrintUsage();
Environment.Exit(1);
}
for (int i = 0; i < args.Length; i++)
{
switch (args [i])
{
case "--url":
case "--title":
case "--sourcefile":
if (i + 1 >= args.Length ||
args [i + 1].StartsWith("--"))
{
PrintUsage();
Environment.Exit(1);
}
break;
}
switch (args [i])
{
case "--url":
uriStr = args [++i];
break;
case "--title":
title = args [++i];
break;
case "--sourcefile":
sourcefile = args [++i];
break;
case "--deletesourcefile":
deletesourcefile = true;
break;
case "--help":
PrintUsage();
return;
case "--version":
VersionFu.PrintVersion();
return;
}
}
if (uriStr == null)
{
Logger.Log.Error("URI not specified!\n");
PrintUsage();
Environment.Exit(1);
}
Uri uri = new Uri(uriStr, true);
if (uri.Scheme == Uri.UriSchemeHttps)
{
// For security/privacy reasons, we don't index any
// SSL-encrypted pages.
Logger.Log.Error("Indexing secure https:// URIs is not secure!");
Environment.Exit(1);
}
// We don't index file: Uris. Silently exit.
if (uri.IsFile)
{
return;
}
// We *definitely* don't index mailto: Uris. Silently exit.
if (uri.Scheme == Uri.UriSchemeMailto)
{
return;
}
Indexable indexable;
indexable = new Indexable(uri);
indexable.HitType = "WebHistory";
indexable.MimeType = "text/html";
indexable.Timestamp = DateTime.Now;
if (title != null)
{
indexable.AddProperty(Property.New("dc:title", title));
}
if (sourcefile != null)
{
if (!File.Exists(sourcefile))
{
Logger.Log.Error("sourcefile '{0}' does not exist!", sourcefile);
Environment.Exit(1);
}
indexable.ContentUri = UriFu.PathToFileUri(sourcefile);
indexable.DeleteContent = deletesourcefile;
}
else
{
Stream stdin = Console.OpenStandardInput();
if (stdin == null)
{
Logger.Log.Error("No sourcefile specified, and no standard input!\n");
PrintUsage();
Environment.Exit(1);
}
indexable.SetTextReader(new StreamReader(stdin));
}
IndexingServiceRequest req = new IndexingServiceRequest();
req.Add(indexable);
try {
Logger.Log.Info("Indexing");
Logger.Log.Debug("SendAsync");
req.SendAsync();
Logger.Log.Debug("Close");
req.Close();
Logger.Log.Debug("Done");
} catch (Exception e) {
Logger.Log.Error("Indexing failed: {0}", e);
// Still clean up after ourselves, even if we couldn't
// index the content.
if (deletesourcefile)
{
File.Delete(sourcefile);
}
Environment.Exit(1);
}
}
private void AddChannelInformation (Indexable indexable)
{
// FIXME: Parse identity information from konversation .config file
//AddProperty (Beagle.Property.NewUnsearched ("fixme:identity", log.Identity));
// Get server name, channel name from the filename and add it here
indexable.AddProperty (Beagle.Property.NewKeyword ("fixme:server", server_name));
indexable.AddProperty (Beagle.Property.NewKeyword ("fixme:speakingto", speaking_to));
}
/////////////////////////////////////////////////
private Indexable PidginLogToIndexable (string filename)
{
FileInfo info = new FileInfo (filename);
Uri uri = UriFu.PathToFileUri (filename);
Indexable indexable = new Indexable (uri);
indexable.ContentUri = uri;
indexable.Timestamp = info.LastWriteTimeUtc;
indexable.MimeType = "beagle/x-pidgin-log";
indexable.HitType = "IMLog";
indexable.CacheContent = false;
ImBuddy buddy = queryable.ImBuddyListReader.Search (info.Directory.Name);
if (buddy != null && !String.IsNullOrEmpty (buddy.Alias))
indexable.AddProperty (Property.NewKeyword ("fixme:alias", buddy.Alias));
return indexable;
}
private Indexable GetNewXmpIndexable (ref Indexable indexable,
string path,
Guid id,
DirectoryModel parent)
{
// In non-crawl mode, check if a corresponding xmp file is present and not already scheduled and index it.
// If file.xmp and file are rapidly written/updated (in that order), this does the right thing.
// If file and file.xmp are rapidly written/updated (in that order), either
// - file.xmp is present during FileToIndexable(file): in which case xmp properties are
// added to file; and when file.xmp is indexed, it will replace the xmp properties
// - file.xmp is not present during FileToIndexable(file): when the xmp file is later indexed
// it will add the xmp properties
// since the uid file will still be in the uid-cache, correct uid will be used for xmp prop-change indexable
string possible_xmp_file_path = string.Concat (path, ".xmp");
if (! File.Exists (possible_xmp_file_path))
return null;
Guid xmp_id = queryable.RegisterFile (parent, (Path.GetFileName (possible_xmp_file_path)));
if (xmp_id == Guid.Empty)
return null;
XmpFile xmp_file = null;
try {
xmp_file = new XmpFile (possible_xmp_file_path);
} catch {
uid_manager.ForgetNewId (possible_xmp_file_path);
return null;
}
// FIXME: Should also delete previous xmp properties!
foreach (Property p in xmp_file.Properties) {
p.IsMutable = true;
indexable.AddProperty (p);
}
xmp_file.Close ();
// Also need to save some local states for PostAddHook,
// namely, path to the xmp file, path to basefile and generated uid
indexable.LocalState ["XmpFilePath"] = possible_xmp_file_path;
indexable.LocalState ["BaseFilePath"] = path;
indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString (xmp_id);
if (Debug)
Log.Debug ("Adding properties from {0}({2}) to {1}({3})", possible_xmp_file_path, path, GuidFu.ToShortString (xmp_id), GuidFu.ToShortString (id));
return null;
}
private Indexable ToAddRssIndexable (XmlDocument document)
{
string encoding_str = null;
StringReader reader = null;
if (ToBool (GetText (document, "HasOffline"))) {
try {
// RSS does not use OfflineSize but MessageSize instead (for some reason...)
int offset = Convert.ToInt32 (GetText (document, "MessageOffset")),
size = Convert.ToInt32 (GetText (document, "MessageSize"));
reader = GetRssBody (GetText (document, "FolderFile"), offset, size, out encoding_str);
} catch (Exception e) {
Logger.Log.Debug (e, "Failed to parse RSS body");
}
}
Indexable indexable = new Indexable (GenerateUniqueUri (document));
indexable.HitType = "FeedItem";
indexable.MimeType = "text/html";
indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc (Convert.ToInt64 (GetText (document, "Date")));
indexable.CacheContent = true;
indexable.FlushBufferCache = true;
indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird"));
indexable.AddProperty (Property.NewKeyword ("fixme:folder", GetText (document, "Folder")));
indexable.AddProperty (Property.NewUnsearched ("ParentUri", GetText (document, "FolderFile")));
indexable.AddProperty (Property.NewUnsearched ("fixme:uri", GetText (document, "Uri")));
indexable.AddProperty (Property.NewKeyword ("dc:identifier", ExtractUrl (GetText (document, "MessageId"))));
indexable.AddProperty (Property.NewKeyword ("dc:source", GetText (document, "FeedURL")));
indexable.AddProperty (Property.New ("dc:publisher", Mime.HeaderDecodePhrase (GetText (document, "Author"))));
// The title will be added by the filter. In case we add it twice we will just get
// an empty tile in the search tool (a bug maybe?).
if (reader != null) {
// If we got an encoding, make sure we use that
if (!String.IsNullOrEmpty (encoding_str)) {
indexable.AddProperty (Property.New (
String.Format ("{0}encoding", StringFu.UnindexedNamespace), encoding_str));
}
indexable.SetTextReader (reader);
} else
indexable.AddProperty (Property.New ("dc:title", Mime.HeaderDecodePhrase (GetText (document, "Subject"))));
return indexable;
}
// FIXME: This need some more info
private Indexable NntpMessageToIndexable (TB.NntpMessage message)
{
Indexable indexable;
indexable = new Indexable (message.Uri);
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.Timestamp = DateTime.Parse (message.GetString ("date")).ToUniversalTime ();
indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird"));
indexable.AddProperty (Property.NewUnsearched ("fixme:fullyIndexed", message.GetBool ("FullIndex")));
indexable.AddProperty (Property.NewDate ("fixme:indexDateTime", DateTime.UtcNow));
string subject = GMime.Utils.HeaderDecodePhrase (message.GetString ("subject"));
indexable.AddProperty (Property.New ("dc:title", subject));
return indexable;
}
private Indexable MemoToIndexable (CalComponent cc)
{
Indexable indexable = new Indexable (GetComponentUri (cc));
indexable.Timestamp = cc.Dtstart;
indexable.HitType = "Note";
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty (Property.NewUnsearched ("fixme:application","evolution"));
indexable.AddProperty (Property.New ("dc:title", cc.Summary));
// We remember the note's text so that we can stuff it in
// the TextCache later.
// This is here form compability with Tomboy notes.
foreach (string description in cc.Descriptions) {
queryable.IndexableTextCache [indexable.Uri] = description;
StringReader reader = new StringReader (description);
indexable.SetTextReader (reader);
}
return indexable;
}
private Indexable EventToIndexable (CalComponent cc)
{
Indexable indexable = new Indexable (GetComponentUri (cc));
indexable.Timestamp = cc.Dtstart;
indexable.HitType = "Calendar";
indexable.AddProperty (Property.NewUnsearched ("fixme:source_uid", this.source.Uid));
indexable.AddProperty (Property.NewUnsearched ("fixme:uid", cc.Uid));
indexable.AddProperty (Property.NewDate ("fixme:starttime", cc.Dtstart.ToUniversalTime ()));
if (cc.Dtend != DateTime.MinValue)
indexable.AddProperty (Property.NewDate ("fixme:endtime", cc.Dtend.ToUniversalTime ()));
foreach (CalComponentAttendee attendee in cc.Attendees)
indexable.AddProperty (Property.New ("fixme:attendee", attendee.value));
foreach (string comment in cc.Comments)
indexable.AddProperty (Property.New ("fixme:comment", comment));
foreach (string description in cc.Descriptions)
indexable.AddProperty (Property.New ("fixme:description", description));
indexable.AddProperty (Property.New ("fixme:summary", cc.Summary));
foreach (string category in cc.Categories)
indexable.AddProperty (Property.NewUnsearched ("fixme:category", category));
indexable.AddProperty (Property.New ("fixme:location", cc.Location));
return indexable;
}
private Indexable ContactToIndexable (Evolution.Contact contact)
{
DateTime rev = RevStringToDateTime (contact.Rev);
Indexable indexable = new Indexable (GetContactUri (contact));
indexable.Timestamp = rev;
indexable.HitType = "Contact";
indexable.AddProperty (Property.NewKeyword ("fixme:client", "evolution"));
indexable.AddProperty (Property.NewUnsearched ("fixme:source_uid", this.source.Uid));
indexable.AddProperty (Property.NewUnsearched ("fixme:uid", contact.Id));
indexable.AddProperty (Property.New ("fixme:FileAs", contact.FileAs));
indexable.AddProperty (Property.New ("fixme:FullName", contact.FullName));
indexable.AddProperty (Property.New ("fixme:GivenName", contact.GivenName));
indexable.AddProperty (Property.New ("fixme:FamilyName", contact.FamilyName));
indexable.AddProperty (Property.New ("fixme:Nickname", contact.Nickname));
indexable.AddProperty (Property.New ("fixme:AddressLabelHome", contact.AddressLabelHome));
indexable.AddProperty (Property.New ("fixme:AddressLabelWork", contact.AddressLabelWork));
indexable.AddProperty (Property.New ("fixme:AddressLabelOther", contact.AddressLabelOther));
indexable.AddProperty (Property.New ("fixme:AssistantPhone", contact.AssistantPhone));
indexable.AddProperty (Property.New ("fixme:BusinessPhone", contact.BusinessPhone));
indexable.AddProperty (Property.New ("fixme:BusinessPhone2", contact.BusinessPhone2));
indexable.AddProperty (Property.New ("fixme:BusinessFax", contact.BusinessFax));
indexable.AddProperty (Property.New ("fixme:CallbackPhone", contact.CallbackPhone));
indexable.AddProperty (Property.New ("fixme:CarPhone", contact.CarPhone));
indexable.AddProperty (Property.New ("fixme:CompanyPhone", contact.CompanyPhone));
indexable.AddProperty (Property.New ("fixme:HomePhone", contact.HomePhone));
indexable.AddProperty (Property.New ("fixme:HomePhone2", contact.HomePhone2));
indexable.AddProperty (Property.New ("fixme:HomeFax", contact.HomeFax));
indexable.AddProperty (Property.New ("fixme:IsdnPhone", contact.IsdnPhone));
indexable.AddProperty (Property.New ("fixme:MobilePhone", contact.MobilePhone));
indexable.AddProperty (Property.New ("fixme:OtherPhone", contact.OtherPhone));
indexable.AddProperty (Property.New ("fixme:OtherFax", contact.OtherFax));
indexable.AddProperty (Property.New ("fixme:Pager", contact.Pager));
indexable.AddProperty (Property.New ("fixme:PrimaryPhone", contact.PrimaryPhone));
indexable.AddProperty (Property.New ("fixme:Radio", contact.Radio));
indexable.AddProperty (Property.New ("fixme:Telex", contact.Telex));
indexable.AddProperty (Property.NewUnsearched ("fixme:Tty", contact.Tty));
indexable.AddProperty (Property.NewKeyword ("fixme:Email1", contact.Email1));
indexable.AddProperty (Property.NewKeyword ("fixme:Email2", contact.Email2));
indexable.AddProperty (Property.NewKeyword ("fixme:Email3", contact.Email3));
indexable.AddProperty (Property.NewUnsearched ("fixme:Mailer", contact.Mailer));
indexable.AddProperty (Property.New ("fixme:Org", contact.Org));
indexable.AddProperty (Property.New ("fixme:OrgUnit", contact.OrgUnit));
indexable.AddProperty (Property.New ("fixme:Office", contact.Office));
indexable.AddProperty (Property.New ("fixme:Title", contact.Title));
indexable.AddProperty (Property.New ("fixme:Role", contact.Role));
indexable.AddProperty (Property.New ("fixme:Manager", contact.Manager));
indexable.AddProperty (Property.New ("fixme:Assistant", contact.Assistant));
indexable.AddProperty (Property.NewKeyword ("fixme:HomepageUrl", contact.HomepageUrl));
indexable.AddProperty (Property.NewKeyword ("fixme:BlogUrl", contact.BlogUrl));
indexable.AddProperty (Property.NewUnsearched ("fixme:Categories", contact.Categories));
indexable.AddProperty (Property.NewUnsearched ("fixme:Caluri", contact.Caluri));
indexable.AddProperty (Property.NewUnsearched ("fixme:Icscalendar", contact.Icscalendar));
indexable.AddProperty (Property.New ("fixme:Spouse", contact.Spouse));
indexable.AddProperty (Property.New ("fixme:Note", contact.Note));
Evolution.ContactPhoto photo = contact.Photo;
if (photo.Data != null && photo.Data.Length > 0) {
string photo_filename = GetPhotoFilename (contact.Id);
Stream s = new FileStream (photo_filename, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
BinaryWriter w = new BinaryWriter (s);
w.Write (photo.Data);
w.Close ();
s.Close ();
indexable.AddProperty (Property.NewUnsearched ("beagle:Photo", photo_filename));
}
foreach (string im in contact.ImAim)
indexable.AddProperty (Property.NewUnsearched ("fixme:ImAim", im));
foreach (string im in contact.ImIcq)
indexable.AddProperty (Property.NewUnsearched ("fixme:ImIcq", im));
foreach (string im in contact.ImJabber)
indexable.AddProperty (Property.NewUnsearched ("fixme:ImJabber", im));
foreach (string im in contact.ImMsn)
indexable.AddProperty (Property.NewUnsearched ("fixme:ImMsn", im));
foreach (string im in contact.ImYahoo)
indexable.AddProperty (Property.NewUnsearched ("fixme:ImYahoo", im));
foreach (string im in contact.ImGroupwise)
indexable.AddProperty (Property.NewUnsearched ("fixme:ImGroupwise", im));
String name = "";
if (contact.GivenName != null && contact.GivenName != "")
name = contact.GivenName;
if (contact.FamilyName != null && contact.FamilyName != "")
name += " " + contact.FamilyName;
if (name.Length > 0)
indexable.AddProperty (Property.New ("fixme:Name", name));
if (contact.Email1 != null)
indexable.AddProperty (Property.NewKeyword ("fixme:Email",
contact.Email1));
return indexable;
}
/* Returns false if content can't/needn't be indexed.
* If AlreadyFiltered, then we don't return a filter but return true.
*/
static public bool FilterIndexable (Indexable indexable, TextCache text_cache, out Filter filter)
{
filter = null;
ICollection filters = null;
if (indexable.Filtering == IndexableFiltering.AlreadyFiltered)
return true;
if (! ShouldWeFilterThis (indexable))
return false;
string path = null;
// First, figure out which filter we should use to deal with
// the indexable.
// If a specific mime type is specified, try to index as that type.
if (indexable.MimeType != null)
filters = CreateFiltersFromMimeType (indexable.MimeType);
if (indexable.ContentUri.IsFile) {
path = indexable.ContentUri.LocalPath;
// Otherwise, set the mime type for a directory,
// or sniff it from the file.
if (indexable.MimeType == null) {
if (Directory.Exists (path)) {
indexable.MimeType = "inode/directory";
indexable.NoContent = true;
} else if (File.Exists (path)) {
indexable.MimeType = XdgMime.GetMimeType (path);
} else {
Log.Warn ("Unable to filter {0}. {1} not found.", indexable.DisplayUri, path);
return false;
}
}
// Set the timestamp to the last write time, if it isn't
// set by the backend.
if (! indexable.ValidTimestamp && indexable.IsNonTransient)
indexable.Timestamp = FileSystem.GetLastWriteTimeUtc (path);
// Check the timestamp to make sure the file hasn't
// disappeared from underneath us.
if (! FileSystem.ExistsByDateTime (indexable.Timestamp)) {
Log.Warn ("Unable to filter {0}. {1} appears to have disappeared from underneath us", indexable.DisplayUri, path);
return false;
}
if (filters == null || filters.Count == 0) {
filters = CreateFiltersFromIndexable (indexable);
}
}
// We don't know how to filter this, so there is nothing else to do.
if (filters.Count == 0) {
if (! indexable.NoContent)
Logger.Log.Debug ("No filter for {0} ({1}) [{2}]", indexable.DisplayUri, path, indexable.MimeType);
return false;
}
foreach (Filter candidate_filter in filters) {
if (Debug)
Logger.Log.Debug ("Testing filter: {0}", candidate_filter);
// Hook up the snippet writer.
if (candidate_filter.SnippetMode && text_cache != null) {
if (candidate_filter.OriginalIsText && indexable.IsNonTransient) {
text_cache.MarkAsSelfCached (indexable.Uri);
} else if (indexable.CacheContent) {
TextWriter writer = text_cache.GetWriter (indexable.Uri);
candidate_filter.AttachSnippetWriter (writer);
}
}
// Set the indexable on the filter.
candidate_filter.Indexable = indexable;
// Open the filter, copy the file's properties to the indexable,
// and hook up the TextReaders.
bool successful_open = false;
TextReader text_reader;
Stream binary_stream;
if (path != null)
successful_open = candidate_filter.Open (path);
else if ((text_reader = indexable.GetTextReader ()) != null)
successful_open = candidate_filter.Open (text_reader);
else if ((binary_stream = indexable.GetBinaryStream ()) != null)
successful_open = candidate_filter.Open (binary_stream);
if (successful_open) {
// Set FileType
indexable.AddProperty (Property.NewKeyword ("beagrep:FileType", candidate_filter.FileType));
indexable.SetTextReader (candidate_filter.GetTextReader ());
indexable.SetHotTextReader (candidate_filter.GetHotTextReader ());
if (Debug)
Logger.Log.Debug ("Successfully filtered {0} with {1}", path, candidate_filter);
filter = candidate_filter;
return true;
} else {
Log.Warn ("Error in filtering {0} with {1}, falling back", path, candidate_filter);
candidate_filter.Cleanup ();
}
}
if (Debug)
Logger.Log.Debug ("None of the matching filters could process the file: {0}", path);
return false;
}
private Indexable current_itemToIndexable ()
{
Indexable indexable;
try {
indexable = new Indexable (new Uri (String.Format ("{0};item={1}", feed_source, current_item.Source)));
} catch (System.UriFormatException) {
indexable = new Indexable (new Uri (String.Format ("liferea://dummy?{0};item={1}", feed_source, current_item.Source)));
}
indexable.ParentUri = UriFu.PathToFileUri (feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
DateTime date = DateTimeUtil.UnixToDateTimeUtc (0);
date = date.AddSeconds (current_item.Timestamp);
indexable.Timestamp = date;
// cleaning up the property names as far as possible
// this way querying for specific field is possible
// following DC element names wherever applicable
indexable.AddProperty (Property.New ("dc:title", current_item.Title));
Attribute[] attribs = current_item.Attribs.AttribArray;
if (attribs != null) {
foreach (Attribute attrib in attribs) {
if (attrib.Name != "author")
continue;
indexable.AddProperty (Property.New ("dc:creator", attrib.Value));
}
}
indexable.AddProperty (Property.NewKeyword ("dc:identifier", current_item.Source));
indexable.AddProperty (Property.NewKeyword ("dc:source", feed_source));
indexable.AddProperty (Property.New ("dc:publisher", publisher));
if (File.Exists (icon_file))
indexable.AddProperty (Property.NewUnsearched ("fixme:cachedimg", icon_file));
StringReader reader = new StringReader (current_item.Description);
indexable.SetTextReader (reader);
return indexable;
}
/////////////////////////////////////////////////
private Indexable NoteToIndexable (FileInfo file, Note note)
{
Indexable indexable = new Indexable (note.Uri);
indexable.ContentUri = UriFu.PathToFileUri (file.FullName);
indexable.Timestamp = note.timestamp;
indexable.HitType = "Note";
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty (Property.New ("dc:title", note.subject));
indexable.AddProperty (Property.NewUnsearched ("fixme:application","tomboy"));
// FIXME: tagging is disabled in Tomboy-0.8.x and is planned for 0.10.0
foreach( string s in note.tags)
indexable.AddProperty (Property.New ("note:tag", s));
// We remember the note's text so that we can stuff it in
// the TextCache later.
note_text_cache [note.Uri] = note.text;
StringReader reader = new StringReader (note.text);
indexable.SetTextReader (reader);
return indexable;
}
private void IndexLauncher (FileInfo file, Scheduler.Priority priority)
{
if ((! file.Exists)
|| (this.FileAttributesStore.IsUpToDate (file.FullName)))
return;
/* Check to see if file is a launcher */
if (Beagle.Util.VFS.Mime.GetMimeType (file.FullName) != "application/x-desktop")
return;
StreamReader reader;
try {
reader = new StreamReader (file.Open (FileMode.Open, FileAccess.Read, FileShare.Read));
} catch (Exception e) {
log.Warn ("Could not open '{0}': {1}", file.FullName, e.Message);
return;
}
if (reader.ReadLine () != "[Desktop Entry]") {
reader.Close ();
return;
}
/* I'm convinced it is a launcher */
Indexable indexable = new Indexable (UriFu.PathToFileUri (file.FullName));
indexable.Timestamp = file.LastWriteTime;
indexable.Type = "Launcher";
indexable.MimeType = "application/x-desktop";
// desktop files must have a name
bool have_name = false;
String line;
while ((line = reader.ReadLine ()) != null) {
string [] sline = line.Split ('=');
if (sline.Length != 2)
continue;
// FIXME: We shouldnt really search fields that are in other locales than the current should we?
if (sline [0].Equals ("Icon") || sline [0].Equals ("Exec")) {
indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:" + sline[0], sline[1]));
} else if (sline [0].StartsWith ("Name")) {
if (sline [0] == "Name")
have_name = true;
indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:" + sline[0], sline[1]));
} else if (sline[0].StartsWith ("Comment")) {
indexable.AddProperty (Beagle.Property.New ("fixme:" + sline[0], sline[1]));
}
}
reader.Close ();
if (have_name) {
Scheduler.Task task = NewAddTask (indexable);
task.Priority = priority;
ThisScheduler.Add (task);
}
}
Indexable BuildAddIndexable(Post post)
{
Indexable indexable = new Indexable (IndexableType.Add, new Uri(post.Href));
indexable.Timestamp = post.Time;
indexable.Filtering = IndexableFiltering.Always;
indexable.HitType = "Bookmark";
indexable.AddProperty (Property.New ("dc:title", post.Description));
indexable.AddProperty (Property.New ("dc:source", post.Href));
indexable.AddProperty (Property.New ("dc:keywords", post.Tag));
indexable.AddProperty (Property.New ("fixme:keywords", post.Tag));
indexable.AddProperty (Property.New ("fixme:host", new Uri (post.Href).Host));
// May want to support multiple accounts eventually
indexable.AddProperty (Property.New ("delicious:account", requester.Account.Username));
AddContentToIndexable (indexable, post);
return indexable;
}
/////////////////////////////////////////////////
public Indexable GetIndexable (NautilusTools.NautilusMetadata nm)
{
Indexable indexable = new Indexable (nm.Uri);
indexable.Type = IndexableType.PropertyChange;
Property prop;
// Reset the notes property.
if (nm.Notes == null)
nm.Notes = String.Empty;
prop = Property.New ("nautilus:notes", nm.Notes);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty (prop);
foreach (string emblem in nm.Emblems) {
prop = Property.NewKeyword ("nautilus:emblem", emblem);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty (prop);
}
// We add an empty keyword so that the property is reset
if (nm.Emblems.Count == 0) {
prop = Property.NewKeyword ("nautilus:emblem", String.Empty);
prop.IsMutable = true;
prop.IsPersistent = true;
indexable.AddProperty (prop);
}
return indexable;
}
protected override Indexable MessageToIndexable (Message message)
{
Uri uri = new Uri (String.Format ("feed:///{0};id={1}", message.Path, message.Id));
Indexable indexable = new Indexable (uri);
indexable.MimeType = "text/html";
indexable.Type = "FeedItem";
indexable.AddProperty (Property.New ("fixme:client", "mozilla"));
indexable.AddProperty(Property.New ("dc:title", message.Subject));
indexable.AddProperty(Property.New ("fixme:author", message.From));
//indexable.AddProperty(Property.NewDate ("fixme:published", item.PubDate));
indexable.AddProperty(Property.NewUnsearched ("fixme:itemuri", message.Headers ["Content-Base"]));
indexable.AddProperty (Property.New ("fixme:offset", message.Offset));
StringReader reader = new StringReader (message.Body);
indexable.SetTextReader (reader);
return indexable;
}
public Indexable GetNextIndexable ()
{
string line;
string_builder.Length = 0;
DateTime dt = DateTime.MinValue;
string uid = null;
// Keep reading till "END:VJOURNAL"
while ((line = reader.ReadLine ()) != null) {
//UID:libkcal-1467827482.768
//LAST-MODIFIED:20061015T085606Z
if (line == "END:VJOURNAL")
break;
else if (line.StartsWith ("UID:"))
uid = line.Substring (4);
else if (line.StartsWith ("LAST-MODIFIED:")) {
string dt_string = line.Substring (14);
dt_string = dt_string.Replace ("Z", "+00:00");
dt = DateTime.ParseExact (
dt_string,
fmts,
DateTimeFormatInfo.InvariantInfo,
DateTimeStyles.AdjustToUniversal);
} else {
string_builder.Append (line);
string_builder.Append ('\n');
}
}
if (line == null) {
reader.Close ();
return null;
}
// Bad note
if (string_builder.Length == 0 ||
uid == null ||
dt == DateTime.MinValue)
return null;
// Mark note with uid as seen ('undeleted')
deleted_notes [uid] = false;
lock (last_modified_table) {
if (last_modified_table.Contains (uid)) {
DateTime old_dt = (DateTime) last_modified_table [uid];
// FIXME: Returning null for more than 179 times will cause trouble
if (dt == old_dt)
return null;
else {
//Log.Debug ("Updating last_mod_date [{0}] = {1}", uid, dt);
last_modified_table [uid] = dt;
}
} else {
//Log.Debug ("Adding last_mod_date [{0}] = {1}", uid, dt);
last_modified_table [uid] = dt;
}
}
if (initial_scan)
return null;
// Open knotes notes as
//dcop knotes KNotesIface text <UID>
// where the uri is given as knotes://uid
Uri uri = new Uri (String.Format ("knotes:///{0}", uid));
Indexable indexable = new Indexable (uri);
indexable.ParentUri = UriFu.PathToFileUri (knotes_file);
indexable.MimeType = ICalParser.KnotesMimeType;
indexable.HitType = "Note";
indexable.Timestamp = dt;
// Add uid as a keyword field for convenience
indexable.AddProperty (Property.NewUnsearched ("fixme:uid", uid));
// FIXME: Comment this Debug statement after the backend stabilizes
//Log.Debug ("Creating {0} from:[{1}]", uri, string_builder.ToString ());
StringReader string_reader = new StringReader (string_builder.ToString());
indexable.SetTextReader (string_reader);
return indexable;
}
private Indexable ToAddMailMessageIndexable (XmlDocument document)
{
GMime.Message message = null;
// Check if the entire message is available
if (ToBool (GetText (document, "HasOffline"))) {
// We must make sure we don't get an exception here since we can fallback to
// other information
try {
int offset = Convert.ToInt32 (GetText (document, "MessageOffset")),
size = Convert.ToInt32 (GetText (document, "OfflineSize"));
message = GetGMimeMessage (GetText (document, "FolderFile"), offset, size);
} catch (Exception e) {
Logger.Log.Debug (e, "Failed to parse GMime message");
}
}
if (message == null)
message = GetStubMessage (document);
Indexable indexable = new Indexable (GenerateUniqueUri (document));
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc (Convert.ToInt64 (GetText (document, "Date")));
indexable.CacheContent = true;
indexable.FlushBufferCache = true;
indexable.SetBinaryStream (message.Stream);
indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird"));
indexable.AddProperty (Property.NewKeyword ("fixme:folder", GetText (document, "Folder")));
indexable.AddProperty (Property.NewUnsearched ("ParentUri", GetText (document, "FolderFile")));
indexable.AddProperty (Property.NewUnsearched ("fixme:uri", GetText (document, "Uri")));
message.Dispose ();
return indexable;
}
private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags)
{
// Don't index messages flagged as junk
if (CheckFlags (flags, B_U_Camel.CamelFlags.Junk))
return null;
System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
Indexable indexable = new Indexable (uri);
indexable.Timestamp = message.Date.ToUniversalTime ();
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution"));
indexable.AddProperty (Property.NewUnsearched ("fixme:account", "Local"));
indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name));
GMime.InternetAddressList addrs;
if (this.folder_name == "Sent") {
addrs = message.GetRecipients (GMime.RecipientType.To);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (this.folder_name == "Sent") {
addrs = message.GetRecipients (GMime.RecipientType.Cc);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (this.folder_name != "Sent") {
addrs = GMime.InternetAddressList.Parse (message.Sender);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose ();
}
if (this.folder_name == "Sent")
indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
Property flag_prop = Property.NewUnsearched ("fixme:flags", flags);
flag_prop.IsMutable = true;
indexable.AddProperty (flag_prop);
if (CheckFlags (flags, B_U_Camel.CamelFlags.Answered))
indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Deleted))
indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Draft))
indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Flagged))
indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Seen))
indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.AnsweredAll))
indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
indexable.SetBinaryStream (message.Stream);
return indexable;
}
public override bool GenerateNextIndexable (out Indexable child)
{
child = null;
if (bib_process == null && ! InitBibparse ())
return false;
string line = null;
string type = null, name = null;
while ((line = reader.ReadLine ()) != null) {
if (line == String.Empty || line [0] != '@')
continue;
int i = line.IndexOf (' ');
if (i == -1 || line.Length == i + 1)
continue;
type = line.Substring (1, i - 1).ToLower ();
name = line.Substring (i + 1);
break;
}
if (line == null)
return false;
child = new Indexable (UriFu.AddFragment (Indexable.Uri, name, false));
child.CacheContent = false;
child.MimeType = "text/x-bibtex";
child.DisplayUri = child.Uri;
child.NoContent = true;
child.AddProperty (Property.NewKeyword ("bibtex:type", type));
string key, value;
// Now fill in properties from the key=value lines
while ((line = reader.ReadLine ()) != null) {
// Entries are separated by empty lines
if (line == String.Empty)
break;
int i = line.IndexOf ('=');
// ensure non-empty key
if (i < 1 || line.Length == i + i)
continue;
key = line.Substring (0, i).ToLower ();
value = line.Substring (i + 1);
foreach (Property prop in EntryLineToProperty (key, value))
child.AddProperty (prop);
}
child.SetChildOf (Indexable);
return true;
}
public override Indexable GetNextIndexable ()
{
Indexable indexable = null;
// No more new messages to index, so start on the removals.
if (this.delete_mode) {
string uid = (string) this.deleted_list [0];
Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
indexable = new Indexable (IndexableType.Remove, uri);
this.deleted_list.RemoveAt (0);
this.tracker.Remove (uid);
this.delete_count++;
return indexable;
}
B_U_Camel.MessageInfo mi = (B_U_Camel.MessageInfo) this.summary_enumerator.Current;
++this.count;
if (Debug) {
Logger.Log.Debug ("Constructed message {0} with uid {1}, flags {2}.",
this.count, mi.uid, mi.flags);
}
uint flags;
bool found = this.tracker.Get (mi.uid, out flags);
if (! found) {
// New, previously unseen message
string msg_file;
if (this.backend_type == ImapBackendType.Imap)
msg_file = Path.Combine (summary_info.DirectoryName, mi.uid + ".");
else {
// This is taken from e-d-s's camel-data-cache.c. No doubt
// NotZed would scream bloody murder if he saw this here.
int hash = (g_str_hash (mi.uid) >> 5) & CAMEL_DATA_CACHE_MASK;
string cache_path = String.Format ("cache/{0:x}/{1}", hash, mi.uid);
msg_file = Path.Combine (summary_info.DirectoryName, cache_path);
}
indexable = this.CamelMessageToIndexable (mi, msg_file);
if (Debug)
Logger.Log.Debug ("Unseen message, indexable {0} null", indexable == null ? "" : "not");
if (indexable != null)
++this.indexed_count;
} else if (found && flags != mi.flags) {
// Previously seen message, but flags have changed.
Uri uri = CamelMessageUri (mi);
indexable = new Indexable (uri);
indexable.Type = IndexableType.PropertyChange;
Property flag_prop = Property.NewUnsearched ("fixme:flags", mi.flags);
flag_prop.IsMutable = true;
indexable.AddProperty (flag_prop);
if (Debug)
Logger.Log.Debug ("Previously seen message, flags changed: {0} -> {1}", flags, mi.flags);
++this.indexed_count;
} else {
if (Debug)
Logger.Log.Debug ("Previously seen message, unchanged.");
}
this.tracker.Update (mi.uid, mi.flags);
return indexable;
}
public Indexable GetXmpQueryable (string path, Guid id, DirectoryModel parent)
{
Log.Debug ("Asked to create xmp indexable for ({0}) {1}", GuidFu.ToShortString (id), path);
// Should be at least 6 characters /<...>.xmp
if (path.Length < 6)
return null;
string basefile_path = Path.ChangeExtension (path, null);
// Ignore xmp files by itself
// FIXME: To support indexing independent xmp files will require even greater trouble
if (! File.Exists (basefile_path))
return null;
XmpFile xmp_file = null;
try {
xmp_file = new XmpFile (path);
} catch {
Log.Warn ("Cannot create xmpfile from {0}", path);
return null;
}
// Try to get the correct uid for the basefile
// First we need to see if basefile is already scheduled (yet to be dispatched)
Uri basefile_uri = null;
Indexable base_indexable;
if (uid_manager.HasNewId (basefile_path)) {
// Since uid_manager has a new id for this basefile, so basefile is already scheduled
// Get basefile uid from there
Guid basefile_id = uid_manager.GetNewId (basefile_path);
basefile_uri = GuidFu.ToUri (basefile_id);
Log.Debug ("{0} is already scheduled with uri {1}", basefile_path, basefile_uri);
} else {
// Basefile is not scheduled in the current batch
string basefile_name = Path.GetFileName (basefile_path);
// Try to schedule it for addition
base_indexable = queryable.GetCrawlingFileIndexable (parent, basefile_name);
if (base_indexable == null) {
// GetCrawlingFileIndexable returns null if file does not need to be indexed
// So basefile is up-to-date
// Need to figure out id from uid manager
Guid basefile_id = uid_manager.GetIdByNameAndParentId (basefile_name, parent.UniqueId);
basefile_uri = GuidFu.ToUri (basefile_id);
Log.Debug ("{0} is not scheduled and need not be, uri is {1}", basefile_path, basefile_uri);
} else {
Log.Debug ("Need to index {0}", basefile_path);
// basefile needs to be indexed
// FIXME: Move the task business out of handler and into FSQ.cs
Scheduler.Task task;
task = queryable.NewAddTask (base_indexable);
// FIXME: What is the correct priority ?
// If should have similar priority to the one that this xmp-indexable will be a part of
task.Priority = Scheduler.Priority.Immediate;
queryable.ThisScheduler.Add (task);
// Get the basefile uri from the indexable
basefile_uri = base_indexable.Uri;
}
}
Log.Debug ("Adding xmp-indexable for {0} (basefile uri {1}) with uid {2}",
path,
basefile_uri,
GuidFu.ToShortString (id));
Indexable indexable = new Indexable (IndexableType.PropertyChange, basefile_uri);
// Set the timestamp of the indexable as the timestamp of the basefile
// It could have also been skipped, the original Indexable.Add would anyway have it
indexable.Timestamp = File.GetLastWriteTimeUtc (basefile_path);
indexable.DisplayUri = UriFu.PathToFileUri (path);
// If the file was somehow deleted before this point, bail out.
if (! FileSystem.ExistsByDateTime (indexable.Timestamp)) {
xmp_file.Close ();
return null;
}
// Save some local states for PostAddHook, namely, path to the xmp file, path to basefile and generated uid
indexable.LocalState ["XmpFilePath"] = path;
indexable.LocalState ["BaseFilePath"] = basefile_path;
indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString (id);
// FIXME: Should also delete previous xmp properties!
foreach (Property p in xmp_file.Properties) {
p.IsMutable = true;
indexable.AddProperty (p);
}
xmp_file.Close ();
return indexable;
}
private Indexable CamelMessageToIndexable (B_U_Camel.MessageInfo messageInfo, string msg_file)
{
// Don't index messages flagged as junk
if (messageInfo.IsJunk)
return null;
// Many properties will be set by the filter when
// processing the cached data, if it's there. So
// don't set a number of properties in that case.
bool have_content = File.Exists (msg_file);
Uri uri = CamelMessageUri (messageInfo);
Indexable indexable = new Indexable (uri);
indexable.Timestamp = messageInfo.SentDate;
indexable.MimeType = "message/rfc822";
indexable.HitType = "MailMessage";
indexable.AddProperty (Property.NewUnsearched ("fixme:account", this.imap_name));
indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name));
indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution"));
if (!have_content) {
indexable.AddProperty (Property.New ("dc:title", GMime.Utils.HeaderDecodePhrase (messageInfo.subject)));
indexable.AddProperty (Property.NewDate ("fixme:date", messageInfo.SentDate));
}
GMime.InternetAddressList addrs;
addrs = GMime.InternetAddressList.Parse (messageInfo.to);
foreach (GMime.InternetAddress ia in addrs) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
if (!have_content) {
indexable.AddProperty (Property.NewUnsearched ("fixme:to", ia.ToString (false)));
if (ia is GMime.InternetAddressMailbox)
indexable.AddProperty (Property.New ("fixme:to_address", mailbox.Address));
indexable.AddProperty (Property.New ("fixme:to_name", ia.Name));
}
if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox)
indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address));
}
addrs.Dispose ();
addrs = GMime.InternetAddressList.Parse (messageInfo.cc);
foreach (GMime.InternetAddress ia in addrs) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
if (!have_content) {
indexable.AddProperty (Property.NewUnsearched ("fixme:cc", ia.ToString (false)));
if (ia is GMime.InternetAddressMailbox)
indexable.AddProperty (Property.New ("fixme:cc_address", mailbox.Address));
indexable.AddProperty (Property.New ("fixme:cc_name", ia.Name));
}
if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox)
indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address));
}
addrs.Dispose ();
addrs = GMime.InternetAddressList.Parse (messageInfo.from);
foreach (GMime.InternetAddress ia in addrs) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
if (!have_content) {
indexable.AddProperty (Property.NewUnsearched ("fixme:from", ia.ToString (false)));
if (ia is GMime.InternetAddressMailbox)
indexable.AddProperty (Property.New ("fixme:from_address", mailbox.Address));
indexable.AddProperty (Property.New ("fixme:from_name", ia.Name));
}
if (this.folder_name != "Sent" && ia is GMime.InternetAddressMailbox)
indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address));
}
addrs.Dispose ();
indexable.AddProperty (Property.NewKeyword ("fixme:mlist", messageInfo.mlist));
Property flag_prop = Property.NewUnsearched ("fixme:flags", messageInfo.flags);
flag_prop.IsMutable = true;
indexable.AddProperty (flag_prop);
if (this.folder_name == "Sent")
indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
if (messageInfo.IsAnswered)
indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
if (messageInfo.IsDeleted)
indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
if (messageInfo.IsDraft)
indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
if (messageInfo.IsFlagged)
indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
if (messageInfo.IsSeen)
indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
if (messageInfo.HasAttachments && !have_content)
indexable.AddProperty (Property.NewFlag ("fixme:hasAttachments"));
if (messageInfo.IsAnsweredAll)
indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
if (have_content)
indexable.ContentUri = UriFu.PathToFileUri (msg_file);
else
indexable.NoContent = true;
return indexable;
}
public Indexable GetNextIndexable ()
{
string id = reader.GetAttribute ("Id");
string title = reader.GetAttribute ("Title");
string author = reader.GetAttribute ("Author");
// FIXME stupid mono bug; DateTime.ParseExact ("0001-01-01T00:00:00.0000000+00:00", ...)
// http://bugzilla.ximian.com/show_bug.cgi?id=76082
// Still present in 1.1.9.2
DateTime pub_date;
try {
pub_date = DateTime.ParseExact (
reader.GetAttribute ("PubDate"),
"yyyy-MM-ddTHH:mm:ss.fffffffzzz",
null);
} catch (Exception e) {
pub_date = DateTime.MinValue;
}
string link = reader.GetAttribute ("Link");
string text = reader.GetAttribute ("Text");
reader.Read ();
Uri uri = new Uri (String.Format ("feed:{0};item={1}", channel_url, id));
Logger.Log.Debug ("BlamQ: Indexing [" + channel_name + "] " + title);
Indexable indexable = new Indexable (uri);
indexable.ParentUri = UriFu.PathToFileUri (feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
indexable.Timestamp = pub_date.ToUniversalTime ();
// change property names to DC names, as far as allowed
indexable.AddProperty (Property.New ("dc:title", title));
indexable.AddProperty (Property.New ("dc:creator", author));
indexable.AddProperty (Property.NewKeyword ("dc:identifier", link));
indexable.AddProperty (Property.NewKeyword ("dc:source", channel_url));
indexable.AddProperty (Property.New ("dc:publisher", channel_name));
string img = null;
int i = text.IndexOf ("<img src=\"");
if (i != -1) {
i += "<img src=\"".Length;
int j = text.IndexOf ("\"", i);
if (j != -1)
img = text.Substring (i, j-i);
}
if (img != null) {
string path = Path.Combine (Path.Combine (blam_dir, "Cache"),
img.GetHashCode ().ToString ());
indexable.AddProperty (Property.NewUnsearched ("fixme:cachedimg", path));
}
StringReader string_reader = new StringReader (text);
indexable.SetTextReader (string_reader);
return indexable;
}
private Indexable FileToIndexable (FileInfo data_file)
{
FileInfo meta_file = new FileInfo (Path.Combine (data_file.DirectoryName, "." + data_file.Name));
FileStream meta_stream;
try {
meta_stream = meta_file.Open (FileMode.Open, FileAccess.Read, FileShare.Read);
} catch (FileNotFoundException) {
// The meta file disappeared before we could
// open it.
return null;
}
StreamReader reader = new StreamReader (meta_stream);
// First line of the file is a URI
string line = reader.ReadLine ();
Uri uri;
try {
uri = new Uri (line);
} catch (Exception e) {
Logger.Log.Warn (e, "IndexingService: Unable to parse URI in {0}:", meta_file.FullName);
meta_stream.Close ();
return null;
}
Indexable indexable = new Indexable (uri);
indexable.Timestamp = data_file.LastWriteTimeUtc;
indexable.ContentUri = UriFu.PathToFileUri (data_file.FullName);
indexable.DeleteContent = true;
indexable.AddProperty( Property.New("fixme:host",uri.Host));
// Second line is the hit type
line = reader.ReadLine ();
if (line == null) {
Logger.Log.Warn ("IndexingService: EOF reached trying to read hit type from {0}",
meta_file.FullName);
meta_stream.Close ();
return null;
} else if (line != String.Empty)
indexable.HitType = line;
// Third line is the mime type
line = reader.ReadLine ();
if (line == null) {
Logger.Log.Warn ("IndexingService: EOF reached trying to read mime type from {0}",
meta_file.FullName);
meta_stream.Close ();
return null;
} else if (line != String.Empty)
indexable.MimeType = line;
// Following lines are properties in "t:key=value" format
do {
line = reader.ReadLine ();
if (line != null && line != String.Empty) {
bool keyword = false;
if (line[0] == 'k')
keyword = true;
else if (line[0] != 't') {
Logger.Log.Warn ("IndexingService: Unknown property type: '{0}'", line[0]);
continue;
}
int i = line.IndexOf ('=');
if (i == -1) {
Logger.Log.Warn ("IndexingService: Unknown property line: '{0}'", line);
continue;
}
// FIXME: We should probably handle date types
if (keyword) {
indexable.AddProperty (Property.NewUnsearched (line.Substring (2, i - 2),
line.Substring (i + 1)));
} else {
indexable.AddProperty (Property.New (line.Substring (2, i - 2),
line.Substring (i + 1)));
}
}
} while (line != null);
indexable.LocalState ["MetaFile"] = meta_file;
// Ok, we're finished with the meta file. It will be
// deleted in PostAddHook ().
meta_stream.Close ();
return indexable;
}
private Indexable current_itemToIndexable ()
{
// sanity check
if (current_item == null)
return null;
//Log.Debug ("Indexing " + channel_link + ":" + current_item.Link);
Indexable indexable = new Indexable (new Uri (String.Format ("feed:{0};item={1}", channel_link, current_item.Link)));
indexable.ParentUri = UriFu.PathToFileUri (feed_file);
indexable.MimeType = "text/html";
indexable.HitType = "FeedItem";
string RFC822 = "ddd, dd MMM yyyy HH:mm:ss zzz";
DateTime date = DateTime.ParseExact(current_item.PubDate, RFC822, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal);
indexable.Timestamp = date;
// replace property names with Dublin Core names
indexable.AddProperty (Property.New ("dc:title", current_item.Title));
indexable.AddProperty (Property.NewKeyword ("dc:identifier", current_item.Link));
indexable.AddProperty (Property.NewKeyword ("dc:source", channel_link));
indexable.AddProperty (Property.New ("dc:publisher", channel_title));
StringReader reader = new StringReader (current_item.Description);
indexable.SetTextReader (reader);
return indexable;
}
/**
* Create an indexable from a maildir message
*/
public Indexable MaildirMessageToIndexable (string filename, bool crawl)
{
//Logger.Log.Debug ("+ indexing maildir mail:" + filename);
String folder = GetFolderMaildir(filename);
Uri file_uri = UriFu.PathToFileUri (filename);
Indexable indexable = new Indexable (file_uri);
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.FlushBufferCache = crawl;
indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail"));
indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name));
indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder));
indexable.ContentUri = file_uri;
return indexable;
}
/**
* Create an indexable from an mbox message
* Most of the code here is from Evo backend
*/
public Indexable MessageToIndexable (string file_name, System.Uri uri, GMime.Message message, string folder_name)
{
//Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name);
Indexable indexable = new Indexable (uri);
// set parent uri to the filename so that when an mbox file
// is deleted, all the messages in that file can be deleted
indexable.ParentUri = UriFu.PathToFileUri (file_name);
indexable.Timestamp = message.Date.ToUniversalTime ();
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail"));
indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name));
indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder_name));
GMime.InternetAddressList addrs;
if (folder_name == Queryable.SentMailFolderName) {
addrs = message.GetRecipients (GMime.RecipientType.To);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (folder_name == Queryable.SentMailFolderName) {
addrs = message.GetRecipients (GMime.RecipientType.Cc);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (folder_name != Queryable.SentMailFolderName) {
addrs = GMime.InternetAddressList.Parse (message.Sender);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewKeyword ("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose ();
}
if (folder_name == Queryable.SentMailFolderName)
indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
else {
string kmail_msg_sent = message.GetHeader ("X-KMail-Link-Type");
if (kmail_msg_sent == "reply")
indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
}
// no need to store date again, use the issent flag to determine if the date is sentdate or not
#if false
if (folder_name == Queryable.SentMailFolderName)
indexable.AddProperty (Property.NewDate ("fixme:sentdate", message.Date.ToUniversalTime ()));
else
indexable.AddProperty (Property.NewDate ("fixme:received", message.Date.ToUniversalTime ()));
#endif
indexable.SetBinaryStream (message.Stream);
return indexable;
}
public Indexable GetNextIndexable ()
{
DateTime line_dt = DateTime.MinValue;
while (log_line_as_sb != null) {
//Log.Debug ("Checking line from {0}:[{1}]", log_file, log_line_as_sb);
bool in_session = AppendLogText (log_line_as_sb, out line_dt);
if (! in_session)
break;
prev_line_offset = reader.Position;
log_line_as_sb = reader.ReadLineAsStringBuilder ();
}
// Check if there is new data to index
if (data_sb.Length == 0) {
session_begin_time = line_dt;
return null;
}
Uri uri = new Uri (String.Format ("konversation://{0}@dumb/{1}", session_begin_offset, log_file));
Log.Debug ("Creating indexable {0}", uri);
Indexable indexable = new Indexable (uri);
indexable.ParentUri = UriFu.PathToFileUri (log_file);
indexable.Timestamp = session_begin_time;
indexable.HitType = "IMLog";
indexable.CacheContent = false;
indexable.Filtering = IndexableFiltering.AlreadyFiltered;
indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:session_begin_offset", session_begin_offset));
indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:session_end_offset", prev_line_offset));
indexable.AddProperty (Beagle.Property.NewDate ("fixme:starttime", session_begin_time));
indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:client", "Konversation"));
indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:protocol", "IRC"));
AddChannelInformation (indexable);
foreach (string speaker in speakers.Keys)
indexable.AddProperty (Beagle.Property.NewUnstored ("fixme:speaker", speaker));
StringReader data_reader = new StringReader (data_sb.ToString ());
indexable.SetTextReader (data_reader);
// update session begin time to the date of the current line which is not in this session
session_begin_time = line_dt;
return indexable;
}
