///////////////////////////////////////////////// private Indexable NoteToIndexable(FileInfo file, Note note) { Indexable indexable = new Indexable(note.Uri); indexable.ContentUri = UriFu.PathToFileUri(file.FullName); indexable.Timestamp = note.timestamp; indexable.HitType = "Note"; indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty(Property.New("dc:title", note.subject)); indexable.AddProperty(Property.NewUnsearched("fixme:application", "tomboy")); // FIXME: tagging is disabled in Tomboy-0.8.x and is planned for 0.10.0 foreach (string s in note.tags) { indexable.AddProperty(Property.New("note:tag", s)); } // We remember the note's text so that we can stuff it in // the TextCache later. note_text_cache [note.Uri] = note.text; StringReader reader = new StringReader(note.text); indexable.SetTextReader(reader); return(indexable); }
private Indexable BookmarkToIndexable() { Indexable indexable = new Indexable(new Uri(current_bookmark_id)); indexable.ParentUri = UriFu.PathToFileUri(bookmark_file); indexable.HitType = "Bookmark"; indexable.Timestamp = current_dt; indexable.NoContent = true; indexable.AddProperty(Property.New("dc:title", current_bookmark.Title)); indexable.AddProperty(Property.New("dc:identifier", current_bookmark.Href)); indexable.AddProperty(Property.NewUnsearched("fixme:icon", current_bookmark.Icon)); if (current_bookmark.Info != null && current_bookmark.Info.Metadata != null && current_bookmark.Info.Metadata.NumVisited != 0) { indexable.AddProperty(Property.NewUnsearched("fixme:visit_count", current_bookmark.Info.Metadata.NumVisited)); } foreach (string folder in folder_stack) { indexable.AddProperty(Property.New("fixme:folder", folder)); } return(indexable); }
private Indexable current_itemToIndexable() { // sanity check if (current_item == null) { return(null); } //Log.Debug ("Indexing " + channel_link + ":" + current_item.Link); Indexable indexable = new Indexable(new Uri(String.Format("feed:{0};item={1}", channel_link, current_item.Link))); indexable.ParentUri = UriFu.PathToFileUri(feed_file); indexable.MimeType = "text/html"; indexable.HitType = "FeedItem"; string RFC822 = "ddd, dd MMM yyyy HH:mm:ss zzz"; DateTime date = DateTime.ParseExact(current_item.PubDate, RFC822, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal); indexable.Timestamp = date; // replace property names with Dublin Core names indexable.AddProperty(Property.New("dc:title", current_item.Title)); indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Link)); indexable.AddProperty(Property.NewKeyword("dc:source", channel_link)); indexable.AddProperty(Property.New("dc:publisher", channel_title)); StringReader reader = new StringReader(current_item.Description); indexable.SetTextReader(reader); return(indexable); }
public Indexable GetNextIndexable() { if (initial_scan) { return(null); } Uri uri = new Uri(String.Format("kabc:///{0}", current_uid)); Indexable indexable = new Indexable(uri); indexable.ParentUri = UriFu.PathToFileUri(kabc_file); indexable.MimeType = ICalParser.KabcMimeType; indexable.HitType = "Contact"; indexable.Timestamp = current_dt; indexable.AddProperty(Property.NewKeyword("fixme:client", "KAddressBook")); // Add uid as a keyword field for convenience indexable.AddProperty(Property.NewUnsearched("fixme:uid", current_uid)); // FIXME: Comment this Debug statement after the backend stabilizes //Log.Debug ("Creating {0} from:[{1}]", uri, string_builder.ToString ()); StringReader string_reader = new StringReader(string_builder.ToString()); indexable.SetTextReader(string_reader); return(indexable); }
private Indexable OperaRowToIndexable(OperaHistory.Row row) { // It's unsafe to index secure content since it may contain sensitive data if (row.Address.Scheme == Uri.UriSchemeHttps) { return(null); } Indexable indexable = new Indexable(row.Address); indexable.HitType = "WebHistory"; indexable.MimeType = row.MimeType; indexable.Timestamp = row.LastVisited; indexable.AddProperty(Beagle.Property.New("fixme:host", row.Address.Host)); indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:size", row.Length)); // hint for the filter about the charset indexable.AddProperty(Property.NewUnsearched(StringFu.UnindexedNamespace + "charset", row.Encoding.ToString())); if (row.Compression == "gzip") { indexable.SetBinaryStream(new GZipInputStream(File.OpenRead(Path.Combine(cache_dir, row.LocalFileName)))); } else { indexable.ContentUri = new Uri(Path.Combine(cache_dir, row.LocalFileName)); } indexer.AttributeStore.AttachLastWriteTime(Path.Combine(cache_dir, row.LocalFileName), DateTime.UtcNow); return(indexable); }
static Indexable FileToIndexable(FileInfo file) { if (!file.Exists) { return(null); } if (fa_store.IsUpToDateAndFiltered(PathInIndex(file.FullName), FileSystem.GetLastWriteTimeUtc(file.FullName))) { return(null); } // Create the indexable and add the standard properties we // use in the FileSystemQueryable. Uri uri = PathToUri(file.FullName); Indexable indexable = new Indexable(uri); indexable.Timestamp = file.LastWriteTimeUtc; indexable.FlushBufferCache = true; indexable.AddProperty(Property.NewUnsearched("fixme:filesize", file.Length)); FSQ.AddStandardPropertiesToIndexable(indexable, file.Name, Guid.Empty, false); // Store directory name in the index string dirname = file.DirectoryName; indexable.AddProperty(Property.NewUnsearched(Property.ParentDirUriPropKey, PathToUri(dirname))); return(indexable); }
private Indexable MemoToIndexable(CalComponent cc) { Indexable indexable = new Indexable(GetComponentUri(cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Note"; indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty(Property.NewUnsearched("fixme:application", "evolution")); indexable.AddProperty(Property.New("dc:title", cc.Summary)); // We remember the note's text so that we can stuff it in // the TextCache later. // This is here form compability with Tomboy notes. foreach (string description in cc.Descriptions) { queryable.IndexableTextCache [indexable.Uri] = description; StringReader reader = new StringReader(description); indexable.SetTextReader(reader); } return(indexable); }
static Indexable DirectoryToIndexable(DirectoryInfo dir, Queue modified_directories) { if (!dir.Exists) { return(null); } // Check if the directory information is stored in attributes store // And if the mtime of the directory is same as that in the attributes store FileAttributes attr = fa_store.Read(PathInIndex(dir.FullName)); // If the directory exists in the fa store, then it is already indexed. if (attr != null) { // If we don't care about deleted content then we are fine. // If the attributes are up-to-date, then we are fine too. if (!arg_delete || FileAttributesStore.IsUpToDate(attr, FileSystem.GetLastWriteTimeUtc(dir.FullName))) { return(null); } // But the last write time needs to be uptodate to support enable-deletion, // so we actually index the directories, even if --disable-directories // is set. modified_directories.Enqueue(dir); } // Create the indexable and add the standard properties we // use in the FileSystemQueryable. Uri uri = PathToUri(dir.FullName); Indexable indexable = new Indexable(uri); indexable.MimeType = "inode/directory"; indexable.NoContent = true; indexable.Timestamp = dir.LastWriteTimeUtc; // Store the directory information in the index anyway, but if --disable-directories // was passed, then do not store the names and other standard properties // used during searching if (!arg_disable_directories) { FSQ.AddStandardPropertiesToIndexable(indexable, dir.Name, Guid.Empty, false); } // Add directory name property string dirname = dir.Parent.FullName; indexable.AddProperty(Property.NewUnsearched(Property.ParentDirUriPropKey, PathToUri(dirname))); indexable.AddProperty(Property.NewBool(Property.IsDirectoryPropKey, true)); if (arg_removable) { indexable.AddProperty(Property.NewKeyword("beagle:removable", volume_label)); } return(indexable); }
private void AddChannelInformation(Indexable indexable) { // FIXME: Parse identity information from konversation .config file //AddProperty (Beagle.Property.NewUnsearched ("fixme:identity", log.Identity)); // Get server name, channel name from the filename and add it here indexable.AddProperty(Beagle.Property.NewKeyword("fixme:server", server_name)); indexable.AddProperty(Beagle.Property.NewKeyword("fixme:speakingto", speaking_to)); }
private Indexable ToAddRssIndexable(XmlDocument document) { string encoding_str = null; StringReader reader = null; if (ToBool(GetText(document, "HasOffline"))) { try { // RSS does not use OfflineSize but MessageSize instead (for some reason...) int offset = Convert.ToInt32(GetText(document, "MessageOffset")), size = Convert.ToInt32(GetText(document, "MessageSize")); reader = GetRssBody(GetText(document, "FolderFile"), offset, size, out encoding_str); } catch (Exception e) { Logger.Log.Debug(e, "Failed to parse RSS body"); } } Indexable indexable = new Indexable(GenerateUniqueUri(document)); indexable.HitType = "FeedItem"; indexable.MimeType = "text/html"; indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc(Convert.ToInt64(GetText(document, "Date"))); indexable.CacheContent = true; indexable.FlushBufferCache = true; indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewKeyword("fixme:folder", GetText(document, "Folder"))); indexable.AddProperty(Property.NewUnsearched("ParentUri", GetText(document, "FolderFile"))); indexable.AddProperty(Property.NewUnsearched("fixme:uri", GetText(document, "Uri"))); indexable.AddProperty(Property.NewKeyword("dc:identifier", ExtractUrl(GetText(document, "MessageId")))); indexable.AddProperty(Property.NewKeyword("dc:source", GetText(document, "FeedURL"))); indexable.AddProperty(Property.New("dc:publisher", Mime.HeaderDecodePhrase(GetText(document, "Author")))); // The title will be added by the filter. In case we add it twice we will just get // an empty tile in the search tool (a bug maybe?). if (reader != null) { // If we got an encoding, make sure we use that if (!String.IsNullOrEmpty(encoding_str)) { indexable.AddProperty(Property.New( String.Format("{0}encoding", StringFu.UnindexedNamespace), encoding_str)); } indexable.SetTextReader(reader); } else { indexable.AddProperty(Property.New("dc:title", Mime.HeaderDecodePhrase(GetText(document, "Subject")))); } return(indexable); }
static private Indexable MemberNodeToIndexable(XmlNode node, Uri base_uri, string parentName) { char memberType = MemberTypeToChar(node.SelectSingleNode("MemberType").InnerText); StringBuilder memberFullName = new StringBuilder(); memberFullName.Append(memberType + ":" + parentName); if (memberType != 'C') { memberFullName.Append("." + node.Attributes["MemberName"].Value); } if (memberType == 'C' || memberType == 'M' || memberType == 'E') { memberFullName.Append("("); bool inside = false; foreach (XmlNode parameter in node.SelectNodes("Parameters/Parameter")) { if (!inside) { inside = true; } else { memberFullName.Append(","); } memberFullName.Append(parameter.Attributes["Type"].Value); } memberFullName.Append(")"); } Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, memberFullName.ToString(), false)); indexable.MimeType = "text/html"; indexable.HitType = "MonodocEntry"; indexable.AddProperty(Property.New("dc:title", memberFullName.ToString())); indexable.AddProperty(Property.New("fixme:name", memberFullName.ToString())); indexable.AddProperty(Property.NewUnsearched("fixme:type", node.SelectSingleNode("MemberType").InnerText.ToLower())); StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml); indexable.SetTextReader(reader); return(indexable); }
protected virtual Indexable NewIndexable(Uri uri, DateTime timestamp, string hit_type) { Indexable indexable; indexable = new Indexable(uri); indexable.HitType = hit_type; indexable.Timestamp = timestamp; indexable.AddProperty(Property.NewKeyword("fixme:account", account.Server)); indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewUnsearched("fixme:fullyIndexed", full_index)); indexable.AddProperty(Property.NewUnsearched("fixme:file", RelativePath)); indexable.AddProperty(Property.NewDate("fixme:indexDateTime", DateTime.UtcNow)); return(indexable); }
///////////////////////////////////////////////////////////////// static void AddToRequest(Indexable indexable) { if (indexable == null) { return; } // Disable filtering and only index file attributes if (arg_disable_filtering) { indexable.Filtering = IndexableFiltering.Never; } // Tag the item for easy identification (for say, removal) if (arg_tag != null) { indexable.AddProperty(Property.NewUnsearched("Tag", arg_tag)); } indexable.Source = arg_source; pending_request.Add(indexable); bool reschedule = false; do { if (Shutdown.ShutdownRequested) { break; } if (!reschedule && pending_request.Count < BATCH_SIZE) { break; } if (reschedule) { Logger.Log.Debug("Continuing indexing indexer generated indexables"); } else { Logger.Log.Debug("Flushing driver, {0} items in queue", pending_request.Count); } reschedule = FlushIndexer(driver); // Super Lame Hack: gtk-sharp up to 2.10 requires a main loop // to dispose of any managed wrappers around GObjects. Since // we don't have one, we'll process all the pending items in // a loop here. This is particularly an issue with maildirs, // because we need the loop to clean up after GMime. Without // it, GMime's streams are never completely unref'd, the // file descriptors aren't closed, and we run out and crash. while (GLib.MainContext.Pending()) { GLib.MainContext.Iteration(); } } while (reschedule); }
private Indexable TodoToIndexable(CalComponent cc) { Indexable indexable = new Indexable(GetComponentUri(cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Task"; indexable.AddProperty(Property.NewUnsearched("fixme:source_uid", this.source.Uid)); indexable.AddProperty(Property.NewUnsearched("fixme:uid", cc.Uid)); indexable.AddProperty(Property.NewDate("fixme:starttime", cc.Dtstart.ToUniversalTime())); if (cc.Dtend != DateTime.MinValue) { indexable.AddProperty(Property.NewDate("fixme:endtime", cc.Dtend.ToUniversalTime())); } foreach (string description in cc.Descriptions) { indexable.AddProperty(Property.New("fixme:description", description)); } indexable.AddProperty(Property.New("fixme:summary", cc.Summary)); foreach (string category in cc.Categories) { indexable.AddProperty(Property.NewUnsearched("fixme:category", category)); } return(indexable); }
static private Indexable TypeNodeToIndexable(XmlNode node, Uri base_uri) { string fragment = "T:" + node.Attributes ["FullName"].Value; Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, fragment, false)); indexable.MimeType = "text/html"; indexable.HitType = "MonodocEntry"; indexable.AddProperty(Property.New("dc:title", "T:" + node.Attributes["FullName"].Value)); indexable.AddProperty(Property.NewUnsearched("fixme:name", "T:" + node.Attributes["FullName"].Value)); indexable.AddProperty(Property.NewUnsearched("fixme:type", "type")); StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml); indexable.SetTextReader(reader); return(indexable); }
public static void SendUpdate(IBrowsableItem item) { Indexable indexable = new Indexable(item.DefaultVersionUri); indexable.Type = IndexableType.PropertyChange; Beagle.Property prop; // Clear the existing tags prop = Beagle.Property.NewKeyword("fspot:Tag", ""); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); prop = Beagle.Property.NewKeyword("image:Tag", ""); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); foreach (Tag t in item.Tags) { prop = Beagle.Property.NewKeyword("fspot:Tag", t.Name); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); prop = Beagle.Property.NewKeyword("image:Tag", t.Name); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); } prop = Beagle.Property.New("fspot:Description", item.Description); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); // Create a message to send to the daemon with this information. // The source tells it what index the existing "/home/joe/test.txt" document lives. IndexingServiceRequest req = new IndexingServiceRequest(); req.Keepalive = false; req.Source = "Files"; req.Add(indexable); req.SendAsync(); }
private Indexable GetNewXmpIndexable(ref Indexable indexable, string path, Guid id, DirectoryModel parent) { // In non-crawl mode, check if a corresponding xmp file is present and not already scheduled and index it. // If file.xmp and file are rapidly written/updated (in that order), this does the right thing. // If file and file.xmp are rapidly written/updated (in that order), either // - file.xmp is present during FileToIndexable(file): in which case xmp properties are // added to file; and when file.xmp is indexed, it will replace the xmp properties // - file.xmp is not present during FileToIndexable(file): when the xmp file is later indexed // it will add the xmp properties // since the uid file will still be in the uid-cache, correct uid will be used for xmp prop-change indexable string possible_xmp_file_path = string.Concat(path, ".xmp"); if (!File.Exists(possible_xmp_file_path)) { return(null); } Guid xmp_id = queryable.RegisterFile(parent, (Path.GetFileName(possible_xmp_file_path))); if (xmp_id == Guid.Empty) { return(null); } XmpFile xmp_file = null; try { xmp_file = new XmpFile(possible_xmp_file_path); } catch { uid_manager.ForgetNewId(possible_xmp_file_path); return(null); } // FIXME: Should also delete previous xmp properties! foreach (Property p in xmp_file.Properties) { p.IsMutable = true; indexable.AddProperty(p); } xmp_file.Close(); // Also need to save some local states for PostAddHook, // namely, path to the xmp file, path to basefile and generated uid indexable.LocalState ["XmpFilePath"] = possible_xmp_file_path; indexable.LocalState ["BaseFilePath"] = path; indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString(xmp_id); if (Debug) { Log.Debug("Adding properties from {0}({2}) to {1}({3})", possible_xmp_file_path, path, GuidFu.ToShortString(xmp_id), GuidFu.ToShortString(id)); } return(null); }
// FIXME: This need some more info private Indexable NntpMessageToIndexable(TB.NntpMessage message) { Indexable indexable; indexable = new Indexable(message.Uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTime.Parse(message.GetString("date")).ToUniversalTime(); indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewUnsearched("fixme:fullyIndexed", message.GetBool("FullIndex"))); indexable.AddProperty(Property.NewDate("fixme:indexDateTime", DateTime.UtcNow)); string subject = GMime.Utils.HeaderDecodePhrase(message.GetString("subject")); indexable.AddProperty(Property.New("dc:title", subject)); return(indexable); }
private Indexable EventToIndexable(CalComponent cc) { Indexable indexable = new Indexable(GetComponentUri(cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Calendar"; indexable.AddProperty(Property.NewUnsearched("fixme:source_uid", this.source.Uid)); indexable.AddProperty(Property.NewUnsearched("fixme:uid", cc.Uid)); indexable.AddProperty(Property.NewDate("fixme:starttime", cc.Dtstart.ToUniversalTime())); if (cc.Dtend != DateTime.MinValue) { indexable.AddProperty(Property.NewDate("fixme:endtime", cc.Dtend.ToUniversalTime())); } foreach (CalComponentAttendee attendee in cc.Attendees) { indexable.AddProperty(Property.New("fixme:attendee", attendee.value)); } foreach (string comment in cc.Comments) { indexable.AddProperty(Property.New("fixme:comment", comment)); } foreach (string description in cc.Descriptions) { indexable.AddProperty(Property.New("fixme:description", description)); } indexable.AddProperty(Property.New("fixme:summary", cc.Summary)); foreach (string category in cc.Categories) { indexable.AddProperty(Property.NewUnsearched("fixme:category", category)); } indexable.AddProperty(Property.New("fixme:location", cc.Location)); return(indexable); }
///////////////////////////////////////////////// private static Indexable NoteToIndexable(FileInfo file) { Indexable indexable = new Indexable(UriFu.PathToFileUri(file.FullName)); indexable.Timestamp = file.LastWriteTimeUtc; indexable.HitType = "Note"; indexable.MimeType = "x-beagle/x-labyrinth-note"; indexable.AddProperty(Property.NewUnsearched("fixme:application", "labyrinth")); return(indexable); }
/** * Create an indexable from a maildir message */ public Indexable MaildirMessageToIndexable(string filename, bool crawl) { //Logger.Log.Debug ("+ indexing maildir mail:" + filename); String folder = GetFolderMaildir(filename); Uri file_uri = UriFu.PathToFileUri(filename); Indexable indexable = new Indexable(file_uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.FlushBufferCache = crawl; indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail")); indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name)); indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder)); indexable.ContentUri = file_uri; return(indexable); }
private Indexable ToAddMailMessageIndexable(XmlDocument document) { GMime.Message message = null; // Check if the entire message is available if (ToBool(GetText(document, "HasOffline"))) { // We must make sure we don't get an exception here since we can fallback to // other information try { int offset = Convert.ToInt32(GetText(document, "MessageOffset")), size = Convert.ToInt32(GetText(document, "OfflineSize")); message = GetGMimeMessage(GetText(document, "FolderFile"), offset, size); } catch (Exception e) { Logger.Log.Debug(e, "Failed to parse GMime message"); } } if (message == null) { message = GetStubMessage(document); } Indexable indexable = new Indexable(GenerateUniqueUri(document)); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc(Convert.ToInt64(GetText(document, "Date"))); indexable.CacheContent = true; indexable.FlushBufferCache = true; indexable.SetBinaryStream(message.Stream); indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewKeyword("fixme:folder", GetText(document, "Folder"))); indexable.AddProperty(Property.NewUnsearched("ParentUri", GetText(document, "FolderFile"))); indexable.AddProperty(Property.NewUnsearched("fixme:uri", GetText(document, "Uri"))); message.Dispose(); return(indexable); }
///////////////////////////////////////////////// public Indexable GetIndexable(NautilusTools.NautilusMetadata nm) { Indexable indexable = new Indexable(nm.Uri); indexable.Type = IndexableType.PropertyChange; Property prop; // Reset the notes property. if (nm.Notes == null) { nm.Notes = String.Empty; } prop = Property.New("nautilus:notes", nm.Notes); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); foreach (string emblem in nm.Emblems) { prop = Property.NewKeyword("nautilus:emblem", emblem); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); } // We add an empty keyword so that the property is reset if (nm.Emblems.Count == 0) { prop = Property.NewKeyword("nautilus:emblem", String.Empty); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); } return(indexable); }
public Indexable GetNextIndexable() { DateTime line_dt = DateTime.MinValue; while (log_line_as_sb != null) { //Log.Debug ("Checking line from {0}:[{1}]", log_file, log_line_as_sb); bool in_session = AppendLogText(log_line_as_sb, out line_dt); if (!in_session) { break; } prev_line_offset = reader.Position; log_line_as_sb = reader.ReadLineAsStringBuilder(); } // Check if there is new data to index if (data_sb.Length == 0) { session_begin_time = line_dt; return(null); } Uri uri = new Uri(String.Format("konversation://{0}@dumb/{1}", session_begin_offset, log_file)); Log.Debug("Creating indexable {0}", uri); Indexable indexable = new Indexable(uri); indexable.ParentUri = UriFu.PathToFileUri(log_file); indexable.Timestamp = session_begin_time; indexable.HitType = "IMLog"; indexable.CacheContent = false; indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_begin_offset", session_begin_offset)); indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:session_end_offset", prev_line_offset)); indexable.AddProperty(Beagle.Property.NewDate("fixme:starttime", session_begin_time)); indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:client", "Konversation")); indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:protocol", "IRC")); AddChannelInformation(indexable); foreach (string speaker in speakers.Keys) { indexable.AddProperty(Beagle.Property.NewUnstored("fixme:speaker", speaker)); } StringReader data_reader = new StringReader(data_sb.ToString()); indexable.SetTextReader(data_reader); // update session begin time to the date of the current line which is not in this session session_begin_time = line_dt; return(indexable); }
protected override Indexable MessageToIndexable(Message message) { Uri uri = new Uri(String.Format("feed:///{0};id={1}", message.Path, message.Id)); Indexable indexable = new Indexable(uri); indexable.MimeType = "text/html"; indexable.Type = "FeedItem"; indexable.AddProperty(Property.New("fixme:client", "mozilla")); indexable.AddProperty(Property.New("dc:title", message.Subject)); indexable.AddProperty(Property.New("fixme:author", message.From)); //indexable.AddProperty(Property.NewDate ("fixme:published", item.PubDate)); indexable.AddProperty(Property.NewUnsearched("fixme:itemuri", message.Headers ["Content-Base"])); indexable.AddProperty(Property.New("fixme:offset", message.Offset)); StringReader reader = new StringReader(message.Body); indexable.SetTextReader(reader); return(indexable); }
Indexable TypeNodeToIndexable(XmlNode node, FileInfo file) { Indexable indexable = new Indexable( new Uri("monodoc:///" + file + ";item=T:" + node.Attributes["FullName"].Value)); indexable.MimeType = "application/monodoc"; indexable.Type = "Monodoc"; indexable.AddProperty(Property.NewUnsearched("fixme:type", "type")); indexable.AddProperty(Property.NewUnsearched("fixme:name", "T:" + node.Attributes["FullName"].Value)); string splitname = String.Join(" ", StringFu.FuzzySplit(node.Attributes["FullName"].Value.ToString())); indexable.AddProperty(Property.NewKeyword("fixme:splitname", splitname)); // Should we add other stuff here? Implemented interfaces etc? StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml); indexable.SetTextReader(reader); return(indexable); }
private Indexable current_itemToIndexable() { Indexable indexable; try { indexable = new Indexable(new Uri(String.Format("{0};item={1}", feed_source, current_item.Source))); } catch (System.UriFormatException) { indexable = new Indexable(new Uri(String.Format("liferea://dummy?{0};item={1}", feed_source, current_item.Source))); } indexable.ParentUri = UriFu.PathToFileUri(feed_file); indexable.MimeType = "text/html"; indexable.HitType = "FeedItem"; DateTime date = DateTimeUtil.UnixToDateTimeUtc(0); date = date.AddSeconds(current_item.Timestamp); indexable.Timestamp = date; // cleaning up the property names as far as possible // this way querying for specific field is possible // following DC element names wherever applicable indexable.AddProperty(Property.New("dc:title", current_item.Title)); Attribute[] attribs = current_item.Attribs.AttribArray; if (attribs != null) { foreach (Attribute attrib in attribs) { if (attrib.Name != "author") { continue; } indexable.AddProperty(Property.New("dc:creator", attrib.Value)); } } indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Source)); indexable.AddProperty(Property.NewKeyword("dc:source", feed_source)); indexable.AddProperty(Property.New("dc:publisher", publisher)); if (File.Exists(icon_file)) { indexable.AddProperty(Property.NewUnsearched("fixme:cachedimg", icon_file)); } StringReader reader = new StringReader(current_item.Description); indexable.SetTextReader(reader); return(indexable); }
///////////////////////////////////////////////// private Indexable PidginLogToIndexable(string filename) { FileInfo info = new FileInfo(filename); Uri uri = UriFu.PathToFileUri(filename); Indexable indexable = new Indexable(uri); indexable.ContentUri = uri; indexable.Timestamp = info.LastWriteTimeUtc; indexable.MimeType = "beagle/x-pidgin-log"; indexable.HitType = "IMLog"; indexable.CacheContent = false; ImBuddy buddy = queryable.ImBuddyListReader.Search(info.Directory.Name); if (buddy != null && !String.IsNullOrEmpty(buddy.Alias)) { indexable.AddProperty(Property.NewKeyword("fixme:alias", buddy.Alias)); } return(indexable); }
protected override Indexable MessageToIndexable (Message message) { Uri uri = new Uri (String.Format ("email:///{0};id={1}", message.Path, message.Id)); Indexable indexable = new Indexable (uri); indexable.Type = "MailMessage"; indexable.AddProperty (Property.New ("fixme:client", "mozilla")); indexable.AddProperty (Property.New ("dc:title", message.Subject)); indexable.AddProperty (Property.New ("fixme:subject", message.Subject)); indexable.AddProperty (Property.New ("fixme:to", message.To)); indexable.AddProperty (Property.New ("fixme:from", message.From)); indexable.AddProperty (Property.New ("fixme:offset", message.Offset)); StringReader reader = new StringReader (message.Body); indexable.SetTextReader (reader); return indexable; }
protected override Indexable MessageToIndexable(Message message) { Uri uri = new Uri(String.Format("email:///{0};id={1}", message.Path, message.Id)); Indexable indexable = new Indexable(uri); indexable.Type = "MailMessage"; indexable.AddProperty(Property.New("fixme:client", "mozilla")); indexable.AddProperty(Property.New("dc:title", message.Subject)); indexable.AddProperty(Property.New("fixme:subject", message.Subject)); indexable.AddProperty(Property.New("fixme:to", message.To)); indexable.AddProperty(Property.New("fixme:from", message.From)); indexable.AddProperty(Property.New("fixme:offset", message.Offset)); StringReader reader = new StringReader(message.Body); indexable.SetTextReader(reader); return(indexable); }
override protected void DoPullProperties () { Stopwatch watch = new Stopwatch (); watch.Start (); while (reader.Read ()) { switch (reader.NodeType) { case XmlNodeType.Element: if (reader.Name.StartsWith ("sect") || reader.Name.StartsWith ("chapter")) { string id = reader.GetAttribute ("id"); if (id != null && id != String.Empty) { DocbookEntry entry = new DocbookEntry (); entry.Id = id; entry.Depth = reader.Depth; string language = reader.GetAttribute ("lang"); if (language != null && language != String.Empty) entry.Language = language; entries_stack.Push (entry); } } else if (reader.Name == "article" || reader.Name == "book") { string language = reader.GetAttribute ("lang"); if (language != null && language != String.Empty) base_language = language; } else if (reader.Name == "title") { reader.Read (); // Go to the text node if (entries_stack.Count == 0 && base_title == null) { // This is probably the book title base_title = reader.Value; } else if (entries_stack.Count > 0) { DocbookEntry entry = (DocbookEntry) entries_stack.Peek (); if (entry.Title == null) entry.Title = reader.Value; } } else if (reader.Name == "keyword") { reader.Read (); // read the text node AddProperty (Property.NewKeyword ("dc:subject", reader.Value)); } break; case XmlNodeType.Text: // Append text to the child indexable if (entries_stack.Count > 0) ((DocbookEntry) entries_stack.Peek ()).Content.Append (reader.Value); // Append text to the main indexable else AppendWord (reader.Value); break; case XmlNodeType.EndElement: if (entries_stack.Count > 0 && ((DocbookEntry) entries_stack.Peek ()).Depth == reader.Depth) { DocbookEntry entry, parent_entry = null; entry = (DocbookEntry) entries_stack.Pop (); if (entries_stack.Count > 0) parent_entry = (DocbookEntry) entries_stack.Peek (); Indexable indexable; indexable = new Indexable (UriFu.AddFragment (Indexable.Uri, entry.Id, false)); indexable.HitType = "DocbookEntry"; indexable.MimeType = "text/x-docbook-entry"; indexable.AddProperty (Property.NewKeyword ("beagle:FileType", "documentation")); indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty (Property.NewUnsearched ("fixme:id", entry.Id)); indexable.AddProperty (Property.New ("dc:title", entry.Title)); // Add the docbook book title indexable.AddProperty (Property.NewUnsearched ("fixme:base_title", base_title)); // Add the child language (or docbook language if none is specified) if (entry.Language != null) indexable.AddProperty (Property.NewUnsearched ("fixme:language", entry.Language)); else if (base_language != null) indexable.AddProperty (Property.NewUnsearched ("fixme:language", base_language)); // Add any parent (as in docbook parent entry, not beagle) data if we have it if (parent_entry != null) { indexable.AddProperty (Property.NewUnsearched ("fixme:parent_id", parent_entry.Id)); indexable.AddProperty (Property.NewUnsearched ("fixme:parent_title", parent_entry.Title)); } StringReader content_reader = new StringReader (entry.Content.ToString ()); indexable.SetTextReader (content_reader); indexable.SetChildOf (this.Indexable); AddIndexable (indexable); } break; } } // Add the common properties to the top-level // file item such as Title, Language etc. AddProperty (Property.New ("dc:title", base_title)); AddProperty (Property.NewUnsearched ("fixme:language", base_language)); watch.Stop (); // If we've successfully crawled the file but haven't // found any indexables, we shouldn't consider it // successfull at all (unless we have a title, which // means that it's actually a docbook file, just without // sections. if (! HasGeneratedIndexable && base_title == null) { Log.Error ("Probably not a docbook. Ignoring {0}!", base_path); Error (); return; } Logger.Log.Debug ("Parsed docbook file in {0}", watch); Finished (); }
static void Main(String[] args) { string uriStr = null; string title = null; string sourcefile = null; bool deletesourcefile = false; if (args.Length == 0 || Array.IndexOf(args, "--help") > -1) { PrintUsage(); Environment.Exit(1); } for (int i = 0; i < args.Length; i++) { switch (args [i]) { case "--url": case "--title": case "--sourcefile": if (i + 1 >= args.Length || args [i + 1].StartsWith("--")) { PrintUsage(); Environment.Exit(1); } break; } switch (args [i]) { case "--url": uriStr = args [++i]; break; case "--title": title = args [++i]; break; case "--sourcefile": sourcefile = args [++i]; break; case "--deletesourcefile": deletesourcefile = true; break; case "--help": PrintUsage(); return; case "--version": VersionFu.PrintVersion(); return; } } if (uriStr == null) { Logger.Log.Error("URI not specified!\n"); PrintUsage(); Environment.Exit(1); } Uri uri = new Uri(uriStr, true); if (uri.Scheme == Uri.UriSchemeHttps) { // For security/privacy reasons, we don't index any // SSL-encrypted pages. Logger.Log.Error("Indexing secure https:// URIs is not secure!"); Environment.Exit(1); } // We don't index file: Uris. Silently exit. if (uri.IsFile) { return; } // We *definitely* don't index mailto: Uris. Silently exit. if (uri.Scheme == Uri.UriSchemeMailto) { return; } Indexable indexable; indexable = new Indexable(uri); indexable.HitType = "WebHistory"; indexable.MimeType = "text/html"; indexable.Timestamp = DateTime.Now; if (title != null) { indexable.AddProperty(Property.New("dc:title", title)); } if (sourcefile != null) { if (!File.Exists(sourcefile)) { Logger.Log.Error("sourcefile '{0}' does not exist!", sourcefile); Environment.Exit(1); } indexable.ContentUri = UriFu.PathToFileUri(sourcefile); indexable.DeleteContent = deletesourcefile; } else { Stream stdin = Console.OpenStandardInput(); if (stdin == null) { Logger.Log.Error("No sourcefile specified, and no standard input!\n"); PrintUsage(); Environment.Exit(1); } indexable.SetTextReader(new StreamReader(stdin)); } IndexingServiceRequest req = new IndexingServiceRequest(); req.Add(indexable); try { Logger.Log.Info("Indexing"); Logger.Log.Debug("SendAsync"); req.SendAsync(); Logger.Log.Debug("Close"); req.Close(); Logger.Log.Debug("Done"); } catch (Exception e) { Logger.Log.Error("Indexing failed: {0}", e); // Still clean up after ourselves, even if we couldn't // index the content. if (deletesourcefile) { File.Delete(sourcefile); } Environment.Exit(1); } }
private void AddChannelInformation (Indexable indexable) { // FIXME: Parse identity information from konversation .config file //AddProperty (Beagle.Property.NewUnsearched ("fixme:identity", log.Identity)); // Get server name, channel name from the filename and add it here indexable.AddProperty (Beagle.Property.NewKeyword ("fixme:server", server_name)); indexable.AddProperty (Beagle.Property.NewKeyword ("fixme:speakingto", speaking_to)); }
///////////////////////////////////////////////// private Indexable PidginLogToIndexable (string filename) { FileInfo info = new FileInfo (filename); Uri uri = UriFu.PathToFileUri (filename); Indexable indexable = new Indexable (uri); indexable.ContentUri = uri; indexable.Timestamp = info.LastWriteTimeUtc; indexable.MimeType = "beagle/x-pidgin-log"; indexable.HitType = "IMLog"; indexable.CacheContent = false; ImBuddy buddy = queryable.ImBuddyListReader.Search (info.Directory.Name); if (buddy != null && !String.IsNullOrEmpty (buddy.Alias)) indexable.AddProperty (Property.NewKeyword ("fixme:alias", buddy.Alias)); return indexable; }
private Indexable GetNewXmpIndexable (ref Indexable indexable, string path, Guid id, DirectoryModel parent) { // In non-crawl mode, check if a corresponding xmp file is present and not already scheduled and index it. // If file.xmp and file are rapidly written/updated (in that order), this does the right thing. // If file and file.xmp are rapidly written/updated (in that order), either // - file.xmp is present during FileToIndexable(file): in which case xmp properties are // added to file; and when file.xmp is indexed, it will replace the xmp properties // - file.xmp is not present during FileToIndexable(file): when the xmp file is later indexed // it will add the xmp properties // since the uid file will still be in the uid-cache, correct uid will be used for xmp prop-change indexable string possible_xmp_file_path = string.Concat (path, ".xmp"); if (! File.Exists (possible_xmp_file_path)) return null; Guid xmp_id = queryable.RegisterFile (parent, (Path.GetFileName (possible_xmp_file_path))); if (xmp_id == Guid.Empty) return null; XmpFile xmp_file = null; try { xmp_file = new XmpFile (possible_xmp_file_path); } catch { uid_manager.ForgetNewId (possible_xmp_file_path); return null; } // FIXME: Should also delete previous xmp properties! foreach (Property p in xmp_file.Properties) { p.IsMutable = true; indexable.AddProperty (p); } xmp_file.Close (); // Also need to save some local states for PostAddHook, // namely, path to the xmp file, path to basefile and generated uid indexable.LocalState ["XmpFilePath"] = possible_xmp_file_path; indexable.LocalState ["BaseFilePath"] = path; indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString (xmp_id); if (Debug) Log.Debug ("Adding properties from {0}({2}) to {1}({3})", possible_xmp_file_path, path, GuidFu.ToShortString (xmp_id), GuidFu.ToShortString (id)); return null; }
private Indexable ToAddRssIndexable (XmlDocument document) { string encoding_str = null; StringReader reader = null; if (ToBool (GetText (document, "HasOffline"))) { try { // RSS does not use OfflineSize but MessageSize instead (for some reason...) int offset = Convert.ToInt32 (GetText (document, "MessageOffset")), size = Convert.ToInt32 (GetText (document, "MessageSize")); reader = GetRssBody (GetText (document, "FolderFile"), offset, size, out encoding_str); } catch (Exception e) { Logger.Log.Debug (e, "Failed to parse RSS body"); } } Indexable indexable = new Indexable (GenerateUniqueUri (document)); indexable.HitType = "FeedItem"; indexable.MimeType = "text/html"; indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc (Convert.ToInt64 (GetText (document, "Date"))); indexable.CacheContent = true; indexable.FlushBufferCache = true; indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird")); indexable.AddProperty (Property.NewKeyword ("fixme:folder", GetText (document, "Folder"))); indexable.AddProperty (Property.NewUnsearched ("ParentUri", GetText (document, "FolderFile"))); indexable.AddProperty (Property.NewUnsearched ("fixme:uri", GetText (document, "Uri"))); indexable.AddProperty (Property.NewKeyword ("dc:identifier", ExtractUrl (GetText (document, "MessageId")))); indexable.AddProperty (Property.NewKeyword ("dc:source", GetText (document, "FeedURL"))); indexable.AddProperty (Property.New ("dc:publisher", Mime.HeaderDecodePhrase (GetText (document, "Author")))); // The title will be added by the filter. In case we add it twice we will just get // an empty tile in the search tool (a bug maybe?). if (reader != null) { // If we got an encoding, make sure we use that if (!String.IsNullOrEmpty (encoding_str)) { indexable.AddProperty (Property.New ( String.Format ("{0}encoding", StringFu.UnindexedNamespace), encoding_str)); } indexable.SetTextReader (reader); } else indexable.AddProperty (Property.New ("dc:title", Mime.HeaderDecodePhrase (GetText (document, "Subject")))); return indexable; }
// FIXME: This need some more info private Indexable NntpMessageToIndexable (TB.NntpMessage message) { Indexable indexable; indexable = new Indexable (message.Uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTime.Parse (message.GetString ("date")).ToUniversalTime (); indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird")); indexable.AddProperty (Property.NewUnsearched ("fixme:fullyIndexed", message.GetBool ("FullIndex"))); indexable.AddProperty (Property.NewDate ("fixme:indexDateTime", DateTime.UtcNow)); string subject = GMime.Utils.HeaderDecodePhrase (message.GetString ("subject")); indexable.AddProperty (Property.New ("dc:title", subject)); return indexable; }
private Indexable MemoToIndexable (CalComponent cc) { Indexable indexable = new Indexable (GetComponentUri (cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Note"; indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty (Property.NewUnsearched ("fixme:application","evolution")); indexable.AddProperty (Property.New ("dc:title", cc.Summary)); // We remember the note's text so that we can stuff it in // the TextCache later. // This is here form compability with Tomboy notes. foreach (string description in cc.Descriptions) { queryable.IndexableTextCache [indexable.Uri] = description; StringReader reader = new StringReader (description); indexable.SetTextReader (reader); } return indexable; }
private Indexable EventToIndexable (CalComponent cc) { Indexable indexable = new Indexable (GetComponentUri (cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Calendar"; indexable.AddProperty (Property.NewUnsearched ("fixme:source_uid", this.source.Uid)); indexable.AddProperty (Property.NewUnsearched ("fixme:uid", cc.Uid)); indexable.AddProperty (Property.NewDate ("fixme:starttime", cc.Dtstart.ToUniversalTime ())); if (cc.Dtend != DateTime.MinValue) indexable.AddProperty (Property.NewDate ("fixme:endtime", cc.Dtend.ToUniversalTime ())); foreach (CalComponentAttendee attendee in cc.Attendees) indexable.AddProperty (Property.New ("fixme:attendee", attendee.value)); foreach (string comment in cc.Comments) indexable.AddProperty (Property.New ("fixme:comment", comment)); foreach (string description in cc.Descriptions) indexable.AddProperty (Property.New ("fixme:description", description)); indexable.AddProperty (Property.New ("fixme:summary", cc.Summary)); foreach (string category in cc.Categories) indexable.AddProperty (Property.NewUnsearched ("fixme:category", category)); indexable.AddProperty (Property.New ("fixme:location", cc.Location)); return indexable; }
private Indexable ContactToIndexable (Evolution.Contact contact) { DateTime rev = RevStringToDateTime (contact.Rev); Indexable indexable = new Indexable (GetContactUri (contact)); indexable.Timestamp = rev; indexable.HitType = "Contact"; indexable.AddProperty (Property.NewKeyword ("fixme:client", "evolution")); indexable.AddProperty (Property.NewUnsearched ("fixme:source_uid", this.source.Uid)); indexable.AddProperty (Property.NewUnsearched ("fixme:uid", contact.Id)); indexable.AddProperty (Property.New ("fixme:FileAs", contact.FileAs)); indexable.AddProperty (Property.New ("fixme:FullName", contact.FullName)); indexable.AddProperty (Property.New ("fixme:GivenName", contact.GivenName)); indexable.AddProperty (Property.New ("fixme:FamilyName", contact.FamilyName)); indexable.AddProperty (Property.New ("fixme:Nickname", contact.Nickname)); indexable.AddProperty (Property.New ("fixme:AddressLabelHome", contact.AddressLabelHome)); indexable.AddProperty (Property.New ("fixme:AddressLabelWork", contact.AddressLabelWork)); indexable.AddProperty (Property.New ("fixme:AddressLabelOther", contact.AddressLabelOther)); indexable.AddProperty (Property.New ("fixme:AssistantPhone", contact.AssistantPhone)); indexable.AddProperty (Property.New ("fixme:BusinessPhone", contact.BusinessPhone)); indexable.AddProperty (Property.New ("fixme:BusinessPhone2", contact.BusinessPhone2)); indexable.AddProperty (Property.New ("fixme:BusinessFax", contact.BusinessFax)); indexable.AddProperty (Property.New ("fixme:CallbackPhone", contact.CallbackPhone)); indexable.AddProperty (Property.New ("fixme:CarPhone", contact.CarPhone)); indexable.AddProperty (Property.New ("fixme:CompanyPhone", contact.CompanyPhone)); indexable.AddProperty (Property.New ("fixme:HomePhone", contact.HomePhone)); indexable.AddProperty (Property.New ("fixme:HomePhone2", contact.HomePhone2)); indexable.AddProperty (Property.New ("fixme:HomeFax", contact.HomeFax)); indexable.AddProperty (Property.New ("fixme:IsdnPhone", contact.IsdnPhone)); indexable.AddProperty (Property.New ("fixme:MobilePhone", contact.MobilePhone)); indexable.AddProperty (Property.New ("fixme:OtherPhone", contact.OtherPhone)); indexable.AddProperty (Property.New ("fixme:OtherFax", contact.OtherFax)); indexable.AddProperty (Property.New ("fixme:Pager", contact.Pager)); indexable.AddProperty (Property.New ("fixme:PrimaryPhone", contact.PrimaryPhone)); indexable.AddProperty (Property.New ("fixme:Radio", contact.Radio)); indexable.AddProperty (Property.New ("fixme:Telex", contact.Telex)); indexable.AddProperty (Property.NewUnsearched ("fixme:Tty", contact.Tty)); indexable.AddProperty (Property.NewKeyword ("fixme:Email1", contact.Email1)); indexable.AddProperty (Property.NewKeyword ("fixme:Email2", contact.Email2)); indexable.AddProperty (Property.NewKeyword ("fixme:Email3", contact.Email3)); indexable.AddProperty (Property.NewUnsearched ("fixme:Mailer", contact.Mailer)); indexable.AddProperty (Property.New ("fixme:Org", contact.Org)); indexable.AddProperty (Property.New ("fixme:OrgUnit", contact.OrgUnit)); indexable.AddProperty (Property.New ("fixme:Office", contact.Office)); indexable.AddProperty (Property.New ("fixme:Title", contact.Title)); indexable.AddProperty (Property.New ("fixme:Role", contact.Role)); indexable.AddProperty (Property.New ("fixme:Manager", contact.Manager)); indexable.AddProperty (Property.New ("fixme:Assistant", contact.Assistant)); indexable.AddProperty (Property.NewKeyword ("fixme:HomepageUrl", contact.HomepageUrl)); indexable.AddProperty (Property.NewKeyword ("fixme:BlogUrl", contact.BlogUrl)); indexable.AddProperty (Property.NewUnsearched ("fixme:Categories", contact.Categories)); indexable.AddProperty (Property.NewUnsearched ("fixme:Caluri", contact.Caluri)); indexable.AddProperty (Property.NewUnsearched ("fixme:Icscalendar", contact.Icscalendar)); indexable.AddProperty (Property.New ("fixme:Spouse", contact.Spouse)); indexable.AddProperty (Property.New ("fixme:Note", contact.Note)); Evolution.ContactPhoto photo = contact.Photo; if (photo.Data != null && photo.Data.Length > 0) { string photo_filename = GetPhotoFilename (contact.Id); Stream s = new FileStream (photo_filename, FileMode.Create, FileAccess.Write, FileShare.ReadWrite); BinaryWriter w = new BinaryWriter (s); w.Write (photo.Data); w.Close (); s.Close (); indexable.AddProperty (Property.NewUnsearched ("beagle:Photo", photo_filename)); } foreach (string im in contact.ImAim) indexable.AddProperty (Property.NewUnsearched ("fixme:ImAim", im)); foreach (string im in contact.ImIcq) indexable.AddProperty (Property.NewUnsearched ("fixme:ImIcq", im)); foreach (string im in contact.ImJabber) indexable.AddProperty (Property.NewUnsearched ("fixme:ImJabber", im)); foreach (string im in contact.ImMsn) indexable.AddProperty (Property.NewUnsearched ("fixme:ImMsn", im)); foreach (string im in contact.ImYahoo) indexable.AddProperty (Property.NewUnsearched ("fixme:ImYahoo", im)); foreach (string im in contact.ImGroupwise) indexable.AddProperty (Property.NewUnsearched ("fixme:ImGroupwise", im)); String name = ""; if (contact.GivenName != null && contact.GivenName != "") name = contact.GivenName; if (contact.FamilyName != null && contact.FamilyName != "") name += " " + contact.FamilyName; if (name.Length > 0) indexable.AddProperty (Property.New ("fixme:Name", name)); if (contact.Email1 != null) indexable.AddProperty (Property.NewKeyword ("fixme:Email", contact.Email1)); return indexable; }
/* Returns false if content can't/needn't be indexed. * If AlreadyFiltered, then we don't return a filter but return true. */ static public bool FilterIndexable (Indexable indexable, TextCache text_cache, out Filter filter) { filter = null; ICollection filters = null; if (indexable.Filtering == IndexableFiltering.AlreadyFiltered) return true; if (! ShouldWeFilterThis (indexable)) return false; string path = null; // First, figure out which filter we should use to deal with // the indexable. // If a specific mime type is specified, try to index as that type. if (indexable.MimeType != null) filters = CreateFiltersFromMimeType (indexable.MimeType); if (indexable.ContentUri.IsFile) { path = indexable.ContentUri.LocalPath; // Otherwise, set the mime type for a directory, // or sniff it from the file. if (indexable.MimeType == null) { if (Directory.Exists (path)) { indexable.MimeType = "inode/directory"; indexable.NoContent = true; } else if (File.Exists (path)) { indexable.MimeType = XdgMime.GetMimeType (path); } else { Log.Warn ("Unable to filter {0}. {1} not found.", indexable.DisplayUri, path); return false; } } // Set the timestamp to the last write time, if it isn't // set by the backend. if (! indexable.ValidTimestamp && indexable.IsNonTransient) indexable.Timestamp = FileSystem.GetLastWriteTimeUtc (path); // Check the timestamp to make sure the file hasn't // disappeared from underneath us. if (! FileSystem.ExistsByDateTime (indexable.Timestamp)) { Log.Warn ("Unable to filter {0}. {1} appears to have disappeared from underneath us", indexable.DisplayUri, path); return false; } if (filters == null || filters.Count == 0) { filters = CreateFiltersFromIndexable (indexable); } } // We don't know how to filter this, so there is nothing else to do. if (filters.Count == 0) { if (! indexable.NoContent) Logger.Log.Debug ("No filter for {0} ({1}) [{2}]", indexable.DisplayUri, path, indexable.MimeType); return false; } foreach (Filter candidate_filter in filters) { if (Debug) Logger.Log.Debug ("Testing filter: {0}", candidate_filter); // Hook up the snippet writer. if (candidate_filter.SnippetMode && text_cache != null) { if (candidate_filter.OriginalIsText && indexable.IsNonTransient) { text_cache.MarkAsSelfCached (indexable.Uri); } else if (indexable.CacheContent) { TextWriter writer = text_cache.GetWriter (indexable.Uri); candidate_filter.AttachSnippetWriter (writer); } } // Set the indexable on the filter. candidate_filter.Indexable = indexable; // Open the filter, copy the file's properties to the indexable, // and hook up the TextReaders. bool successful_open = false; TextReader text_reader; Stream binary_stream; if (path != null) successful_open = candidate_filter.Open (path); else if ((text_reader = indexable.GetTextReader ()) != null) successful_open = candidate_filter.Open (text_reader); else if ((binary_stream = indexable.GetBinaryStream ()) != null) successful_open = candidate_filter.Open (binary_stream); if (successful_open) { // Set FileType indexable.AddProperty (Property.NewKeyword ("beagrep:FileType", candidate_filter.FileType)); indexable.SetTextReader (candidate_filter.GetTextReader ()); indexable.SetHotTextReader (candidate_filter.GetHotTextReader ()); if (Debug) Logger.Log.Debug ("Successfully filtered {0} with {1}", path, candidate_filter); filter = candidate_filter; return true; } else { Log.Warn ("Error in filtering {0} with {1}, falling back", path, candidate_filter); candidate_filter.Cleanup (); } } if (Debug) Logger.Log.Debug ("None of the matching filters could process the file: {0}", path); return false; }
private Indexable current_itemToIndexable () { Indexable indexable; try { indexable = new Indexable (new Uri (String.Format ("{0};item={1}", feed_source, current_item.Source))); } catch (System.UriFormatException) { indexable = new Indexable (new Uri (String.Format ("liferea://dummy?{0};item={1}", feed_source, current_item.Source))); } indexable.ParentUri = UriFu.PathToFileUri (feed_file); indexable.MimeType = "text/html"; indexable.HitType = "FeedItem"; DateTime date = DateTimeUtil.UnixToDateTimeUtc (0); date = date.AddSeconds (current_item.Timestamp); indexable.Timestamp = date; // cleaning up the property names as far as possible // this way querying for specific field is possible // following DC element names wherever applicable indexable.AddProperty (Property.New ("dc:title", current_item.Title)); Attribute[] attribs = current_item.Attribs.AttribArray; if (attribs != null) { foreach (Attribute attrib in attribs) { if (attrib.Name != "author") continue; indexable.AddProperty (Property.New ("dc:creator", attrib.Value)); } } indexable.AddProperty (Property.NewKeyword ("dc:identifier", current_item.Source)); indexable.AddProperty (Property.NewKeyword ("dc:source", feed_source)); indexable.AddProperty (Property.New ("dc:publisher", publisher)); if (File.Exists (icon_file)) indexable.AddProperty (Property.NewUnsearched ("fixme:cachedimg", icon_file)); StringReader reader = new StringReader (current_item.Description); indexable.SetTextReader (reader); return indexable; }
///////////////////////////////////////////////// private Indexable NoteToIndexable (FileInfo file, Note note) { Indexable indexable = new Indexable (note.Uri); indexable.ContentUri = UriFu.PathToFileUri (file.FullName); indexable.Timestamp = note.timestamp; indexable.HitType = "Note"; indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty (Property.New ("dc:title", note.subject)); indexable.AddProperty (Property.NewUnsearched ("fixme:application","tomboy")); // FIXME: tagging is disabled in Tomboy-0.8.x and is planned for 0.10.0 foreach( string s in note.tags) indexable.AddProperty (Property.New ("note:tag", s)); // We remember the note's text so that we can stuff it in // the TextCache later. note_text_cache [note.Uri] = note.text; StringReader reader = new StringReader (note.text); indexable.SetTextReader (reader); return indexable; }
private void IndexLauncher (FileInfo file, Scheduler.Priority priority) { if ((! file.Exists) || (this.FileAttributesStore.IsUpToDate (file.FullName))) return; /* Check to see if file is a launcher */ if (Beagle.Util.VFS.Mime.GetMimeType (file.FullName) != "application/x-desktop") return; StreamReader reader; try { reader = new StreamReader (file.Open (FileMode.Open, FileAccess.Read, FileShare.Read)); } catch (Exception e) { log.Warn ("Could not open '{0}': {1}", file.FullName, e.Message); return; } if (reader.ReadLine () != "[Desktop Entry]") { reader.Close (); return; } /* I'm convinced it is a launcher */ Indexable indexable = new Indexable (UriFu.PathToFileUri (file.FullName)); indexable.Timestamp = file.LastWriteTime; indexable.Type = "Launcher"; indexable.MimeType = "application/x-desktop"; // desktop files must have a name bool have_name = false; String line; while ((line = reader.ReadLine ()) != null) { string [] sline = line.Split ('='); if (sline.Length != 2) continue; // FIXME: We shouldnt really search fields that are in other locales than the current should we? if (sline [0].Equals ("Icon") || sline [0].Equals ("Exec")) { indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:" + sline[0], sline[1])); } else if (sline [0].StartsWith ("Name")) { if (sline [0] == "Name") have_name = true; indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:" + sline[0], sline[1])); } else if (sline[0].StartsWith ("Comment")) { indexable.AddProperty (Beagle.Property.New ("fixme:" + sline[0], sline[1])); } } reader.Close (); if (have_name) { Scheduler.Task task = NewAddTask (indexable); task.Priority = priority; ThisScheduler.Add (task); } }
Indexable BuildAddIndexable(Post post) { Indexable indexable = new Indexable (IndexableType.Add, new Uri(post.Href)); indexable.Timestamp = post.Time; indexable.Filtering = IndexableFiltering.Always; indexable.HitType = "Bookmark"; indexable.AddProperty (Property.New ("dc:title", post.Description)); indexable.AddProperty (Property.New ("dc:source", post.Href)); indexable.AddProperty (Property.New ("dc:keywords", post.Tag)); indexable.AddProperty (Property.New ("fixme:keywords", post.Tag)); indexable.AddProperty (Property.New ("fixme:host", new Uri (post.Href).Host)); // May want to support multiple accounts eventually indexable.AddProperty (Property.New ("delicious:account", requester.Account.Username)); AddContentToIndexable (indexable, post); return indexable; }
///////////////////////////////////////////////// public Indexable GetIndexable (NautilusTools.NautilusMetadata nm) { Indexable indexable = new Indexable (nm.Uri); indexable.Type = IndexableType.PropertyChange; Property prop; // Reset the notes property. if (nm.Notes == null) nm.Notes = String.Empty; prop = Property.New ("nautilus:notes", nm.Notes); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty (prop); foreach (string emblem in nm.Emblems) { prop = Property.NewKeyword ("nautilus:emblem", emblem); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty (prop); } // We add an empty keyword so that the property is reset if (nm.Emblems.Count == 0) { prop = Property.NewKeyword ("nautilus:emblem", String.Empty); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty (prop); } return indexable; }
protected override Indexable MessageToIndexable (Message message) { Uri uri = new Uri (String.Format ("feed:///{0};id={1}", message.Path, message.Id)); Indexable indexable = new Indexable (uri); indexable.MimeType = "text/html"; indexable.Type = "FeedItem"; indexable.AddProperty (Property.New ("fixme:client", "mozilla")); indexable.AddProperty(Property.New ("dc:title", message.Subject)); indexable.AddProperty(Property.New ("fixme:author", message.From)); //indexable.AddProperty(Property.NewDate ("fixme:published", item.PubDate)); indexable.AddProperty(Property.NewUnsearched ("fixme:itemuri", message.Headers ["Content-Base"])); indexable.AddProperty (Property.New ("fixme:offset", message.Offset)); StringReader reader = new StringReader (message.Body); indexable.SetTextReader (reader); return indexable; }
public Indexable GetNextIndexable () { string line; string_builder.Length = 0; DateTime dt = DateTime.MinValue; string uid = null; // Keep reading till "END:VJOURNAL" while ((line = reader.ReadLine ()) != null) { //UID:libkcal-1467827482.768 //LAST-MODIFIED:20061015T085606Z if (line == "END:VJOURNAL") break; else if (line.StartsWith ("UID:")) uid = line.Substring (4); else if (line.StartsWith ("LAST-MODIFIED:")) { string dt_string = line.Substring (14); dt_string = dt_string.Replace ("Z", "+00:00"); dt = DateTime.ParseExact ( dt_string, fmts, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal); } else { string_builder.Append (line); string_builder.Append ('\n'); } } if (line == null) { reader.Close (); return null; } // Bad note if (string_builder.Length == 0 || uid == null || dt == DateTime.MinValue) return null; // Mark note with uid as seen ('undeleted') deleted_notes [uid] = false; lock (last_modified_table) { if (last_modified_table.Contains (uid)) { DateTime old_dt = (DateTime) last_modified_table [uid]; // FIXME: Returning null for more than 179 times will cause trouble if (dt == old_dt) return null; else { //Log.Debug ("Updating last_mod_date [{0}] = {1}", uid, dt); last_modified_table [uid] = dt; } } else { //Log.Debug ("Adding last_mod_date [{0}] = {1}", uid, dt); last_modified_table [uid] = dt; } } if (initial_scan) return null; // Open knotes notes as //dcop knotes KNotesIface text <UID> // where the uri is given as knotes://uid Uri uri = new Uri (String.Format ("knotes:///{0}", uid)); Indexable indexable = new Indexable (uri); indexable.ParentUri = UriFu.PathToFileUri (knotes_file); indexable.MimeType = ICalParser.KnotesMimeType; indexable.HitType = "Note"; indexable.Timestamp = dt; // Add uid as a keyword field for convenience indexable.AddProperty (Property.NewUnsearched ("fixme:uid", uid)); // FIXME: Comment this Debug statement after the backend stabilizes //Log.Debug ("Creating {0} from:[{1}]", uri, string_builder.ToString ()); StringReader string_reader = new StringReader (string_builder.ToString()); indexable.SetTextReader (string_reader); return indexable; }
private Indexable ToAddMailMessageIndexable (XmlDocument document) { GMime.Message message = null; // Check if the entire message is available if (ToBool (GetText (document, "HasOffline"))) { // We must make sure we don't get an exception here since we can fallback to // other information try { int offset = Convert.ToInt32 (GetText (document, "MessageOffset")), size = Convert.ToInt32 (GetText (document, "OfflineSize")); message = GetGMimeMessage (GetText (document, "FolderFile"), offset, size); } catch (Exception e) { Logger.Log.Debug (e, "Failed to parse GMime message"); } } if (message == null) message = GetStubMessage (document); Indexable indexable = new Indexable (GenerateUniqueUri (document)); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc (Convert.ToInt64 (GetText (document, "Date"))); indexable.CacheContent = true; indexable.FlushBufferCache = true; indexable.SetBinaryStream (message.Stream); indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird")); indexable.AddProperty (Property.NewKeyword ("fixme:folder", GetText (document, "Folder"))); indexable.AddProperty (Property.NewUnsearched ("ParentUri", GetText (document, "FolderFile"))); indexable.AddProperty (Property.NewUnsearched ("fixme:uri", GetText (document, "Uri"))); message.Dispose (); return indexable; }
private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags) { // Don't index messages flagged as junk if (CheckFlags (flags, B_U_Camel.CamelFlags.Junk)) return null; System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid); Indexable indexable = new Indexable (uri); indexable.Timestamp = message.Date.ToUniversalTime (); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", "Local")); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name)); GMime.InternetAddressList addrs; if (this.folder_name == "Sent") { addrs = message.GetRecipients (GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name == "Sent") { addrs = message.GetRecipients (GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name != "Sent") { addrs = GMime.InternetAddressList.Parse (message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name == "Sent") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); Property flag_prop = Property.NewUnsearched ("fixme:flags", flags); flag_prop.IsMutable = true; indexable.AddProperty (flag_prop); if (CheckFlags (flags, B_U_Camel.CamelFlags.Answered)) indexable.AddProperty (Property.NewFlag ("fixme:isAnswered")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Deleted)) indexable.AddProperty (Property.NewFlag ("fixme:isDeleted")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Draft)) indexable.AddProperty (Property.NewFlag ("fixme:isDraft")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Flagged)) indexable.AddProperty (Property.NewFlag ("fixme:isFlagged")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Seen)) indexable.AddProperty (Property.NewFlag ("fixme:isSeen")); if (CheckFlags (flags, B_U_Camel.CamelFlags.AnsweredAll)) indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll")); indexable.SetBinaryStream (message.Stream); return indexable; }
public override bool GenerateNextIndexable (out Indexable child) { child = null; if (bib_process == null && ! InitBibparse ()) return false; string line = null; string type = null, name = null; while ((line = reader.ReadLine ()) != null) { if (line == String.Empty || line [0] != '@') continue; int i = line.IndexOf (' '); if (i == -1 || line.Length == i + 1) continue; type = line.Substring (1, i - 1).ToLower (); name = line.Substring (i + 1); break; } if (line == null) return false; child = new Indexable (UriFu.AddFragment (Indexable.Uri, name, false)); child.CacheContent = false; child.MimeType = "text/x-bibtex"; child.DisplayUri = child.Uri; child.NoContent = true; child.AddProperty (Property.NewKeyword ("bibtex:type", type)); string key, value; // Now fill in properties from the key=value lines while ((line = reader.ReadLine ()) != null) { // Entries are separated by empty lines if (line == String.Empty) break; int i = line.IndexOf ('='); // ensure non-empty key if (i < 1 || line.Length == i + i) continue; key = line.Substring (0, i).ToLower (); value = line.Substring (i + 1); foreach (Property prop in EntryLineToProperty (key, value)) child.AddProperty (prop); } child.SetChildOf (Indexable); return true; }
public override Indexable GetNextIndexable () { Indexable indexable = null; // No more new messages to index, so start on the removals. if (this.delete_mode) { string uid = (string) this.deleted_list [0]; Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid); indexable = new Indexable (IndexableType.Remove, uri); this.deleted_list.RemoveAt (0); this.tracker.Remove (uid); this.delete_count++; return indexable; } B_U_Camel.MessageInfo mi = (B_U_Camel.MessageInfo) this.summary_enumerator.Current; ++this.count; if (Debug) { Logger.Log.Debug ("Constructed message {0} with uid {1}, flags {2}.", this.count, mi.uid, mi.flags); } uint flags; bool found = this.tracker.Get (mi.uid, out flags); if (! found) { // New, previously unseen message string msg_file; if (this.backend_type == ImapBackendType.Imap) msg_file = Path.Combine (summary_info.DirectoryName, mi.uid + "."); else { // This is taken from e-d-s's camel-data-cache.c. No doubt // NotZed would scream bloody murder if he saw this here. int hash = (g_str_hash (mi.uid) >> 5) & CAMEL_DATA_CACHE_MASK; string cache_path = String.Format ("cache/{0:x}/{1}", hash, mi.uid); msg_file = Path.Combine (summary_info.DirectoryName, cache_path); } indexable = this.CamelMessageToIndexable (mi, msg_file); if (Debug) Logger.Log.Debug ("Unseen message, indexable {0} null", indexable == null ? "" : "not"); if (indexable != null) ++this.indexed_count; } else if (found && flags != mi.flags) { // Previously seen message, but flags have changed. Uri uri = CamelMessageUri (mi); indexable = new Indexable (uri); indexable.Type = IndexableType.PropertyChange; Property flag_prop = Property.NewUnsearched ("fixme:flags", mi.flags); flag_prop.IsMutable = true; indexable.AddProperty (flag_prop); if (Debug) Logger.Log.Debug ("Previously seen message, flags changed: {0} -> {1}", flags, mi.flags); ++this.indexed_count; } else { if (Debug) Logger.Log.Debug ("Previously seen message, unchanged."); } this.tracker.Update (mi.uid, mi.flags); return indexable; }
public Indexable GetXmpQueryable (string path, Guid id, DirectoryModel parent) { Log.Debug ("Asked to create xmp indexable for ({0}) {1}", GuidFu.ToShortString (id), path); // Should be at least 6 characters /<...>.xmp if (path.Length < 6) return null; string basefile_path = Path.ChangeExtension (path, null); // Ignore xmp files by itself // FIXME: To support indexing independent xmp files will require even greater trouble if (! File.Exists (basefile_path)) return null; XmpFile xmp_file = null; try { xmp_file = new XmpFile (path); } catch { Log.Warn ("Cannot create xmpfile from {0}", path); return null; } // Try to get the correct uid for the basefile // First we need to see if basefile is already scheduled (yet to be dispatched) Uri basefile_uri = null; Indexable base_indexable; if (uid_manager.HasNewId (basefile_path)) { // Since uid_manager has a new id for this basefile, so basefile is already scheduled // Get basefile uid from there Guid basefile_id = uid_manager.GetNewId (basefile_path); basefile_uri = GuidFu.ToUri (basefile_id); Log.Debug ("{0} is already scheduled with uri {1}", basefile_path, basefile_uri); } else { // Basefile is not scheduled in the current batch string basefile_name = Path.GetFileName (basefile_path); // Try to schedule it for addition base_indexable = queryable.GetCrawlingFileIndexable (parent, basefile_name); if (base_indexable == null) { // GetCrawlingFileIndexable returns null if file does not need to be indexed // So basefile is up-to-date // Need to figure out id from uid manager Guid basefile_id = uid_manager.GetIdByNameAndParentId (basefile_name, parent.UniqueId); basefile_uri = GuidFu.ToUri (basefile_id); Log.Debug ("{0} is not scheduled and need not be, uri is {1}", basefile_path, basefile_uri); } else { Log.Debug ("Need to index {0}", basefile_path); // basefile needs to be indexed // FIXME: Move the task business out of handler and into FSQ.cs Scheduler.Task task; task = queryable.NewAddTask (base_indexable); // FIXME: What is the correct priority ? // If should have similar priority to the one that this xmp-indexable will be a part of task.Priority = Scheduler.Priority.Immediate; queryable.ThisScheduler.Add (task); // Get the basefile uri from the indexable basefile_uri = base_indexable.Uri; } } Log.Debug ("Adding xmp-indexable for {0} (basefile uri {1}) with uid {2}", path, basefile_uri, GuidFu.ToShortString (id)); Indexable indexable = new Indexable (IndexableType.PropertyChange, basefile_uri); // Set the timestamp of the indexable as the timestamp of the basefile // It could have also been skipped, the original Indexable.Add would anyway have it indexable.Timestamp = File.GetLastWriteTimeUtc (basefile_path); indexable.DisplayUri = UriFu.PathToFileUri (path); // If the file was somehow deleted before this point, bail out. if (! FileSystem.ExistsByDateTime (indexable.Timestamp)) { xmp_file.Close (); return null; } // Save some local states for PostAddHook, namely, path to the xmp file, path to basefile and generated uid indexable.LocalState ["XmpFilePath"] = path; indexable.LocalState ["BaseFilePath"] = basefile_path; indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString (id); // FIXME: Should also delete previous xmp properties! foreach (Property p in xmp_file.Properties) { p.IsMutable = true; indexable.AddProperty (p); } xmp_file.Close (); return indexable; }
private Indexable CamelMessageToIndexable (B_U_Camel.MessageInfo messageInfo, string msg_file) { // Don't index messages flagged as junk if (messageInfo.IsJunk) return null; // Many properties will be set by the filter when // processing the cached data, if it's there. So // don't set a number of properties in that case. bool have_content = File.Exists (msg_file); Uri uri = CamelMessageUri (messageInfo); Indexable indexable = new Indexable (uri); indexable.Timestamp = messageInfo.SentDate; indexable.MimeType = "message/rfc822"; indexable.HitType = "MailMessage"; indexable.AddProperty (Property.NewUnsearched ("fixme:account", this.imap_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution")); if (!have_content) { indexable.AddProperty (Property.New ("dc:title", GMime.Utils.HeaderDecodePhrase (messageInfo.subject))); indexable.AddProperty (Property.NewDate ("fixme:date", messageInfo.SentDate)); } GMime.InternetAddressList addrs; addrs = GMime.InternetAddressList.Parse (messageInfo.to); foreach (GMime.InternetAddress ia in addrs) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; if (!have_content) { indexable.AddProperty (Property.NewUnsearched ("fixme:to", ia.ToString (false))); if (ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.New ("fixme:to_address", mailbox.Address)); indexable.AddProperty (Property.New ("fixme:to_name", ia.Name)); } if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } addrs.Dispose (); addrs = GMime.InternetAddressList.Parse (messageInfo.cc); foreach (GMime.InternetAddress ia in addrs) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; if (!have_content) { indexable.AddProperty (Property.NewUnsearched ("fixme:cc", ia.ToString (false))); if (ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.New ("fixme:cc_address", mailbox.Address)); indexable.AddProperty (Property.New ("fixme:cc_name", ia.Name)); } if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } addrs.Dispose (); addrs = GMime.InternetAddressList.Parse (messageInfo.from); foreach (GMime.InternetAddress ia in addrs) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; if (!have_content) { indexable.AddProperty (Property.NewUnsearched ("fixme:from", ia.ToString (false))); if (ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.New ("fixme:from_address", mailbox.Address)); indexable.AddProperty (Property.New ("fixme:from_name", ia.Name)); } if (this.folder_name != "Sent" && ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address)); } addrs.Dispose (); indexable.AddProperty (Property.NewKeyword ("fixme:mlist", messageInfo.mlist)); Property flag_prop = Property.NewUnsearched ("fixme:flags", messageInfo.flags); flag_prop.IsMutable = true; indexable.AddProperty (flag_prop); if (this.folder_name == "Sent") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); if (messageInfo.IsAnswered) indexable.AddProperty (Property.NewFlag ("fixme:isAnswered")); if (messageInfo.IsDeleted) indexable.AddProperty (Property.NewFlag ("fixme:isDeleted")); if (messageInfo.IsDraft) indexable.AddProperty (Property.NewFlag ("fixme:isDraft")); if (messageInfo.IsFlagged) indexable.AddProperty (Property.NewFlag ("fixme:isFlagged")); if (messageInfo.IsSeen) indexable.AddProperty (Property.NewFlag ("fixme:isSeen")); if (messageInfo.HasAttachments && !have_content) indexable.AddProperty (Property.NewFlag ("fixme:hasAttachments")); if (messageInfo.IsAnsweredAll) indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll")); if (have_content) indexable.ContentUri = UriFu.PathToFileUri (msg_file); else indexable.NoContent = true; return indexable; }
public Indexable GetNextIndexable () { string id = reader.GetAttribute ("Id"); string title = reader.GetAttribute ("Title"); string author = reader.GetAttribute ("Author"); // FIXME stupid mono bug; DateTime.ParseExact ("0001-01-01T00:00:00.0000000+00:00", ...) // http://bugzilla.ximian.com/show_bug.cgi?id=76082 // Still present in 1.1.9.2 DateTime pub_date; try { pub_date = DateTime.ParseExact ( reader.GetAttribute ("PubDate"), "yyyy-MM-ddTHH:mm:ss.fffffffzzz", null); } catch (Exception e) { pub_date = DateTime.MinValue; } string link = reader.GetAttribute ("Link"); string text = reader.GetAttribute ("Text"); reader.Read (); Uri uri = new Uri (String.Format ("feed:{0};item={1}", channel_url, id)); Logger.Log.Debug ("BlamQ: Indexing [" + channel_name + "] " + title); Indexable indexable = new Indexable (uri); indexable.ParentUri = UriFu.PathToFileUri (feed_file); indexable.MimeType = "text/html"; indexable.HitType = "FeedItem"; indexable.Timestamp = pub_date.ToUniversalTime (); // change property names to DC names, as far as allowed indexable.AddProperty (Property.New ("dc:title", title)); indexable.AddProperty (Property.New ("dc:creator", author)); indexable.AddProperty (Property.NewKeyword ("dc:identifier", link)); indexable.AddProperty (Property.NewKeyword ("dc:source", channel_url)); indexable.AddProperty (Property.New ("dc:publisher", channel_name)); string img = null; int i = text.IndexOf ("<img src=\""); if (i != -1) { i += "<img src=\"".Length; int j = text.IndexOf ("\"", i); if (j != -1) img = text.Substring (i, j-i); } if (img != null) { string path = Path.Combine (Path.Combine (blam_dir, "Cache"), img.GetHashCode ().ToString ()); indexable.AddProperty (Property.NewUnsearched ("fixme:cachedimg", path)); } StringReader string_reader = new StringReader (text); indexable.SetTextReader (string_reader); return indexable; }
private Indexable FileToIndexable (FileInfo data_file) { FileInfo meta_file = new FileInfo (Path.Combine (data_file.DirectoryName, "." + data_file.Name)); FileStream meta_stream; try { meta_stream = meta_file.Open (FileMode.Open, FileAccess.Read, FileShare.Read); } catch (FileNotFoundException) { // The meta file disappeared before we could // open it. return null; } StreamReader reader = new StreamReader (meta_stream); // First line of the file is a URI string line = reader.ReadLine (); Uri uri; try { uri = new Uri (line); } catch (Exception e) { Logger.Log.Warn (e, "IndexingService: Unable to parse URI in {0}:", meta_file.FullName); meta_stream.Close (); return null; } Indexable indexable = new Indexable (uri); indexable.Timestamp = data_file.LastWriteTimeUtc; indexable.ContentUri = UriFu.PathToFileUri (data_file.FullName); indexable.DeleteContent = true; indexable.AddProperty( Property.New("fixme:host",uri.Host)); // Second line is the hit type line = reader.ReadLine (); if (line == null) { Logger.Log.Warn ("IndexingService: EOF reached trying to read hit type from {0}", meta_file.FullName); meta_stream.Close (); return null; } else if (line != String.Empty) indexable.HitType = line; // Third line is the mime type line = reader.ReadLine (); if (line == null) { Logger.Log.Warn ("IndexingService: EOF reached trying to read mime type from {0}", meta_file.FullName); meta_stream.Close (); return null; } else if (line != String.Empty) indexable.MimeType = line; // Following lines are properties in "t:key=value" format do { line = reader.ReadLine (); if (line != null && line != String.Empty) { bool keyword = false; if (line[0] == 'k') keyword = true; else if (line[0] != 't') { Logger.Log.Warn ("IndexingService: Unknown property type: '{0}'", line[0]); continue; } int i = line.IndexOf ('='); if (i == -1) { Logger.Log.Warn ("IndexingService: Unknown property line: '{0}'", line); continue; } // FIXME: We should probably handle date types if (keyword) { indexable.AddProperty (Property.NewUnsearched (line.Substring (2, i - 2), line.Substring (i + 1))); } else { indexable.AddProperty (Property.New (line.Substring (2, i - 2), line.Substring (i + 1))); } } } while (line != null); indexable.LocalState ["MetaFile"] = meta_file; // Ok, we're finished with the meta file. It will be // deleted in PostAddHook (). meta_stream.Close (); return indexable; }
private Indexable current_itemToIndexable () { // sanity check if (current_item == null) return null; //Log.Debug ("Indexing " + channel_link + ":" + current_item.Link); Indexable indexable = new Indexable (new Uri (String.Format ("feed:{0};item={1}", channel_link, current_item.Link))); indexable.ParentUri = UriFu.PathToFileUri (feed_file); indexable.MimeType = "text/html"; indexable.HitType = "FeedItem"; string RFC822 = "ddd, dd MMM yyyy HH:mm:ss zzz"; DateTime date = DateTime.ParseExact(current_item.PubDate, RFC822, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal); indexable.Timestamp = date; // replace property names with Dublin Core names indexable.AddProperty (Property.New ("dc:title", current_item.Title)); indexable.AddProperty (Property.NewKeyword ("dc:identifier", current_item.Link)); indexable.AddProperty (Property.NewKeyword ("dc:source", channel_link)); indexable.AddProperty (Property.New ("dc:publisher", channel_title)); StringReader reader = new StringReader (current_item.Description); indexable.SetTextReader (reader); return indexable; }
/** * Create an indexable from a maildir message */ public Indexable MaildirMessageToIndexable (string filename, bool crawl) { //Logger.Log.Debug ("+ indexing maildir mail:" + filename); String folder = GetFolderMaildir(filename); Uri file_uri = UriFu.PathToFileUri (filename); Indexable indexable = new Indexable (file_uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.FlushBufferCache = crawl; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder)); indexable.ContentUri = file_uri; return indexable; }
/** * Create an indexable from an mbox message * Most of the code here is from Evo backend */ public Indexable MessageToIndexable (string file_name, System.Uri uri, GMime.Message message, string folder_name) { //Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name); Indexable indexable = new Indexable (uri); // set parent uri to the filename so that when an mbox file // is deleted, all the messages in that file can be deleted indexable.ParentUri = UriFu.PathToFileUri (file_name); indexable.Timestamp = message.Date.ToUniversalTime (); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder_name)); GMime.InternetAddressList addrs; if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients (GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients (GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (folder_name != Queryable.SentMailFolderName) { addrs = GMime.InternetAddressList.Parse (message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose (); } if (folder_name == Queryable.SentMailFolderName) indexable.AddProperty (Property.NewFlag ("fixme:isSent")); else { string kmail_msg_sent = message.GetHeader ("X-KMail-Link-Type"); if (kmail_msg_sent == "reply") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); } // no need to store date again, use the issent flag to determine if the date is sentdate or not #if false if (folder_name == Queryable.SentMailFolderName) indexable.AddProperty (Property.NewDate ("fixme:sentdate", message.Date.ToUniversalTime ())); else indexable.AddProperty (Property.NewDate ("fixme:received", message.Date.ToUniversalTime ())); #endif indexable.SetBinaryStream (message.Stream); return indexable; }
public Indexable GetNextIndexable () { DateTime line_dt = DateTime.MinValue; while (log_line_as_sb != null) { //Log.Debug ("Checking line from {0}:[{1}]", log_file, log_line_as_sb); bool in_session = AppendLogText (log_line_as_sb, out line_dt); if (! in_session) break; prev_line_offset = reader.Position; log_line_as_sb = reader.ReadLineAsStringBuilder (); } // Check if there is new data to index if (data_sb.Length == 0) { session_begin_time = line_dt; return null; } Uri uri = new Uri (String.Format ("konversation://{0}@dumb/{1}", session_begin_offset, log_file)); Log.Debug ("Creating indexable {0}", uri); Indexable indexable = new Indexable (uri); indexable.ParentUri = UriFu.PathToFileUri (log_file); indexable.Timestamp = session_begin_time; indexable.HitType = "IMLog"; indexable.CacheContent = false; indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:session_begin_offset", session_begin_offset)); indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:session_end_offset", prev_line_offset)); indexable.AddProperty (Beagle.Property.NewDate ("fixme:starttime", session_begin_time)); indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:client", "Konversation")); indexable.AddProperty (Beagle.Property.NewUnsearched ("fixme:protocol", "IRC")); AddChannelInformation (indexable); foreach (string speaker in speakers.Keys) indexable.AddProperty (Beagle.Property.NewUnstored ("fixme:speaker", speaker)); StringReader data_reader = new StringReader (data_sb.ToString ()); indexable.SetTextReader (data_reader); // update session begin time to the date of the current line which is not in this session session_begin_time = line_dt; return indexable; }