private Indexable OperaRowToIndexable(OperaHistory.Row row) { // It's unsafe to index secure content since it may contain sensitive data if (row.Address.Scheme == Uri.UriSchemeHttps) { return(null); } Indexable indexable = new Indexable(row.Address); indexable.HitType = "WebHistory"; indexable.MimeType = row.MimeType; indexable.Timestamp = row.LastVisited; indexable.AddProperty(Beagle.Property.New("fixme:host", row.Address.Host)); indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:size", row.Length)); // hint for the filter about the charset indexable.AddProperty(Property.NewUnsearched(StringFu.UnindexedNamespace + "charset", row.Encoding.ToString())); if (row.Compression == "gzip") { indexable.SetBinaryStream(new GZipInputStream(File.OpenRead(Path.Combine(cache_dir, row.LocalFileName)))); } else { indexable.ContentUri = new Uri(Path.Combine(cache_dir, row.LocalFileName)); } indexer.AttributeStore.AttachLastWriteTime(Path.Combine(cache_dir, row.LocalFileName), DateTime.UtcNow); return(indexable); }
private Indexable ToAddMailMessageIndexable(XmlDocument document) { GMime.Message message = null; // Check if the entire message is available if (ToBool(GetText(document, "HasOffline"))) { // We must make sure we don't get an exception here since we can fallback to // other information try { int offset = Convert.ToInt32(GetText(document, "MessageOffset")), size = Convert.ToInt32(GetText(document, "OfflineSize")); message = GetGMimeMessage(GetText(document, "FolderFile"), offset, size); } catch (Exception e) { Logger.Log.Debug(e, "Failed to parse GMime message"); } } if (message == null) { message = GetStubMessage(document); } Indexable indexable = new Indexable(GenerateUniqueUri(document)); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc(Convert.ToInt64(GetText(document, "Date"))); indexable.CacheContent = true; indexable.FlushBufferCache = true; indexable.SetBinaryStream(message.Stream); indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewKeyword("fixme:folder", GetText(document, "Folder"))); indexable.AddProperty(Property.NewUnsearched("ParentUri", GetText(document, "FolderFile"))); indexable.AddProperty(Property.NewUnsearched("fixme:uri", GetText(document, "Uri"))); message.Dispose(); return(indexable); }
/** * Create an indexable from an mbox message * Most of the code here is from Evo backend */ public Indexable MessageToIndexable (string file_name, System.Uri uri, GMime.Message message, string folder_name) { //Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name); Indexable indexable = new Indexable (uri); // set parent uri to the filename so that when an mbox file // is deleted, all the messages in that file can be deleted indexable.ParentUri = UriFu.PathToFileUri (file_name); indexable.Timestamp = message.Date.ToUniversalTime (); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder_name)); GMime.InternetAddressList addrs; if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients (GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients (GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (folder_name != Queryable.SentMailFolderName) { addrs = GMime.InternetAddressList.Parse (message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose (); } if (folder_name == Queryable.SentMailFolderName) indexable.AddProperty (Property.NewFlag ("fixme:isSent")); else { string kmail_msg_sent = message.GetHeader ("X-KMail-Link-Type"); if (kmail_msg_sent == "reply") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); } // no need to store date again, use the issent flag to determine if the date is sentdate or not #if false if (folder_name == Queryable.SentMailFolderName) indexable.AddProperty (Property.NewDate ("fixme:sentdate", message.Date.ToUniversalTime ())); else indexable.AddProperty (Property.NewDate ("fixme:received", message.Date.ToUniversalTime ())); #endif indexable.SetBinaryStream (message.Stream); return indexable; }
/** * Create an indexable from an mbox message * Most of the code here is from Evo backend */ public Indexable MessageToIndexable(string file_name, System.Uri uri, GMime.Message message, string folder_name) { //Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name); Indexable indexable = new Indexable(uri); // set parent uri to the filename so that when an mbox file // is deleted, all the messages in that file can be deleted indexable.ParentUri = UriFu.PathToFileUri(file_name); indexable.Timestamp = message.Date.ToUniversalTime(); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail")); indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name)); indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder_name)); GMime.InternetAddressList addrs; if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients(GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty(Property.NewKeyword("fixme:sentTo", mailbox.Address)); } } addrs.Dispose(); } if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients(GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty(Property.NewKeyword("fixme:sentTo", mailbox.Address)); } } addrs.Dispose(); } if (folder_name != Queryable.SentMailFolderName) { addrs = GMime.InternetAddressList.Parse(message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty(Property.NewKeyword("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose(); } if (folder_name == Queryable.SentMailFolderName) { indexable.AddProperty(Property.NewFlag("fixme:isSent")); } else { string kmail_msg_sent = message.GetHeader("X-KMail-Link-Type"); if (kmail_msg_sent == "reply") { indexable.AddProperty(Property.NewFlag("fixme:isSent")); } } // no need to store date again, use the issent flag to determine if the date is sentdate or not #if false if (folder_name == Queryable.SentMailFolderName) { indexable.AddProperty(Property.NewDate("fixme:sentdate", message.Date.ToUniversalTime())); } else { indexable.AddProperty(Property.NewDate("fixme:received", message.Date.ToUniversalTime())); } #endif indexable.SetBinaryStream(message.Stream); return(indexable); }
void AddContentToIndexable(Indexable indexable, Post post) { try { HttpWebRequest request = requester.CreateWebRequest (indexable.Uri.OriginalString); HttpWebResponse response = request.GetResponse () as HttpWebResponse; if (response.StatusCode == HttpStatusCode.OK ) { indexable.MimeType = FilterContentType (response.Headers ["Content-Type"]); indexable.SetBinaryStream (response.GetResponseStream ()); } } catch (WebException e) { // Can't index the content of the bookmark at this time. Log.Warn (e, "Error retrieving content for " + indexable.Uri.ToString ()); } }
private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags) { // Don't index messages flagged as junk if (CheckFlags (flags, B_U_Camel.CamelFlags.Junk)) return null; System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid); Indexable indexable = new Indexable (uri); indexable.Timestamp = message.Date.ToUniversalTime (); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", "Local")); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name)); GMime.InternetAddressList addrs; if (this.folder_name == "Sent") { addrs = message.GetRecipients (GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name == "Sent") { addrs = message.GetRecipients (GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name != "Sent") { addrs = GMime.InternetAddressList.Parse (message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name == "Sent") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); Property flag_prop = Property.NewUnsearched ("fixme:flags", flags); flag_prop.IsMutable = true; indexable.AddProperty (flag_prop); if (CheckFlags (flags, B_U_Camel.CamelFlags.Answered)) indexable.AddProperty (Property.NewFlag ("fixme:isAnswered")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Deleted)) indexable.AddProperty (Property.NewFlag ("fixme:isDeleted")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Draft)) indexable.AddProperty (Property.NewFlag ("fixme:isDraft")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Flagged)) indexable.AddProperty (Property.NewFlag ("fixme:isFlagged")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Seen)) indexable.AddProperty (Property.NewFlag ("fixme:isSeen")); if (CheckFlags (flags, B_U_Camel.CamelFlags.AnsweredAll)) indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll")); indexable.SetBinaryStream (message.Stream); return indexable; }
private Indexable GMimeMessageToIndexable(string uid, GMime.Message message, uint flags) { // Don't index messages flagged as junk if (CheckFlags(flags, B_U_Camel.CamelFlags.Junk)) { return(null); } System.Uri uri = EvolutionMailQueryable.EmailUri(this.account_name, this.folder_name, uid); Indexable indexable = new Indexable(uri); indexable.Timestamp = message.Date.ToUniversalTime(); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty(Property.NewUnsearched("fixme:client", "evolution")); indexable.AddProperty(Property.NewUnsearched("fixme:account", "Local")); indexable.AddProperty(Property.NewUnsearched("fixme:folder", this.folder_name)); GMime.InternetAddressList addrs; if (this.folder_name == "Sent") { addrs = message.GetRecipients(GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty(Property.NewUnsearched("fixme:sentTo", mailbox.Address)); } } addrs.Dispose(); } if (this.folder_name == "Sent") { addrs = message.GetRecipients(GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty(Property.NewUnsearched("fixme:sentTo", mailbox.Address)); } } addrs.Dispose(); } if (this.folder_name != "Sent") { addrs = GMime.InternetAddressList.Parse(message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty(Property.NewUnsearched("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose(); } if (this.folder_name == "Sent") { indexable.AddProperty(Property.NewFlag("fixme:isSent")); } Property flag_prop = Property.NewUnsearched("fixme:flags", flags); flag_prop.IsMutable = true; indexable.AddProperty(flag_prop); if (CheckFlags(flags, B_U_Camel.CamelFlags.Answered)) { indexable.AddProperty(Property.NewFlag("fixme:isAnswered")); } if (CheckFlags(flags, B_U_Camel.CamelFlags.Deleted)) { indexable.AddProperty(Property.NewFlag("fixme:isDeleted")); } if (CheckFlags(flags, B_U_Camel.CamelFlags.Draft)) { indexable.AddProperty(Property.NewFlag("fixme:isDraft")); } if (CheckFlags(flags, B_U_Camel.CamelFlags.Flagged)) { indexable.AddProperty(Property.NewFlag("fixme:isFlagged")); } if (CheckFlags(flags, B_U_Camel.CamelFlags.Seen)) { indexable.AddProperty(Property.NewFlag("fixme:isSeen")); } if (CheckFlags(flags, B_U_Camel.CamelFlags.AnsweredAll)) { indexable.AddProperty(Property.NewFlag("fixme:isAnsweredAll")); } indexable.SetBinaryStream(message.Stream); return(indexable); }
private Indexable ToAddMailMessageIndexable (XmlDocument document) { GMime.Message message = null; // Check if the entire message is available if (ToBool (GetText (document, "HasOffline"))) { // We must make sure we don't get an exception here since we can fallback to // other information try { int offset = Convert.ToInt32 (GetText (document, "MessageOffset")), size = Convert.ToInt32 (GetText (document, "OfflineSize")); message = GetGMimeMessage (GetText (document, "FolderFile"), offset, size); } catch (Exception e) { Logger.Log.Debug (e, "Failed to parse GMime message"); } } if (message == null) message = GetStubMessage (document); Indexable indexable = new Indexable (GenerateUniqueUri (document)); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc (Convert.ToInt64 (GetText (document, "Date"))); indexable.CacheContent = true; indexable.FlushBufferCache = true; indexable.SetBinaryStream (message.Stream); indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird")); indexable.AddProperty (Property.NewKeyword ("fixme:folder", GetText (document, "Folder"))); indexable.AddProperty (Property.NewUnsearched ("ParentUri", GetText (document, "FolderFile"))); indexable.AddProperty (Property.NewUnsearched ("fixme:uri", GetText (document, "Uri"))); message.Dispose (); return indexable; }
public void OnEachPart(GMime.Object mime_part) { GMime.Object part = null; bool part_needs_dispose = false; //for (int i = 0; i < this.depth; i++) // Console.Write (" "); //Console.WriteLine ("Content-Type: {0}", mime_part.ContentType); ++depth; if (mime_part is GMime.MessagePart) { GMime.MessagePart msg_part = (GMime.MessagePart)mime_part; using (GMime.Message message = msg_part.Message) { using (GMime.Object subpart = message.MimePart) this.OnEachPart(subpart); } } else if (mime_part is GMime.Multipart) { GMime.Multipart multipart = (GMime.Multipart)mime_part; int num_parts = multipart.Count; // If the mimetype is multipart/alternative, we only want to index // one part -- the richest one we can filter. if (mime_part.ContentType.MediaSubtype.ToLower() == "alternative") { // The richest formats are at the end, so work from there // backward. for (int i = num_parts - 1; i >= 0; i--) { GMime.Object subpart = multipart[i]; if (IsMimeTypeHandled(subpart.ContentType.ToString())) { part = subpart; part_needs_dispose = true; break; } else { subpart.Dispose(); } } } // If it's not alternative, or we don't know how to filter any of // the parts, treat them like a bunch of attachments. if (part == null) { for (int i = 0; i < num_parts; i++) { using (GMime.Object subpart = multipart[i]) this.OnEachPart(subpart); } } } else if (mime_part is GMime.Part) { part = mime_part; } else { throw new Exception(String.Format("Unknown part type: {0}", part.GetType())); } if (part != null) { System.IO.Stream stream = null; using (GMime.DataWrapper content_obj = ((GMime.Part)part).ContentObject) stream = content_obj.Stream; // If this is the only part and it's plain text, we // want to just attach it to our filter instead of // creating a child indexable for it. bool no_child_needed = false; string mime_type = part.ContentType.ToString().ToLower(); if (this.depth == 1 && this.count == 0) { if (mime_type == "text/plain") { no_child_needed = true; this.reader = new StreamReader(stream); } else if (mime_type == "text/html") { no_child_needed = true; html_part = true; string enc = part.ContentType.GetParameter("charset"); // DataWrapper.Stream is a very limited stream // and does not allow Seek or Tell // HtmlFilter requires Stream.Position=0. // Play safe and create a memorystream // for HTML parsing. GMime.StreamMem mem_stream; mem_stream = new GMime.StreamMem(); GMime.Stream data_stream; data_stream = ((StreamWrapper)stream).GMimeStream; data_stream.WriteToStream(mem_stream); data_stream.Flush(); // The StreamWrapper and hence the memory_stream // will be closed when the reader is closed // after Pull()-ing is done. System.IO.Stream html_stream; html_stream = new StreamWrapper(mem_stream); html_stream.Seek(0, SeekOrigin.Begin); stream.Close(); try { this.reader = FilterHtml.GetHtmlReader(html_stream, enc, link_handler); } catch (Exception e) { Log.Debug(e, "Exception while filtering HTML email {0}", this.indexable.Uri); this.reader = null; html_stream.Close(); html_part = false; } } } if (!no_child_needed) { // Check the mime type against the blacklist and don't index any // parts that are contained within. That way the user doesn't // get flooded with pointless signatures and vcard and ical // attachments along with (real) attachments. if (Array.IndexOf(blacklisted_mime_types, mime_type) == -1) { string sub_uri = "#" + this.count; Indexable child; child = new Indexable(UriFu.AddFragment(this.indexable.Uri, sub_uri, true)); child.DisplayUri = new Uri(this.indexable.DisplayUri.ToString() + "#" + this.count); // This is a special case. // Even for mails found on disk, MailMessage hitype is set child.HitType = "MailMessage"; child.MimeType = mime_type; // If this is the richest part we found for multipart emails, add its content to textcache if (snippet_attachment || (this.depth == 1 && this.count == 0)) { child.CacheContent = true; } else { child.CacheContent = false; } string filename = ((GMime.Part)part).Filename; if (!String.IsNullOrEmpty(filename)) { child.AddProperty(Property.NewKeyword("fixme:attachment_title", filename)); foreach (Property prop in Property.StandardFileProperties(filename, false)) { child.AddProperty(prop); } } // Store length of attachment long length = stream.Length; if (length != -1) { child.AddProperty(Property.NewUnsearched("fixme:filesize", length)); } if (part.ContentType.MediaType.ToLower() == "text") { child.SetTextReader(new StreamReader(stream)); } else { child.SetBinaryStream(stream); } child.SetChildOf(this.indexable); child.StoreStream(); child.CloseStreams(); this.child_indexables.Add(child); } else { Log.Debug("Skipping attachment {0}#{1} with blacklisted mime type {2}", this.indexable.Uri, this.count, mime_type); } } this.count++; } if (part_needs_dispose) { part.Dispose(); } --depth; }