private Indexable OperaRowToIndexable(OperaHistory.Row row)
{
// It's unsafe to index secure content since it may contain sensitive data
if (row.Address.Scheme == Uri.UriSchemeHttps)
{
return(null);
}
Indexable indexable = new Indexable(row.Address);
indexable.HitType = "WebHistory";
indexable.MimeType = row.MimeType;
indexable.Timestamp = row.LastVisited;
indexable.AddProperty(Beagle.Property.New("fixme:host", row.Address.Host));
indexable.AddProperty(Beagle.Property.NewUnsearched("fixme:size", row.Length));
// hint for the filter about the charset
indexable.AddProperty(Property.NewUnsearched(StringFu.UnindexedNamespace + "charset", row.Encoding.ToString()));
if (row.Compression == "gzip")
{
indexable.SetBinaryStream(new GZipInputStream(File.OpenRead(Path.Combine(cache_dir, row.LocalFileName))));
}
else
{
indexable.ContentUri = new Uri(Path.Combine(cache_dir, row.LocalFileName));
}
indexer.AttributeStore.AttachLastWriteTime(Path.Combine(cache_dir, row.LocalFileName), DateTime.UtcNow);
return(indexable);
}
private Indexable ToAddMailMessageIndexable(XmlDocument document)
{
GMime.Message message = null;
// Check if the entire message is available
if (ToBool(GetText(document, "HasOffline")))
{
// We must make sure we don't get an exception here since we can fallback to
// other information
try {
int offset = Convert.ToInt32(GetText(document, "MessageOffset")),
size = Convert.ToInt32(GetText(document, "OfflineSize"));
message = GetGMimeMessage(GetText(document, "FolderFile"), offset, size);
} catch (Exception e) {
Logger.Log.Debug(e, "Failed to parse GMime message");
}
}
if (message == null)
{
message = GetStubMessage(document);
}
Indexable indexable = new Indexable(GenerateUniqueUri(document));
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc(Convert.ToInt64(GetText(document, "Date")));
indexable.CacheContent = true;
indexable.FlushBufferCache = true;
indexable.SetBinaryStream(message.Stream);
indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird"));
indexable.AddProperty(Property.NewKeyword("fixme:folder", GetText(document, "Folder")));
indexable.AddProperty(Property.NewUnsearched("ParentUri", GetText(document, "FolderFile")));
indexable.AddProperty(Property.NewUnsearched("fixme:uri", GetText(document, "Uri")));
message.Dispose();
return(indexable);
}
/**
* Create an indexable from an mbox message
* Most of the code here is from Evo backend
*/
public Indexable MessageToIndexable (string file_name, System.Uri uri, GMime.Message message, string folder_name)
{
//Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name);
Indexable indexable = new Indexable (uri);
// set parent uri to the filename so that when an mbox file
// is deleted, all the messages in that file can be deleted
indexable.ParentUri = UriFu.PathToFileUri (file_name);
indexable.Timestamp = message.Date.ToUniversalTime ();
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail"));
indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name));
indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder_name));
GMime.InternetAddressList addrs;
if (folder_name == Queryable.SentMailFolderName) {
addrs = message.GetRecipients (GMime.RecipientType.To);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (folder_name == Queryable.SentMailFolderName) {
addrs = message.GetRecipients (GMime.RecipientType.Cc);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (folder_name != Queryable.SentMailFolderName) {
addrs = GMime.InternetAddressList.Parse (message.Sender);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewKeyword ("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose ();
}
if (folder_name == Queryable.SentMailFolderName)
indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
else {
string kmail_msg_sent = message.GetHeader ("X-KMail-Link-Type");
if (kmail_msg_sent == "reply")
indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
}
// no need to store date again, use the issent flag to determine if the date is sentdate or not
#if false
if (folder_name == Queryable.SentMailFolderName)
indexable.AddProperty (Property.NewDate ("fixme:sentdate", message.Date.ToUniversalTime ()));
else
indexable.AddProperty (Property.NewDate ("fixme:received", message.Date.ToUniversalTime ()));
#endif
indexable.SetBinaryStream (message.Stream);
return indexable;
}
/**
* Create an indexable from an mbox message
* Most of the code here is from Evo backend
*/
public Indexable MessageToIndexable(string file_name, System.Uri uri, GMime.Message message, string folder_name)
{
//Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name);
Indexable indexable = new Indexable(uri);
// set parent uri to the filename so that when an mbox file
// is deleted, all the messages in that file can be deleted
indexable.ParentUri = UriFu.PathToFileUri(file_name);
indexable.Timestamp = message.Date.ToUniversalTime();
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail"));
indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name));
indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder_name));
GMime.InternetAddressList addrs;
if (folder_name == Queryable.SentMailFolderName)
{
addrs = message.GetRecipients(GMime.RecipientType.To);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewKeyword("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
}
if (folder_name == Queryable.SentMailFolderName)
{
addrs = message.GetRecipients(GMime.RecipientType.Cc);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewKeyword("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
}
if (folder_name != Queryable.SentMailFolderName)
{
addrs = GMime.InternetAddressList.Parse(message.Sender);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewKeyword("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose();
}
if (folder_name == Queryable.SentMailFolderName)
{
indexable.AddProperty(Property.NewFlag("fixme:isSent"));
}
else
{
string kmail_msg_sent = message.GetHeader("X-KMail-Link-Type");
if (kmail_msg_sent == "reply")
{
indexable.AddProperty(Property.NewFlag("fixme:isSent"));
}
}
// no need to store date again, use the issent flag to determine if the date is sentdate or not
#if false
if (folder_name == Queryable.SentMailFolderName)
{
indexable.AddProperty(Property.NewDate("fixme:sentdate", message.Date.ToUniversalTime()));
}
else
{
indexable.AddProperty(Property.NewDate("fixme:received", message.Date.ToUniversalTime()));
}
#endif
indexable.SetBinaryStream(message.Stream);
return(indexable);
}
void AddContentToIndexable(Indexable indexable, Post post)
{
try
{
HttpWebRequest request = requester.CreateWebRequest (indexable.Uri.OriginalString);
HttpWebResponse response = request.GetResponse () as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.OK )
{
indexable.MimeType = FilterContentType (response.Headers ["Content-Type"]);
indexable.SetBinaryStream (response.GetResponseStream ());
}
} catch (WebException e)
{
// Can't index the content of the bookmark at this time.
Log.Warn (e, "Error retrieving content for " + indexable.Uri.ToString ());
}
}
private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags)
{
// Don't index messages flagged as junk
if (CheckFlags (flags, B_U_Camel.CamelFlags.Junk))
return null;
System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
Indexable indexable = new Indexable (uri);
indexable.Timestamp = message.Date.ToUniversalTime ();
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution"));
indexable.AddProperty (Property.NewUnsearched ("fixme:account", "Local"));
indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name));
GMime.InternetAddressList addrs;
if (this.folder_name == "Sent") {
addrs = message.GetRecipients (GMime.RecipientType.To);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (this.folder_name == "Sent") {
addrs = message.GetRecipients (GMime.RecipientType.Cc);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose ();
}
if (this.folder_name != "Sent") {
addrs = GMime.InternetAddressList.Parse (message.Sender);
foreach (GMime.InternetAddress ia in addrs) {
if (ia is GMime.InternetAddressMailbox) {
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose ();
}
if (this.folder_name == "Sent")
indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
Property flag_prop = Property.NewUnsearched ("fixme:flags", flags);
flag_prop.IsMutable = true;
indexable.AddProperty (flag_prop);
if (CheckFlags (flags, B_U_Camel.CamelFlags.Answered))
indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Deleted))
indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Draft))
indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Flagged))
indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.Seen))
indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
if (CheckFlags (flags, B_U_Camel.CamelFlags.AnsweredAll))
indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
indexable.SetBinaryStream (message.Stream);
return indexable;
}
private Indexable GMimeMessageToIndexable(string uid, GMime.Message message, uint flags)
{
// Don't index messages flagged as junk
if (CheckFlags(flags, B_U_Camel.CamelFlags.Junk))
{
return(null);
}
System.Uri uri = EvolutionMailQueryable.EmailUri(this.account_name, this.folder_name, uid);
Indexable indexable = new Indexable(uri);
indexable.Timestamp = message.Date.ToUniversalTime();
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.CacheContent = true;
indexable.AddProperty(Property.NewUnsearched("fixme:client", "evolution"));
indexable.AddProperty(Property.NewUnsearched("fixme:account", "Local"));
indexable.AddProperty(Property.NewUnsearched("fixme:folder", this.folder_name));
GMime.InternetAddressList addrs;
if (this.folder_name == "Sent")
{
addrs = message.GetRecipients(GMime.RecipientType.To);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewUnsearched("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
}
if (this.folder_name == "Sent")
{
addrs = message.GetRecipients(GMime.RecipientType.Cc);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewUnsearched("fixme:sentTo", mailbox.Address));
}
}
addrs.Dispose();
}
if (this.folder_name != "Sent")
{
addrs = GMime.InternetAddressList.Parse(message.Sender);
foreach (GMime.InternetAddress ia in addrs)
{
if (ia is GMime.InternetAddressMailbox)
{
GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox;
indexable.AddProperty(Property.NewUnsearched("fixme:gotFrom", mailbox.Address));
}
}
addrs.Dispose();
}
if (this.folder_name == "Sent")
{
indexable.AddProperty(Property.NewFlag("fixme:isSent"));
}
Property flag_prop = Property.NewUnsearched("fixme:flags", flags);
flag_prop.IsMutable = true;
indexable.AddProperty(flag_prop);
if (CheckFlags(flags, B_U_Camel.CamelFlags.Answered))
{
indexable.AddProperty(Property.NewFlag("fixme:isAnswered"));
}
if (CheckFlags(flags, B_U_Camel.CamelFlags.Deleted))
{
indexable.AddProperty(Property.NewFlag("fixme:isDeleted"));
}
if (CheckFlags(flags, B_U_Camel.CamelFlags.Draft))
{
indexable.AddProperty(Property.NewFlag("fixme:isDraft"));
}
if (CheckFlags(flags, B_U_Camel.CamelFlags.Flagged))
{
indexable.AddProperty(Property.NewFlag("fixme:isFlagged"));
}
if (CheckFlags(flags, B_U_Camel.CamelFlags.Seen))
{
indexable.AddProperty(Property.NewFlag("fixme:isSeen"));
}
if (CheckFlags(flags, B_U_Camel.CamelFlags.AnsweredAll))
{
indexable.AddProperty(Property.NewFlag("fixme:isAnsweredAll"));
}
indexable.SetBinaryStream(message.Stream);
return(indexable);
}
private Indexable ToAddMailMessageIndexable (XmlDocument document)
{
GMime.Message message = null;
// Check if the entire message is available
if (ToBool (GetText (document, "HasOffline"))) {
// We must make sure we don't get an exception here since we can fallback to
// other information
try {
int offset = Convert.ToInt32 (GetText (document, "MessageOffset")),
size = Convert.ToInt32 (GetText (document, "OfflineSize"));
message = GetGMimeMessage (GetText (document, "FolderFile"), offset, size);
} catch (Exception e) {
Logger.Log.Debug (e, "Failed to parse GMime message");
}
}
if (message == null)
message = GetStubMessage (document);
Indexable indexable = new Indexable (GenerateUniqueUri (document));
indexable.HitType = "MailMessage";
indexable.MimeType = "message/rfc822";
indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc (Convert.ToInt64 (GetText (document, "Date")));
indexable.CacheContent = true;
indexable.FlushBufferCache = true;
indexable.SetBinaryStream (message.Stream);
indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird"));
indexable.AddProperty (Property.NewKeyword ("fixme:folder", GetText (document, "Folder")));
indexable.AddProperty (Property.NewUnsearched ("ParentUri", GetText (document, "FolderFile")));
indexable.AddProperty (Property.NewUnsearched ("fixme:uri", GetText (document, "Uri")));
message.Dispose ();
return indexable;
}
public void OnEachPart(GMime.Object mime_part)
{
GMime.Object part = null;
bool part_needs_dispose = false;
//for (int i = 0; i < this.depth; i++)
// Console.Write (" ");
//Console.WriteLine ("Content-Type: {0}", mime_part.ContentType);
++depth;
if (mime_part is GMime.MessagePart)
{
GMime.MessagePart msg_part = (GMime.MessagePart)mime_part;
using (GMime.Message message = msg_part.Message) {
using (GMime.Object subpart = message.MimePart)
this.OnEachPart(subpart);
}
}
else if (mime_part is GMime.Multipart)
{
GMime.Multipart multipart = (GMime.Multipart)mime_part;
int num_parts = multipart.Count;
// If the mimetype is multipart/alternative, we only want to index
// one part -- the richest one we can filter.
if (mime_part.ContentType.MediaSubtype.ToLower() == "alternative")
{
// The richest formats are at the end, so work from there
// backward.
for (int i = num_parts - 1; i >= 0; i--)
{
GMime.Object subpart = multipart[i];
if (IsMimeTypeHandled(subpart.ContentType.ToString()))
{
part = subpart;
part_needs_dispose = true;
break;
}
else
{
subpart.Dispose();
}
}
}
// If it's not alternative, or we don't know how to filter any of
// the parts, treat them like a bunch of attachments.
if (part == null)
{
for (int i = 0; i < num_parts; i++)
{
using (GMime.Object subpart = multipart[i])
this.OnEachPart(subpart);
}
}
}
else if (mime_part is GMime.Part)
{
part = mime_part;
}
else
{
throw new Exception(String.Format("Unknown part type: {0}", part.GetType()));
}
if (part != null)
{
System.IO.Stream stream = null;
using (GMime.DataWrapper content_obj = ((GMime.Part)part).ContentObject)
stream = content_obj.Stream;
// If this is the only part and it's plain text, we
// want to just attach it to our filter instead of
// creating a child indexable for it.
bool no_child_needed = false;
string mime_type = part.ContentType.ToString().ToLower();
if (this.depth == 1 && this.count == 0)
{
if (mime_type == "text/plain")
{
no_child_needed = true;
this.reader = new StreamReader(stream);
}
else if (mime_type == "text/html")
{
no_child_needed = true;
html_part = true;
string enc = part.ContentType.GetParameter("charset");
// DataWrapper.Stream is a very limited stream
// and does not allow Seek or Tell
// HtmlFilter requires Stream.Position=0.
// Play safe and create a memorystream
// for HTML parsing.
GMime.StreamMem mem_stream;
mem_stream = new GMime.StreamMem();
GMime.Stream data_stream;
data_stream = ((StreamWrapper)stream).GMimeStream;
data_stream.WriteToStream(mem_stream);
data_stream.Flush();
// The StreamWrapper and hence the memory_stream
// will be closed when the reader is closed
// after Pull()-ing is done.
System.IO.Stream html_stream;
html_stream = new StreamWrapper(mem_stream);
html_stream.Seek(0, SeekOrigin.Begin);
stream.Close();
try {
this.reader = FilterHtml.GetHtmlReader(html_stream, enc, link_handler);
} catch (Exception e) {
Log.Debug(e, "Exception while filtering HTML email {0}", this.indexable.Uri);
this.reader = null;
html_stream.Close();
html_part = false;
}
}
}
if (!no_child_needed)
{
// Check the mime type against the blacklist and don't index any
// parts that are contained within. That way the user doesn't
// get flooded with pointless signatures and vcard and ical
// attachments along with (real) attachments.
if (Array.IndexOf(blacklisted_mime_types, mime_type) == -1)
{
string sub_uri = "#" + this.count;
Indexable child;
child = new Indexable(UriFu.AddFragment(this.indexable.Uri, sub_uri, true));
child.DisplayUri = new Uri(this.indexable.DisplayUri.ToString() + "#" + this.count);
// This is a special case.
// Even for mails found on disk, MailMessage hitype is set
child.HitType = "MailMessage";
child.MimeType = mime_type;
// If this is the richest part we found for multipart emails, add its content to textcache
if (snippet_attachment ||
(this.depth == 1 && this.count == 0))
{
child.CacheContent = true;
}
else
{
child.CacheContent = false;
}
string filename = ((GMime.Part)part).Filename;
if (!String.IsNullOrEmpty(filename))
{
child.AddProperty(Property.NewKeyword("fixme:attachment_title", filename));
foreach (Property prop in Property.StandardFileProperties(filename, false))
{
child.AddProperty(prop);
}
}
// Store length of attachment
long length = stream.Length;
if (length != -1)
{
child.AddProperty(Property.NewUnsearched("fixme:filesize", length));
}
if (part.ContentType.MediaType.ToLower() == "text")
{
child.SetTextReader(new StreamReader(stream));
}
else
{
child.SetBinaryStream(stream);
}
child.SetChildOf(this.indexable);
child.StoreStream();
child.CloseStreams();
this.child_indexables.Add(child);
}
else
{
Log.Debug("Skipping attachment {0}#{1} with blacklisted mime type {2}",
this.indexable.Uri, this.count, mime_type);
}
}
this.count++;
}
if (part_needs_dispose)
{
part.Dispose();
}
--depth;
}
