// We assume that // (a) indexable.Uri is a uid: URI // (b) indexable.ContentUri is a file: URI // (c) indexable.ContentUri's filename is meaningful (i.e. isn't a temporary file) public void Add (Indexable indexable) { driver.Add (indexable); Guid uid = GuidFu.FromUri (indexable.Uri); string name = FileSystem.GetFileName (indexable.ContentUri.LocalPath); name_index.Add (uid, name); }
public override bool GenerateNextIndexable (out Indexable child) { child = null; if (entry_enumerator == null || ! entry_enumerator.MoveNext ()) return false; child = (Indexable) entry_enumerator.Current; return true; }
private void IndexFile(FileInfo data_file) { Indexable indexable = FileToIndexable(data_file); if (indexable == null) // The file disappeared { return; } Scheduler.Task task = NewAddTask(indexable); task.Priority = Scheduler.Priority.Immediate; ThisScheduler.Add(task); }
protected override Uri PostAddHook(Indexable indexable, IndexerAddedReceipt receipt) { base.PostAddHook(indexable, receipt); if (indexable_text_cache.ContainsKey(indexable.Uri)) { string text = (string)indexable_text_cache [indexable.Uri]; TextCache.UserCache.WriteFromString(indexable.Uri, text); indexable_text_cache.Remove(indexable.Uri); } return(indexable.Uri); }
private Indexable current_itemToIndexable() { Indexable indexable; try { indexable = new Indexable(new Uri(String.Format("{0};item={1}", feed_source, current_item.Source))); } catch (System.UriFormatException) { indexable = new Indexable(new Uri(String.Format("liferea://dummy?{0};item={1}", feed_source, current_item.Source))); } indexable.ParentUri = UriFu.PathToFileUri(feed_file); indexable.MimeType = "text/html"; indexable.HitType = "FeedItem"; DateTime date = DateTimeUtil.UnixToDateTimeUtc(0); date = date.AddSeconds(current_item.Timestamp); indexable.Timestamp = date; // cleaning up the property names as far as possible // this way querying for specific field is possible // following DC element names wherever applicable indexable.AddProperty(Property.New("dc:title", current_item.Title)); Attribute[] attribs = current_item.Attribs.AttribArray; if (attribs != null) { foreach (Attribute attrib in attribs) { if (attrib.Name != "author") { continue; } indexable.AddProperty(Property.New("dc:creator", attrib.Value)); } } indexable.AddProperty(Property.NewKeyword("dc:identifier", current_item.Source)); indexable.AddProperty(Property.NewKeyword("dc:source", feed_source)); indexable.AddProperty(Property.New("dc:publisher", publisher)); if (File.Exists(icon_file)) { indexable.AddProperty(Property.NewUnsearched("fixme:cachedimg", icon_file)); } StringReader reader = new StringReader(current_item.Description); indexable.SetTextReader(reader); return(indexable); }
public Indexable GetNextIndexable() { GMime.Message message = null; try { message = mbox_parser.ConstructMessage(); } catch (System.IO.FileNotFoundException e) { Logger.Log.Warn("mbox " + mbox_file + " deleted while parsing."); return(null); } try { // Again comment from Evo :P // Work around what I think is a bug in GMime: If you // have a zero-byte file or seek to the end of a // file, parser.Eos () will return true until it // actually tries to read something off the wire. // Since parser.ConstructMessage() always returns a // message (which may also be a bug), we'll often get // one empty message which we need to deal with here. // // Check if its empty by seeing if the Headers // property is null or empty. if (message == null || message.Headers == null || message.Headers == "") { return(null); } // mbox KIO slave uses the From line as URI - how weird! // are those lines supposed to be unique ??? string id = mbox_parser.From; System.Uri uri = EmailUri(id); Indexable indexable = indexer.MessageToIndexable(mbox_file, uri, message, indexer.GetFolderMbox(mbox_file)); if (indexable == null) { return(null); } ++indexed_count; return(indexable); } finally { if (message != null) { message.Dispose(); } } }
private Indexable ContactToIndexable(TB.Contact contact) { Indexable indexable = NewIndexable(contact.Uri, DateTime.Now.ToUniversalTime(), "Contact"); indexable.AddProperty(Property.New("fixme:FirstName", contact.GetString("FirstName"))); indexable.AddProperty(Property.New("fixme:LastName", contact.GetString("LastName"))); indexable.AddProperty(Property.New("fixme:DisplayName", contact.GetString("LastName"))); indexable.AddProperty(Property.New("fixme:NickName", contact.GetString("NickName"))); indexable.AddProperty(Property.NewKeyword("fixme:PrimaryEmail", contact.GetString("PrimaryEmail"))); indexable.AddProperty(Property.NewKeyword("fixme:SecondEmail", contact.GetString("SecondEmail"))); indexable.AddProperty(Property.New("fixme:WorkPhone", contact.GetString("WorkPhone"))); indexable.AddProperty(Property.New("fixme:FaxNumber", contact.GetString("FaxNumber"))); indexable.AddProperty(Property.New("fixme:HomePhone", contact.GetString("HomePhone"))); indexable.AddProperty(Property.New("fixme:PagerNumber", contact.GetString("PagerNumber"))); indexable.AddProperty(Property.New("fixme:CellularNumber", contact.GetString("CellularNumber"))); indexable.AddProperty(Property.New("fixme:HomeAddress", contact.GetString("HomeAddress"))); indexable.AddProperty(Property.New("fixme:HomeAddress2", contact.GetString("HomeAddress2"))); indexable.AddProperty(Property.New("fixme:HomeCity", contact.GetString("HomeCity"))); indexable.AddProperty(Property.New("fixme:HomeState", contact.GetString("HomeState"))); indexable.AddProperty(Property.New("fixme:HomeZipCode", contact.GetString("HomeZipCode"))); indexable.AddProperty(Property.New("fixme:HomeCountry", contact.GetString("HomeCountry"))); indexable.AddProperty(Property.New("fixme:WorkAddress", contact.GetString("WorkAddress"))); indexable.AddProperty(Property.New("fixme:WorkAddress2", contact.GetString("WorkAddress2"))); indexable.AddProperty(Property.New("fixme:WorkCity", contact.GetString("WorkCity"))); indexable.AddProperty(Property.New("fixme:WorkState", contact.GetString("WorkState"))); indexable.AddProperty(Property.New("fixme:WorkZipCode", contact.GetString("WorkZipCode"))); indexable.AddProperty(Property.New("fixme:WorkCountry", contact.GetString("WorkCountry"))); indexable.AddProperty(Property.New("fixme:JobTitle", contact.GetString("JobTitle"))); indexable.AddProperty(Property.New("fixme:Department", contact.GetString("Department"))); indexable.AddProperty(Property.New("fixme:Company", contact.GetString("Company"))); indexable.AddProperty(Property.New("fixme:_AimScreenName", contact.GetString("_AimScreenName"))); indexable.AddProperty(Property.New("fixme:FamilyName", contact.GetString("FamilyName"))); indexable.AddProperty(Property.NewKeyword("fixme:WebPage1", contact.GetString("WebPage1"))); indexable.AddProperty(Property.NewKeyword("fixme:WebPage2", contact.GetString("WebPage2"))); indexable.AddProperty(Property.New("fixme:BirthYear", contact.GetString("BirthYear"))); indexable.AddProperty(Property.New("fixme:BirthMonth", contact.GetString("BirthMonth"))); indexable.AddProperty(Property.New("fixme:BirthDay", contact.GetString("BirthDay"))); indexable.AddProperty(Property.New("fixme:Custom1", contact.GetString("Custom1"))); indexable.AddProperty(Property.New("fixme:Custom2", contact.GetString("Custom2"))); indexable.AddProperty(Property.New("fixme:Custom3", contact.GetString("Custom3"))); indexable.AddProperty(Property.New("fixme:Custom4", contact.GetString("Custom4"))); indexable.AddProperty(Property.New("fixme:Notes", contact.GetString("Notes"))); indexable.AddProperty(Property.New("fixme:PreferMailFormat", contact.GetString("PreferMailFormat"))); indexable.AddProperty(Property.NewKeyword("fixme:Email", contact.GetString("PrimaryEmail"))); indexable.AddProperty(Property.New("fixme:Name", contact.GetString("DisplayName"))); return(indexable); }
public Registers(int GPLen = 64, bool StartPrivileged = true) { GP = new Int64[GPLen]; CRFlags = new Indexable<int, bool>((K) => { return CR.GetBit(K); }, (K, V) => { CR = CR.SetBit(K, V); }, (K) => { if (K < 0 || K >= sizeof(UInt64) * 8) throw new CPUException("Invalid CRFlags bit range: {0}", K); }); Privileged = StartPrivileged; }
// MergeXmpData can return an entirely new indexable or just change the passed indexable public Indexable MergeXmpData (ref Indexable indexable, string path, Guid id, DirectoryModel parent, bool crawling) { // In crawl mode, whenever xmp file is encountered, it either schedules the basefile or notices that // the basefile is already scheduled. In short, it is properly taken care of. So, during // crawling, we can return the indexable created at this point instead of checking the // existence of foo.xmp for each crawled file foo. if (crawling) return null; else return GetNewXmpIndexable (ref indexable, path, id, parent); }
static void Main(string[] args) { // Indexable ix = new Indexable(); Console.WriteLine(ix[10]); ix[42] = "Xyzzy"; // ImmutableValue(); // MutableValue(); Console.ReadKey(); }
static private Indexable MemberNodeToIndexable(XmlNode node, Uri base_uri, string parentName) { char memberType = MemberTypeToChar(node.SelectSingleNode("MemberType").InnerText); StringBuilder memberFullName = new StringBuilder(); memberFullName.Append(memberType + ":" + parentName); if (memberType != 'C') { memberFullName.Append("." + node.Attributes["MemberName"].Value); } if (memberType == 'C' || memberType == 'M' || memberType == 'E') { memberFullName.Append("("); bool inside = false; foreach (XmlNode parameter in node.SelectNodes("Parameters/Parameter")) { if (!inside) { inside = true; } else { memberFullName.Append(","); } memberFullName.Append(parameter.Attributes["Type"].Value); } memberFullName.Append(")"); } Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, memberFullName.ToString(), false)); indexable.MimeType = "text/html"; indexable.HitType = "MonodocEntry"; indexable.AddProperty(Property.New("dc:title", memberFullName.ToString())); indexable.AddProperty(Property.New("fixme:name", memberFullName.ToString())); indexable.AddProperty(Property.NewUnsearched("fixme:type", node.SelectSingleNode("MemberType").InnerText.ToLower())); StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml); indexable.SetTextReader(reader); return(indexable); }
protected override Uri PostAddHook(Indexable indexable, IndexerAddedReceipt receipt) { FileInfo meta_file = indexable.LocalState ["MetaFile"] as FileInfo; if (meta_file == null) { return(indexable.Uri); } meta_file.Delete(); lock (pending_files) pending_files.Remove(indexable.ContentUri.LocalPath); return(indexable.Uri); }
protected virtual Indexable NewIndexable(Uri uri, DateTime timestamp, string hit_type) { Indexable indexable; indexable = new Indexable(uri); indexable.HitType = hit_type; indexable.Timestamp = timestamp; indexable.AddProperty(Property.NewKeyword("fixme:account", account.Server)); indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewUnsearched("fixme:fullyIndexed", full_index)); indexable.AddProperty(Property.NewUnsearched("fixme:file", RelativePath)); indexable.AddProperty(Property.NewDate("fixme:indexDateTime", DateTime.UtcNow)); return(indexable); }
public void Add(Indexable indexable) { if (indexable == null) { return; } if (indexables_by_uri == null) { indexables_by_uri = UriFu.NewHashtable(); } Indexable prior; prior = indexables_by_uri [indexable.Uri] as Indexable; if (prior != null) { switch (indexable.Type) { case IndexableType.Add: case IndexableType.Remove: // Clobber the prior indexable. indexable.Id = prior.Id; indexables [prior.Id] = indexable; indexables_by_uri [indexable.Uri] = indexable; break; case IndexableType.PropertyChange: if (prior.Type != IndexableType.Remove) { // Merge with the prior indexable. prior.Merge(indexable); } break; } } else { lock (indexable_id_lock) { indexable.Id = indexable_id; indexable_id++; } indexables [indexable.Id] = indexable; indexables_by_uri [indexable.Uri] = indexable; } }
override protected bool PreAddIndexableHook (Indexable indexable) { // None of this applies for Removes if (indexable.Type == IndexableType.Remove) return true; CachedFileInfo info = (CachedFileInfo) file_info_cache [indexable.Uri]; if (info == null) info = new CachedFileInfo (); info.Uri = indexable.Uri; if (indexable.Uri.IsFile && indexable.IsNonTransient) info.Path = indexable.Uri.LocalPath; else if (indexable.ContentUri.IsFile && indexable.IsNonTransient) info.Path = indexable.ContentUri.LocalPath; else if (indexable.ParentUri != null && indexable.ParentUri.IsFile) { info.Path = indexable.ParentUri.LocalPath; info.Shared = true; } // The path could be null in certain cases: // * The indexable is a non-file URI and no // parent URI is set. // * The indexable is a child indexable and the // parent URI is not a file URI. if (info.Path == null) return true; info.Mtime = FileSystem.GetLastWriteTimeUtc (info.Path); if (! FileSystem.ExistsByDateTime (info.Mtime)) { // If we can't get an mtime for the file, it must // have disappeared out from under us. In that case, // don't bother adding anything. return false; } file_info_cache [info.Uri] = info; // If we are all set to authorize this indexable, increment reference count for the path if (info.Shared) IncrementReferenceCount (info.Path); return true; }
public void Add (Indexable indexable, Scheduler.Priority priority) { lock (indexables) { indexables.Enqueue (indexable); if (priority > highest_prio) highest_prio = priority; if (self_task == null) { self_task = queryable.NewAddTask (this); self_task.Priority = highest_prio; queryable.ThisScheduler.Add (self_task); } else { self_task.Priority = highest_prio; } } }
public static T MyMax4 <T, TIndexable, TComparer> (this Indexable <T, TIndexable> indexable, TComparer comparer) where TIndexable : struct, IIndexable <T> where TComparer : struct, IComparer4 <T> { var value = indexable.Value; var maxValue = value[value.Length - 1]; for (int i = value.Length - 1; i >= 0; i--) { if (comparer.IsMore(value[i], maxValue)) { maxValue = value[i]; } } return(maxValue); }
private Indexable GetNewXmpIndexable (ref Indexable indexable, string path, Guid id, DirectoryModel parent) { // In non-crawl mode, check if a corresponding xmp file is present and not already scheduled and index it. // If file.xmp and file are rapidly written/updated (in that order), this does the right thing. // If file and file.xmp are rapidly written/updated (in that order), either // - file.xmp is present during FileToIndexable(file): in which case xmp properties are // added to file; and when file.xmp is indexed, it will replace the xmp properties // - file.xmp is not present during FileToIndexable(file): when the xmp file is later indexed // it will add the xmp properties // since the uid file will still be in the uid-cache, correct uid will be used for xmp prop-change indexable string possible_xmp_file_path = string.Concat (path, ".xmp"); if (! File.Exists (possible_xmp_file_path)) return null; Guid xmp_id = queryable.RegisterFile (parent, (Path.GetFileName (possible_xmp_file_path))); if (xmp_id == Guid.Empty) return null; XmpFile xmp_file = null; try { xmp_file = new XmpFile (possible_xmp_file_path); } catch { uid_manager.ForgetNewId (possible_xmp_file_path); return null; } // FIXME: Should also delete previous xmp properties! foreach (Property p in xmp_file.Properties) { p.IsMutable = true; indexable.AddProperty (p); } xmp_file.Close (); // Also need to save some local states for PostAddHook, // namely, path to the xmp file, path to basefile and generated uid indexable.LocalState ["XmpFilePath"] = possible_xmp_file_path; indexable.LocalState ["BaseFilePath"] = path; indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString (xmp_id); if (Debug) Log.Debug ("Adding properties from {0}({2}) to {1}({3})", possible_xmp_file_path, path, GuidFu.ToShortString (xmp_id), GuidFu.ToShortString (id)); return null; }
override protected Uri PostAddHook(Indexable indexable, IndexerAddedReceipt receipt) { // Retrieve our cached info about the file. CachedFileInfo info; info = file_info_cache [indexable.Uri] as CachedFileInfo; if (info == null) { return(indexable.Uri); } file_info_cache.Remove(info.Uri); // Yeah, this is ghetto. If it's a file that's shared across multiple // indexables, only tag it with when the last indexable has been indexed. if (info.Shared && DecrementReferenceCount(info.Path)) { return(indexable.Uri); } // Since we know that the file has been successfully // indexed, update the file attributes accordingly. // Don't set filter information on a file if multiple // indexables has been created from it. FileAttributes attr; attr = FileAttributesStore.ReadOrCreate(info.Path); attr.LastWriteTime = info.Mtime; // Don't set filter information on a file if multiple indexables has been // created from it. if (!info.Shared) { attr.FilterName = receipt.FilterName; attr.FilterVersion = receipt.FilterVersion; } if (!FileAttributesStore.Write(attr)) { Logger.Log.Warn("Couldn't write attributes for {0}", info.Path); } return(indexable.Uri); }
public void Index(FileInfo file) { Scheduler.TaskGroup group = NewMarkingTaskGroup(file.FullName, file.LastWriteTime); MessageReader reader = new MessageReader(file.FullName); while (reader.HasMoreMessages) { Message message = reader.NextMessage; Indexable indexable = MessageToIndexable(message); Scheduler.Task task = NewAddTask(indexable); task.Priority = Scheduler.Priority.Delayed; task.SubPriority = 0; task.AddTaskGroup(group); ThisScheduler.Add(task); } }
void IndexSingleFile(string path) { if (path.EndsWith(".new")) { return; } Indexable indexable = FileToIndexable(path, false); if (indexable == null) { return; } Scheduler.Task task = NewAddTask(indexable); task.Priority = Scheduler.Priority.Immediate; task.Tag = path; task.SubPriority = 0; ThisScheduler.Add(task); }
public Indexable GetNextIndexable() { FileInfo file = (FileInfo)map_files.Current; if (!file.Exists) { return(null); } if (IsUpToDate(file.FullName)) { return(null); } Indexable indexable = NoteToIndexable(file); return(indexable); }
override protected Uri PostAddHook(Indexable indexable, IndexerAddedReceipt receipt) { base.PostAddHook(indexable, receipt); // Store the note's text in the text cache. // By doing this in the PostAddHook, we ensure that // the TextCache is not modified until we are // sure that the note was actually indexed. string text; text = (string)note_text_cache [indexable.Uri]; // If text == null, this is equivalent to // calling Delete (receipt.Uri) TextCache.UserCache.WriteFromString(indexable.Uri, text); note_text_cache.Remove(indexable.Uri); return(indexable.Uri); }
void AddContentToIndexable(Indexable indexable, Post post) { try { HttpWebRequest request = requester.CreateWebRequest (indexable.Uri.OriginalString); HttpWebResponse response = request.GetResponse () as HttpWebResponse; if (response.StatusCode == HttpStatusCode.OK ) { indexable.MimeType = FilterContentType (response.Headers ["Content-Type"]); indexable.SetBinaryStream (response.GetResponseStream ()); } } catch (WebException e) { // Can't index the content of the bookmark at this time. Log.Warn (e, "Error retrieving content for " + indexable.Uri.ToString ()); } }
static private Indexable TypeNodeToIndexable(XmlNode node, Uri base_uri) { string fragment = "T:" + node.Attributes ["FullName"].Value; Indexable indexable = new Indexable(UriFu.AddFragment(base_uri, fragment, false)); indexable.MimeType = "text/html"; indexable.HitType = "MonodocEntry"; indexable.AddProperty(Property.New("dc:title", "T:" + node.Attributes["FullName"].Value)); indexable.AddProperty(Property.NewUnsearched("fixme:name", "T:" + node.Attributes["FullName"].Value)); indexable.AddProperty(Property.NewUnsearched("fixme:type", "type")); StringReader reader = new StringReader(node.SelectSingleNode("Docs").InnerXml); indexable.SetTextReader(reader); return(indexable); }
static void Main(string[] args) { if (args.Length != 2) { Console.WriteLine("Usage: beagle-master-delete-button index-name uri-to-delete"); return; } string index_name = args [0]; LuceneQueryingDriver driver = new LuceneQueryingDriver(index_name, -1, true); Uri uri = new Uri(args [1], false); Uri uri_to_delete = RemapUri(driver, uri); LuceneIndexingDriver indexer = new LuceneIndexingDriver(index_name, false); Indexable indexable = new Indexable(uri_to_delete); indexable.Type = IndexableType.Remove; IndexerRequest request = new IndexerRequest(); request.Add(indexable); IndexerReceipt [] receipts = indexer.Flush(request); if (receipts == null || receipts.Length == 0) { Console.WriteLine("Uri {0} not found in {1}", uri, index_name); return; } IndexerRemovedReceipt r = receipts [0] as IndexerRemovedReceipt; if (r == null || r.NumRemoved == 0) { Console.WriteLine("Uri {0} not found in {1}", uri, index_name); return; } Console.WriteLine("Uri {0} deleted", uri); }
public static void SendUpdate(IBrowsableItem item) { Indexable indexable = new Indexable(item.DefaultVersionUri); indexable.Type = IndexableType.PropertyChange; Beagle.Property prop; // Clear the existing tags prop = Beagle.Property.NewKeyword("fspot:Tag", ""); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); prop = Beagle.Property.NewKeyword("image:Tag", ""); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); foreach (Tag t in item.Tags) { prop = Beagle.Property.NewKeyword("fspot:Tag", t.Name); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); prop = Beagle.Property.NewKeyword("image:Tag", t.Name); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); } prop = Beagle.Property.New("fspot:Description", item.Description); prop.IsMutable = true; prop.IsPersistent = true; indexable.AddProperty(prop); // Create a message to send to the daemon with this information. // The source tells it what index the existing "/home/joe/test.txt" document lives. IndexingServiceRequest req = new IndexingServiceRequest(); req.Keepalive = false; req.Source = "Files"; req.Add(indexable); req.SendAsync(); }
private void AddDocumentToIndex(Indexable indexable, Document persistent_prop_doc, IndexWriter primary_writer, ref IndexWriter secondary_writer) { Document primary_doc = null, secondary_doc = null; try { BuildDocuments(indexable, out primary_doc, out secondary_doc); primary_writer.AddDocument(primary_doc); } catch (Exception ex) { // If an exception was thrown, something bad probably happened // while we were filtering the content. Set NoContent to true // and try again -- that way it will at least end up in the index, // even if we don't manage to extract the fulltext. Logger.Log.Debug(ex, "First attempt to index {0} failed", indexable.DisplayUri); indexable.NoContent = true; try { BuildDocuments(indexable, out primary_doc, out secondary_doc); primary_writer.AddDocument(primary_doc); } catch (Exception ex2) { Logger.Log.Debug(ex2, "Second attempt to index {0} failed, giving up...", indexable.DisplayUri); } } secondary_doc = MergeDocuments(secondary_doc, persistent_prop_doc); if (secondary_doc != null) { if (secondary_writer == null) { secondary_writer = new IndexWriter(SecondaryStore, IndexingAnalyzer, false); } secondary_writer.AddDocument(secondary_doc); } AdjustItemCount(1); }
private Indexable EventToIndexable(CalComponent cc) { Indexable indexable = new Indexable(GetComponentUri(cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Calendar"; indexable.AddProperty(Property.NewUnsearched("fixme:source_uid", this.source.Uid)); indexable.AddProperty(Property.NewUnsearched("fixme:uid", cc.Uid)); indexable.AddProperty(Property.NewDate("fixme:starttime", cc.Dtstart.ToUniversalTime())); if (cc.Dtend != DateTime.MinValue) { indexable.AddProperty(Property.NewDate("fixme:endtime", cc.Dtend.ToUniversalTime())); } foreach (CalComponentAttendee attendee in cc.Attendees) { indexable.AddProperty(Property.New("fixme:attendee", attendee.value)); } foreach (string comment in cc.Comments) { indexable.AddProperty(Property.New("fixme:comment", comment)); } foreach (string description in cc.Descriptions) { indexable.AddProperty(Property.New("fixme:description", description)); } indexable.AddProperty(Property.New("fixme:summary", cc.Summary)); foreach (string category in cc.Categories) { indexable.AddProperty(Property.NewUnsearched("fixme:category", category)); } indexable.AddProperty(Property.New("fixme:location", cc.Location)); return(indexable); }
private void IndexNote(FileInfo file, Scheduler.Priority priority) { if (!File.Exists(file.FullName)) { return; } if (IsUpToDate(file.FullName)) { return; } Indexable indexable = NoteToIndexable(file); Scheduler.Task task = NewAddTask(indexable); task.Priority = priority; task.SubPriority = 0; ThisScheduler.Add(task); }
// FIXME: This need some more info private Indexable NntpMessageToIndexable(TB.NntpMessage message) { Indexable indexable; indexable = new Indexable(message.Uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTime.Parse(message.GetString("date")).ToUniversalTime(); indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewUnsearched("fixme:fullyIndexed", message.GetBool("FullIndex"))); indexable.AddProperty(Property.NewDate("fixme:indexDateTime", DateTime.UtcNow)); string subject = GMime.Utils.HeaderDecodePhrase(message.GetString("subject")); indexable.AddProperty(Property.New("dc:title", subject)); return(indexable); }
// MergeXmpData can return an entirely new indexable or just change the passed indexable public Indexable MergeXmpData(ref Indexable indexable, string path, Guid id, DirectoryModel parent, bool crawling) { // In crawl mode, whenever xmp file is encountered, it either schedules the basefile or notices that // the basefile is already scheduled. In short, it is properly taken care of. So, during // crawling, we can return the indexable created at this point instead of checking the // existence of foo.xmp for each crawled file foo. if (crawling) { return(null); } else { return(GetNewXmpIndexable(ref indexable, path, id, parent)); } }
// Returns the indexable generating this receipt and also remove it from the cache public Indexable RetrieveRequestIndexable(IndexerReceipt r) { Indexable requested_indexable = null; if (indexables.TryGetValue(r.Id, out requested_indexable)) { indexables.Remove(r.Id); } else if (deferred_indexables.TryGetValue(r.Id, out requested_indexable)) { deferred_indexables.Remove(r.Id); } else { Log.Warn("Unable to match receipt #{0} to any previous request!", r.Id); } return(requested_indexable); }
private void IndexLog(string filename, Scheduler.Priority priority) { if (!File.Exists(filename)) { return; } if (IsUpToDate(filename)) { return; } Indexable indexable = ImLogToIndexable(filename); Scheduler.Task task = NewAddTask(indexable); task.Priority = priority; task.SubPriority = 0; ThisScheduler.Add(task); }
/** * Create an indexable from a maildir message */ public Indexable MaildirMessageToIndexable(string filename, bool crawl) { //Logger.Log.Debug ("+ indexing maildir mail:" + filename); String folder = GetFolderMaildir(filename); Uri file_uri = UriFu.PathToFileUri(filename); Indexable indexable = new Indexable(file_uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.FlushBufferCache = crawl; indexable.AddProperty(Property.NewUnsearched("fixme:client", "kmail")); indexable.AddProperty(Property.NewUnsearched("fixme:account", account_name)); indexable.AddProperty(Property.NewUnsearched("fixme:folder", folder)); indexable.ContentUri = file_uri; return(indexable); }
public virtual void AddItemFields() { try { VerboseLogging.CrawlingLogDebug(() => "AddItemFields start"); if (Options.IndexAllFields) { Indexable.LoadAllFields(); } if (IsParallel) { var exceptions = new ConcurrentQueue <Exception>(); Parallel.ForEach(Indexable.Fields, ParallelOptions, f => { try { CheckAndAddField(Indexable, f); } catch (Exception ex) { exceptions.Enqueue(ex); } }); if (exceptions.Count > 0) { throw new AggregateException(exceptions); } } else { foreach (var field in Indexable.Fields) { CheckAndAddField(Indexable, field); } } } finally { VerboseLogging.CrawlingLogDebug(() => "AddItemFields End"); } }
private Indexable ToAddMailMessageIndexable(XmlDocument document) { GMime.Message message = null; // Check if the entire message is available if (ToBool(GetText(document, "HasOffline"))) { // We must make sure we don't get an exception here since we can fallback to // other information try { int offset = Convert.ToInt32(GetText(document, "MessageOffset")), size = Convert.ToInt32(GetText(document, "OfflineSize")); message = GetGMimeMessage(GetText(document, "FolderFile"), offset, size); } catch (Exception e) { Logger.Log.Debug(e, "Failed to parse GMime message"); } } if (message == null) { message = GetStubMessage(document); } Indexable indexable = new Indexable(GenerateUniqueUri(document)); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTimeUtil.UnixToDateTimeUtc(Convert.ToInt64(GetText(document, "Date"))); indexable.CacheContent = true; indexable.FlushBufferCache = true; indexable.SetBinaryStream(message.Stream); indexable.AddProperty(Property.NewKeyword("fixme:client", "thunderbird")); indexable.AddProperty(Property.NewKeyword("fixme:folder", GetText(document, "Folder"))); indexable.AddProperty(Property.NewUnsearched("ParentUri", GetText(document, "FolderFile"))); indexable.AddProperty(Property.NewUnsearched("fixme:uri", GetText(document, "Uri"))); message.Dispose(); return(indexable); }
///////////////////////////////////////////////// private void OnInotifyEvent(Inotify.Watch watch, string path, string subitem, string srcpath, Inotify.EventType type) { if (subitem == "") { return; } if (Path.GetExtension(subitem) != ".xml") { return; } // We're only handling MovedTo events here. string file = Path.Combine(path, subitem); DateTime last_checked = DateTime.MinValue; FileAttributes attr; attr = FileAttributesStore.Read(file); if (attr != null) { last_checked = attr.LastWriteTime; } foreach (NautilusTools.NautilusMetadata nm in NautilusTools.GetMetadata(file, last_checked)) { Indexable indexable = GetIndexable(nm); Scheduler.Task task; task = this.target_queryable.NewAddTask(indexable); task.Priority = Scheduler.Priority.Immediate; ThisScheduler.Add(task); } }
public void Add (Indexable indexable) { if (indexable == null) return; if (indexables_by_uri == null) indexables_by_uri = UriFu.NewHashtable (); Indexable prior; prior = indexables_by_uri [indexable.Uri] as Indexable; if (prior != null) { switch (indexable.Type) { case IndexableType.Add: case IndexableType.Remove: // Clobber the prior indexable. indexable.Id = prior.Id; indexables [prior.Id] = indexable; indexables_by_uri [indexable.Uri] = indexable; break; case IndexableType.PropertyChange: if (prior.Type != IndexableType.Remove) { // Merge with the prior indexable. prior.Merge (indexable); } break; } } else { lock (indexable_id_lock) { indexable.Id = indexable_id; indexable_id ++; } indexables [indexable.Id] = indexable; indexables_by_uri [indexable.Uri] = indexable; } }
protected override Indexable MessageToIndexable (Message message) { Uri uri = new Uri (String.Format ("email:///{0};id={1}", message.Path, message.Id)); Indexable indexable = new Indexable (uri); indexable.Type = "MailMessage"; indexable.AddProperty (Property.New ("fixme:client", "mozilla")); indexable.AddProperty (Property.New ("dc:title", message.Subject)); indexable.AddProperty (Property.New ("fixme:subject", message.Subject)); indexable.AddProperty (Property.New ("fixme:to", message.To)); indexable.AddProperty (Property.New ("fixme:from", message.From)); indexable.AddProperty (Property.New ("fixme:offset", message.Offset)); StringReader reader = new StringReader (message.Body); indexable.SetTextReader (reader); return indexable; }
private Indexable EventToIndexable (CalComponent cc) { Indexable indexable = new Indexable (GetComponentUri (cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Calendar"; indexable.AddProperty (Property.NewUnsearched ("fixme:source_uid", this.source.Uid)); indexable.AddProperty (Property.NewUnsearched ("fixme:uid", cc.Uid)); indexable.AddProperty (Property.NewDate ("fixme:starttime", cc.Dtstart.ToUniversalTime ())); if (cc.Dtend != DateTime.MinValue) indexable.AddProperty (Property.NewDate ("fixme:endtime", cc.Dtend.ToUniversalTime ())); foreach (CalComponentAttendee attendee in cc.Attendees) indexable.AddProperty (Property.New ("fixme:attendee", attendee.value)); foreach (string comment in cc.Comments) indexable.AddProperty (Property.New ("fixme:comment", comment)); foreach (string description in cc.Descriptions) indexable.AddProperty (Property.New ("fixme:description", description)); indexable.AddProperty (Property.New ("fixme:summary", cc.Summary)); foreach (string category in cc.Categories) indexable.AddProperty (Property.NewUnsearched ("fixme:category", category)); indexable.AddProperty (Property.New ("fixme:location", cc.Location)); return indexable; }
private Indexable MemoToIndexable (CalComponent cc) { Indexable indexable = new Indexable (GetComponentUri (cc)); indexable.Timestamp = cc.Dtstart; indexable.HitType = "Note"; indexable.Filtering = IndexableFiltering.AlreadyFiltered; indexable.AddProperty (Property.NewUnsearched ("fixme:application","evolution")); indexable.AddProperty (Property.New ("dc:title", cc.Summary)); // We remember the note's text so that we can stuff it in // the TextCache later. // This is here form compability with Tomboy notes. foreach (string description in cc.Descriptions) { queryable.IndexableTextCache [indexable.Uri] = description; StringReader reader = new StringReader (description); indexable.SetTextReader (reader); } return indexable; }
public Indexable GetXmpQueryable (string path, Guid id, DirectoryModel parent) { Log.Debug ("Asked to create xmp indexable for ({0}) {1}", GuidFu.ToShortString (id), path); // Should be at least 6 characters /<...>.xmp if (path.Length < 6) return null; string basefile_path = Path.ChangeExtension (path, null); // Ignore xmp files by itself // FIXME: To support indexing independent xmp files will require even greater trouble if (! File.Exists (basefile_path)) return null; XmpFile xmp_file = null; try { xmp_file = new XmpFile (path); } catch { Log.Warn ("Cannot create xmpfile from {0}", path); return null; } // Try to get the correct uid for the basefile // First we need to see if basefile is already scheduled (yet to be dispatched) Uri basefile_uri = null; Indexable base_indexable; if (uid_manager.HasNewId (basefile_path)) { // Since uid_manager has a new id for this basefile, so basefile is already scheduled // Get basefile uid from there Guid basefile_id = uid_manager.GetNewId (basefile_path); basefile_uri = GuidFu.ToUri (basefile_id); Log.Debug ("{0} is already scheduled with uri {1}", basefile_path, basefile_uri); } else { // Basefile is not scheduled in the current batch string basefile_name = Path.GetFileName (basefile_path); // Try to schedule it for addition base_indexable = queryable.GetCrawlingFileIndexable (parent, basefile_name); if (base_indexable == null) { // GetCrawlingFileIndexable returns null if file does not need to be indexed // So basefile is up-to-date // Need to figure out id from uid manager Guid basefile_id = uid_manager.GetIdByNameAndParentId (basefile_name, parent.UniqueId); basefile_uri = GuidFu.ToUri (basefile_id); Log.Debug ("{0} is not scheduled and need not be, uri is {1}", basefile_path, basefile_uri); } else { Log.Debug ("Need to index {0}", basefile_path); // basefile needs to be indexed // FIXME: Move the task business out of handler and into FSQ.cs Scheduler.Task task; task = queryable.NewAddTask (base_indexable); // FIXME: What is the correct priority ? // If should have similar priority to the one that this xmp-indexable will be a part of task.Priority = Scheduler.Priority.Immediate; queryable.ThisScheduler.Add (task); // Get the basefile uri from the indexable basefile_uri = base_indexable.Uri; } } Log.Debug ("Adding xmp-indexable for {0} (basefile uri {1}) with uid {2}", path, basefile_uri, GuidFu.ToShortString (id)); Indexable indexable = new Indexable (IndexableType.PropertyChange, basefile_uri); // Set the timestamp of the indexable as the timestamp of the basefile // It could have also been skipped, the original Indexable.Add would anyway have it indexable.Timestamp = File.GetLastWriteTimeUtc (basefile_path); indexable.DisplayUri = UriFu.PathToFileUri (path); // If the file was somehow deleted before this point, bail out. if (! FileSystem.ExistsByDateTime (indexable.Timestamp)) { xmp_file.Close (); return null; } // Save some local states for PostAddHook, namely, path to the xmp file, path to basefile and generated uid indexable.LocalState ["XmpFilePath"] = path; indexable.LocalState ["BaseFilePath"] = basefile_path; indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString (id); // FIXME: Should also delete previous xmp properties! foreach (Property p in xmp_file.Properties) { p.IsMutable = true; indexable.AddProperty (p); } xmp_file.Close (); return indexable; }
/** * Create an indexable from a maildir message */ public Indexable MaildirMessageToIndexable (string filename, bool crawl) { //Logger.Log.Debug ("+ indexing maildir mail:" + filename); String folder = GetFolderMaildir(filename); Uri file_uri = UriFu.PathToFileUri (filename); Indexable indexable = new Indexable (file_uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.FlushBufferCache = crawl; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder)); indexable.ContentUri = file_uri; return indexable; }
/// <summary> /// 測試數值型別及參考型別 /// </summary> /// <returns></returns> public ActionResult Process6() { List<CalenderEvent> events = new List<CalenderEvent>(); List<CalenderEvent> events2 = new List<CalenderEvent>(); CalenderEvent newEvent = new CalenderEvent() { Title = "", StartTime = new DateTimeOffset(2009, 7, 14, 19, 15, 00, TimeSpan.Zero), Duration = TimeSpan.FromHours(1) }; events.Add(newEvent); //新增到指定位置 events.Insert(1, newEvent); //將同樣的List加進去 events.AddRange(events2); //使用不存任何元素的陣列模式 Indexable ix = new Indexable(); ix[42] = "Xyzzy"; //使用泛型(T) ArrayAndIndexer<int> aai = new ArrayAndIndexer<int>(); aai.TheArray[10] = 42; aai[20] = 99; //使用泛型(T)及struct ArrayAndIndexer<CanChange> aaq = new ArrayAndIndexer<CanChange>(); aaq.TheArray[10].Number = 42; aaq[20] = new CanChange { Number = 99, Name = "My Item" }; //因struct是value type,因此無法修改class裡的值 CanChange elem = aaq[20]; elem.Number = 456; // return Content(aaq[10].Number + "_" + aaq[20].Number + "_" + elem.Number); }
static protected Document RewriteDocument (Document old_secondary_doc, Indexable prop_only_indexable) { Hashtable seen_props; seen_props = new Hashtable (); Document new_doc; new_doc = new Document (); Field uri_f; uri_f = new Field ("Uri", UriFu.UriToEscapedString (prop_only_indexable.Uri), Field.Store.YES, Field.Index.NO_NORMS); new_doc.Add (uri_f); Logger.Log.Debug ("Rewriting {0}", prop_only_indexable.DisplayUri); if (prop_only_indexable.ParentUri != null) { uri_f = new Field ("ParentUri", UriFu.UriToEscapedString (prop_only_indexable.ParentUri), Field.Store.YES, Field.Index.NO_NORMS); new_doc.Add (uri_f); Logger.Log.Debug ("Parent Uri {0}", prop_only_indexable.ParentUri); } // Add the new properties to the new document. To // delete a property, set the Value to null... then it // will be added to seen_props (so the old value will // be ignored below), but AddPropertyToDocument will // return w/o doing anything. foreach (Property prop in prop_only_indexable.Properties) { seen_props [prop.Key] = prop; // Don't add properties that are empty; they // essentially mean "reset this property" if (prop.Value == String.Empty) { Log.Debug ("Resetting prop '{0}'", prop.Key); continue; } AddPropertyToDocument (prop, new_doc); Logger.Log.Debug ("New prop '{0}' = '{1}'", prop.Key, prop.Value); } // Copy the other properties from the old document to the // new one, skipping any properties that we got new values // for out of the Indexable. if (old_secondary_doc != null) { foreach (Field f in old_secondary_doc.Fields ()) { Property prop; prop = GetPropertyFromDocument (f, old_secondary_doc, false); if (prop != null && ! seen_props.Contains (prop.Key)) { Logger.Log.Debug ("Old prop '{0}' = '{1}'", prop.Key, prop.Value); AddPropertyToDocument (prop, new_doc); } } } return new_doc; }
////////////////////////////////////////////////////////////////////////////// // // Dealing with documents // static protected void BuildDocuments (Indexable indexable, out Document primary_doc, out Document secondary_doc) { primary_doc = new Document (); secondary_doc = null; Field f; // During querying, we retrieve a lucene document with only certain fields // like Uri and Timestamp and quickly check if the document is a good one // The field specified document constructor runs faster if the fields that // are asked for are located at the beginning of the document. // Hence it is better to keep "Uri" and "Timestamp" at the beginning. f = new Field ("Uri", UriFu.UriToEscapedString (indexable.Uri), Field.Store.YES, Field.Index.NO_NORMS); primary_doc.Add (f); if (indexable.ParentUri != null) { f = new Field ("ParentUri", UriFu.UriToEscapedString (indexable.ParentUri), Field.Store.YES, Field.Index.NO_NORMS); primary_doc.Add (f); } if (indexable.ValidTimestamp) { // Note that we also want to search in the // Timestamp field when we do a wildcard date // query, so that's why we also add a wildcard // field for each item here. string wildcard_field = TypeToWildcardField (PropertyType.Date); string str = StringFu.DateTimeToString (indexable.Timestamp); f = new Field ("Timestamp", str, Field.Store.YES, Field.Index.NO_NORMS); primary_doc.Add (f); f = new Field (wildcard_field, str, Field.Store.NO, Field.Index.NO_NORMS); primary_doc.Add (f); // Create an inverted timestamp so that we can // sort by timestamp at search-time. long timeval = Convert.ToInt64 (str); // Pad the inverted timestamp with zeroes for proper string comparison during termenum enumeration f = new Field ("InvertedTimestamp", (Int64.MaxValue - timeval).ToString ("d19"), Field.Store.NO, Field.Index.NO_NORMS); primary_doc.Add (f); str = StringFu.DateTimeToYearMonthString (indexable.Timestamp); f = new Field ("Timestamp(YM)", str, Field.Store.YES, Field.Index.NO_NORMS); primary_doc.Add (f); f = new Field (wildcard_field + "(YM)", str, Field.Store.NO, Field.Index.NO_NORMS); primary_doc.Add (f); str = StringFu.DateTimeToDayString (indexable.Timestamp); f = new Field ("Timestamp(D)", str, Field.Store.YES, Field.Index.NO_NORMS); primary_doc.Add (f); f = new Field (wildcard_field + "(D)", str, Field.Store.NO, Field.Index.NO_NORMS); primary_doc.Add (f); } if (indexable.NoContent) { // If there is no content, make a note of that // in a special property. Property prop; prop = Property.NewBool ("beagrep:NoContent", true); AddPropertyToDocument (prop, primary_doc); } else { // Since we might have content, add our text // readers. TextReader reader; // Add the field "Text" first // It is important that the order is preserved reader = indexable.GetTextReader (); if (reader != null) { f = new Field ("Text", reader); primary_doc.Add (f); } // FIXME: HotText is ignored for now! // Then add "HotText" //reader = indexable.GetHotTextReader (); //if (reader != null) { // f = new Field ("HotText", reader); // primary_doc.Add (f); //} } // Store the Type and MimeType in special properties if (indexable.HitType != null) { Property prop; prop = Property.NewUnsearched ("beagrep:HitType", indexable.HitType); AddPropertyToDocument (prop, primary_doc); } if (indexable.MimeType != null) { Property prop; prop = Property.NewUnsearched ("beagrep:MimeType", indexable.MimeType); AddPropertyToDocument (prop, primary_doc); } if (indexable.Source != null) { Property prop; prop = Property.NewUnsearched ("beagrep:Source", indexable.Source); AddPropertyToDocument (prop, primary_doc); } { Property prop; prop = Property.NewBool (Property.IsChildPropKey, indexable.IsChild); AddPropertyToDocument (prop, primary_doc); } // Store the other properties foreach (Property prop in indexable.Properties) { Document target_doc = primary_doc; if (prop.IsMutable) { if (secondary_doc == null) secondary_doc = CreateSecondaryDocument (indexable.Uri, indexable.ParentUri); target_doc = secondary_doc; } AddPropertyToDocument (prop, target_doc); } }
// FIXME: This need some more info private Indexable NntpMessageToIndexable (TB.NntpMessage message) { Indexable indexable; indexable = new Indexable (message.Uri); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.Timestamp = DateTime.Parse (message.GetString ("date")).ToUniversalTime (); indexable.AddProperty (Property.NewKeyword ("fixme:client", "thunderbird")); indexable.AddProperty (Property.NewUnsearched ("fixme:fullyIndexed", message.GetBool ("FullIndex"))); indexable.AddProperty (Property.NewDate ("fixme:indexDateTime", DateTime.UtcNow)); string subject = GMime.Utils.HeaderDecodePhrase (message.GetString ("subject")); indexable.AddProperty (Property.New ("dc:title", subject)); return indexable; }
/// <summary> /// Good filters should replace this by an IEnumerable that does not require generating /// all the indexables beforehand /// </summary> /// <param name="indexable"> /// A <see cref="Indexable"/> /// </param> /// <returns> /// A <see cref="System.Boolean"/> /// </returns> public virtual bool GenerateNextIndexable (out Indexable indexable) { indexable = null; if (generated_indexables.Count == 0) return false; indexable = (Indexable) generated_indexables [0]; generated_indexables.RemoveAt (0); return true; }
public void OnLuceneChildIndexableEvent (Indexable[] child_indexables) { if (ChildIndexableEvent != null) ChildIndexableEvent (child_indexables); }
protected void AddIndexable (Indexable indexable) { this.generated_indexables.Add (indexable); }
private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags) { // Don't index messages flagged as junk if (CheckFlags (flags, B_U_Camel.CamelFlags.Junk)) return null; System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid); Indexable indexable = new Indexable (uri); indexable.Timestamp = message.Date.ToUniversalTime (); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", "Local")); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name)); GMime.InternetAddressList addrs; if (this.folder_name == "Sent") { addrs = message.GetRecipients (GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name == "Sent") { addrs = message.GetRecipients (GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name != "Sent") { addrs = GMime.InternetAddressList.Parse (message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose (); } if (this.folder_name == "Sent") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); Property flag_prop = Property.NewUnsearched ("fixme:flags", flags); flag_prop.IsMutable = true; indexable.AddProperty (flag_prop); if (CheckFlags (flags, B_U_Camel.CamelFlags.Answered)) indexable.AddProperty (Property.NewFlag ("fixme:isAnswered")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Deleted)) indexable.AddProperty (Property.NewFlag ("fixme:isDeleted")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Draft)) indexable.AddProperty (Property.NewFlag ("fixme:isDraft")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Flagged)) indexable.AddProperty (Property.NewFlag ("fixme:isFlagged")); if (CheckFlags (flags, B_U_Camel.CamelFlags.Seen)) indexable.AddProperty (Property.NewFlag ("fixme:isSeen")); if (CheckFlags (flags, B_U_Camel.CamelFlags.AnsweredAll)) indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll")); indexable.SetBinaryStream (message.Stream); return indexable; }
Indexable BuildAddIndexable(Post post) { Indexable indexable = new Indexable (IndexableType.Add, new Uri(post.Href)); indexable.Timestamp = post.Time; indexable.Filtering = IndexableFiltering.Always; indexable.HitType = "Bookmark"; indexable.AddProperty (Property.New ("dc:title", post.Description)); indexable.AddProperty (Property.New ("dc:source", post.Href)); indexable.AddProperty (Property.New ("dc:keywords", post.Tag)); indexable.AddProperty (Property.New ("fixme:keywords", post.Tag)); indexable.AddProperty (Property.New ("fixme:host", new Uri (post.Href).Host)); // May want to support multiple accounts eventually indexable.AddProperty (Property.New ("delicious:account", requester.Account.Username)); AddContentToIndexable (indexable, post); return indexable; }
private Indexable current_itemToIndexable () { // sanity check if (current_item == null) return null; //Log.Debug ("Indexing " + channel_link + ":" + current_item.Link); Indexable indexable = new Indexable (new Uri (String.Format ("feed:{0};item={1}", channel_link, current_item.Link))); indexable.ParentUri = UriFu.PathToFileUri (feed_file); indexable.MimeType = "text/html"; indexable.HitType = "FeedItem"; string RFC822 = "ddd, dd MMM yyyy HH:mm:ss zzz"; DateTime date = DateTime.ParseExact(current_item.PubDate, RFC822, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AdjustToUniversal); indexable.Timestamp = date; // replace property names with Dublin Core names indexable.AddProperty (Property.New ("dc:title", current_item.Title)); indexable.AddProperty (Property.NewKeyword ("dc:identifier", current_item.Link)); indexable.AddProperty (Property.NewKeyword ("dc:source", channel_link)); indexable.AddProperty (Property.New ("dc:publisher", channel_title)); StringReader reader = new StringReader (current_item.Description); indexable.SetTextReader (reader); return indexable; }
public override Indexable GetNextIndexable () { Indexable indexable = null; // No more new messages to index, so start on the removals. if (this.delete_mode) { string uid = (string) this.deleted_list [0]; Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid); indexable = new Indexable (IndexableType.Remove, uri); this.deleted_list.RemoveAt (0); this.tracker.Remove (uid); this.delete_count++; return indexable; } B_U_Camel.MessageInfo mi = (B_U_Camel.MessageInfo) this.summary_enumerator.Current; ++this.count; if (Debug) { Logger.Log.Debug ("Constructed message {0} with uid {1}, flags {2}.", this.count, mi.uid, mi.flags); } uint flags; bool found = this.tracker.Get (mi.uid, out flags); if (! found) { // New, previously unseen message string msg_file; if (this.backend_type == ImapBackendType.Imap) msg_file = Path.Combine (summary_info.DirectoryName, mi.uid + "."); else { // This is taken from e-d-s's camel-data-cache.c. No doubt // NotZed would scream bloody murder if he saw this here. int hash = (g_str_hash (mi.uid) >> 5) & CAMEL_DATA_CACHE_MASK; string cache_path = String.Format ("cache/{0:x}/{1}", hash, mi.uid); msg_file = Path.Combine (summary_info.DirectoryName, cache_path); } indexable = this.CamelMessageToIndexable (mi, msg_file); if (Debug) Logger.Log.Debug ("Unseen message, indexable {0} null", indexable == null ? "" : "not"); if (indexable != null) ++this.indexed_count; } else if (found && flags != mi.flags) { // Previously seen message, but flags have changed. Uri uri = CamelMessageUri (mi); indexable = new Indexable (uri); indexable.Type = IndexableType.PropertyChange; Property flag_prop = Property.NewUnsearched ("fixme:flags", mi.flags); flag_prop.IsMutable = true; indexable.AddProperty (flag_prop); if (Debug) Logger.Log.Debug ("Previously seen message, flags changed: {0} -> {1}", flags, mi.flags); ++this.indexed_count; } else { if (Debug) Logger.Log.Debug ("Previously seen message, unchanged."); } this.tracker.Update (mi.uid, mi.flags); return indexable; }
protected override Uri PostAddHook (Indexable indexable, IndexerAddedReceipt receipt) { FileInfo meta_file = indexable.LocalState ["MetaFile"] as FileInfo; if (meta_file == null) return indexable.Uri; meta_file.Delete (); lock (pending_files) pending_files.Remove (indexable.ContentUri.LocalPath); return indexable.Uri; }
private void AddIndexableTask (Indexable indexable, string tag) { if (indexable == null) return; Scheduler.Task task = queryable.NewAddTask (indexable); task.Priority = Scheduler.Priority.Immediate; task.Tag = tag; queryable.ThisScheduler.Add (task); }
private Indexable CamelMessageToIndexable (B_U_Camel.MessageInfo messageInfo, string msg_file) { // Don't index messages flagged as junk if (messageInfo.IsJunk) return null; // Many properties will be set by the filter when // processing the cached data, if it's there. So // don't set a number of properties in that case. bool have_content = File.Exists (msg_file); Uri uri = CamelMessageUri (messageInfo); Indexable indexable = new Indexable (uri); indexable.Timestamp = messageInfo.SentDate; indexable.MimeType = "message/rfc822"; indexable.HitType = "MailMessage"; indexable.AddProperty (Property.NewUnsearched ("fixme:account", this.imap_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution")); if (!have_content) { indexable.AddProperty (Property.New ("dc:title", GMime.Utils.HeaderDecodePhrase (messageInfo.subject))); indexable.AddProperty (Property.NewDate ("fixme:date", messageInfo.SentDate)); } GMime.InternetAddressList addrs; addrs = GMime.InternetAddressList.Parse (messageInfo.to); foreach (GMime.InternetAddress ia in addrs) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; if (!have_content) { indexable.AddProperty (Property.NewUnsearched ("fixme:to", ia.ToString (false))); if (ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.New ("fixme:to_address", mailbox.Address)); indexable.AddProperty (Property.New ("fixme:to_name", ia.Name)); } if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } addrs.Dispose (); addrs = GMime.InternetAddressList.Parse (messageInfo.cc); foreach (GMime.InternetAddress ia in addrs) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; if (!have_content) { indexable.AddProperty (Property.NewUnsearched ("fixme:cc", ia.ToString (false))); if (ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.New ("fixme:cc_address", mailbox.Address)); indexable.AddProperty (Property.New ("fixme:cc_name", ia.Name)); } if (this.folder_name == "Sent" && ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", mailbox.Address)); } addrs.Dispose (); addrs = GMime.InternetAddressList.Parse (messageInfo.from); foreach (GMime.InternetAddress ia in addrs) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; if (!have_content) { indexable.AddProperty (Property.NewUnsearched ("fixme:from", ia.ToString (false))); if (ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.New ("fixme:from_address", mailbox.Address)); indexable.AddProperty (Property.New ("fixme:from_name", ia.Name)); } if (this.folder_name != "Sent" && ia is GMime.InternetAddressMailbox) indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", mailbox.Address)); } addrs.Dispose (); indexable.AddProperty (Property.NewKeyword ("fixme:mlist", messageInfo.mlist)); Property flag_prop = Property.NewUnsearched ("fixme:flags", messageInfo.flags); flag_prop.IsMutable = true; indexable.AddProperty (flag_prop); if (this.folder_name == "Sent") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); if (messageInfo.IsAnswered) indexable.AddProperty (Property.NewFlag ("fixme:isAnswered")); if (messageInfo.IsDeleted) indexable.AddProperty (Property.NewFlag ("fixme:isDeleted")); if (messageInfo.IsDraft) indexable.AddProperty (Property.NewFlag ("fixme:isDraft")); if (messageInfo.IsFlagged) indexable.AddProperty (Property.NewFlag ("fixme:isFlagged")); if (messageInfo.IsSeen) indexable.AddProperty (Property.NewFlag ("fixme:isSeen")); if (messageInfo.HasAttachments && !have_content) indexable.AddProperty (Property.NewFlag ("fixme:hasAttachments")); if (messageInfo.IsAnsweredAll) indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll")); if (have_content) indexable.ContentUri = UriFu.PathToFileUri (msg_file); else indexable.NoContent = true; return indexable; }
/** * Create an indexable from an mbox message * Most of the code here is from Evo backend */ public Indexable MessageToIndexable (string file_name, System.Uri uri, GMime.Message message, string folder_name) { //Logger.Log.Debug ("Indexing " + uri + " in folder " + folder_name); Indexable indexable = new Indexable (uri); // set parent uri to the filename so that when an mbox file // is deleted, all the messages in that file can be deleted indexable.ParentUri = UriFu.PathToFileUri (file_name); indexable.Timestamp = message.Date.ToUniversalTime (); indexable.HitType = "MailMessage"; indexable.MimeType = "message/rfc822"; indexable.CacheContent = true; indexable.AddProperty (Property.NewUnsearched ("fixme:client", "kmail")); indexable.AddProperty (Property.NewUnsearched ("fixme:account", account_name)); indexable.AddProperty (Property.NewUnsearched ("fixme:folder", folder_name)); GMime.InternetAddressList addrs; if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients (GMime.RecipientType.To); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (folder_name == Queryable.SentMailFolderName) { addrs = message.GetRecipients (GMime.RecipientType.Cc); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", mailbox.Address)); } } addrs.Dispose (); } if (folder_name != Queryable.SentMailFolderName) { addrs = GMime.InternetAddressList.Parse (message.Sender); foreach (GMime.InternetAddress ia in addrs) { if (ia is GMime.InternetAddressMailbox) { GMime.InternetAddressMailbox mailbox = ia as GMime.InternetAddressMailbox; indexable.AddProperty (Property.NewKeyword ("fixme:gotFrom", mailbox.Address)); } } addrs.Dispose (); } if (folder_name == Queryable.SentMailFolderName) indexable.AddProperty (Property.NewFlag ("fixme:isSent")); else { string kmail_msg_sent = message.GetHeader ("X-KMail-Link-Type"); if (kmail_msg_sent == "reply") indexable.AddProperty (Property.NewFlag ("fixme:isSent")); } // no need to store date again, use the issent flag to determine if the date is sentdate or not #if false if (folder_name == Queryable.SentMailFolderName) indexable.AddProperty (Property.NewDate ("fixme:sentdate", message.Date.ToUniversalTime ())); else indexable.AddProperty (Property.NewDate ("fixme:received", message.Date.ToUniversalTime ())); #endif indexable.SetBinaryStream (message.Stream); return indexable; }
private Indexable FileToIndexable (FileInfo data_file) { FileInfo meta_file = new FileInfo (Path.Combine (data_file.DirectoryName, "." + data_file.Name)); FileStream meta_stream; try { meta_stream = meta_file.Open (FileMode.Open, FileAccess.Read, FileShare.Read); } catch (FileNotFoundException) { // The meta file disappeared before we could // open it. return null; } StreamReader reader = new StreamReader (meta_stream); // First line of the file is a URI string line = reader.ReadLine (); Uri uri; try { uri = new Uri (line); } catch (Exception e) { Logger.Log.Warn (e, "IndexingService: Unable to parse URI in {0}:", meta_file.FullName); meta_stream.Close (); return null; } Indexable indexable = new Indexable (uri); indexable.Timestamp = data_file.LastWriteTimeUtc; indexable.ContentUri = UriFu.PathToFileUri (data_file.FullName); indexable.DeleteContent = true; indexable.AddProperty( Property.New("fixme:host",uri.Host)); // Second line is the hit type line = reader.ReadLine (); if (line == null) { Logger.Log.Warn ("IndexingService: EOF reached trying to read hit type from {0}", meta_file.FullName); meta_stream.Close (); return null; } else if (line != String.Empty) indexable.HitType = line; // Third line is the mime type line = reader.ReadLine (); if (line == null) { Logger.Log.Warn ("IndexingService: EOF reached trying to read mime type from {0}", meta_file.FullName); meta_stream.Close (); return null; } else if (line != String.Empty) indexable.MimeType = line; // Following lines are properties in "t:key=value" format do { line = reader.ReadLine (); if (line != null && line != String.Empty) { bool keyword = false; if (line[0] == 'k') keyword = true; else if (line[0] != 't') { Logger.Log.Warn ("IndexingService: Unknown property type: '{0}'", line[0]); continue; } int i = line.IndexOf ('='); if (i == -1) { Logger.Log.Warn ("IndexingService: Unknown property line: '{0}'", line); continue; } // FIXME: We should probably handle date types if (keyword) { indexable.AddProperty (Property.NewUnsearched (line.Substring (2, i - 2), line.Substring (i + 1))); } else { indexable.AddProperty (Property.New (line.Substring (2, i - 2), line.Substring (i + 1))); } } } while (line != null); indexable.LocalState ["MetaFile"] = meta_file; // Ok, we're finished with the meta file. It will be // deleted in PostAddHook (). meta_stream.Close (); return indexable; }