public LogByteSizeMergePolicy(IndexWriter writer):base(writer) { minMergeSize = (long) (DEFAULT_MIN_MERGE_MB * 1024 * 1024); //mgarski - the line below causes an overflow in .NET, resulting in a negative number... //maxMergeSize = (long) (DEFAULT_MAX_MERGE_MB * 1024 * 1024); maxMergeSize = DEFAULT_MAX_MERGE_MB; }
public LogDocMergePolicy(IndexWriter writer):base(writer) { minMergeSize = DEFAULT_MIN_MERGE_DOCS; // maxMergeSize is never used by LogDocMergePolicy; set // it to Long.MAX_VALUE to disable it maxMergeSize = System.Int64.MaxValue; }
/// <summary>Just do the merges in sequence. We do this /// "synchronized" so that even if the application is using /// multiple threads, only one merge may run at a time. /// </summary> public override void Merge(IndexWriter writer) { lock (this) { while (true) { MergePolicy.OneMerge merge = writer.GetNextMerge(); if (merge == null) break; writer.Merge(merge); } } }
public override void PopulateSearchableIndex (IndexWriter writer) { foreach (Node n in Tree.Nodes) { XmlSerializer reader = new XmlSerializer (typeof (ErrorDocumentation)); ErrorDocumentation d = (ErrorDocumentation)reader.Deserialize (GetHelpStream (n.Element.Substring (6))); SearchableDocument doc = new SearchableDocument (); doc.title = d.ErrorName; doc.url = n.Element; doc.text = d.Details != null ? d.Details.ToString () : string.Empty; doc.examples = d.Examples.Cast<string> ().Aggregate ((e1, e2) => e1 + Environment.NewLine + e2); doc.hottext = d.ErrorName; writer.AddDocument (doc.LuceneDoc); } }
public static void MakeSearchIndex (RootTree root) { // Loads the RootTree Console.WriteLine ("Loading the monodoc tree..."); if (root == null) return; string dir = Path.Combine (root.basedir, "search_index"); IndexWriter writer; //try to create the dir to store the index try { if (!Directory.Exists (dir)) Directory.CreateDirectory (dir); writer = new IndexWriter(Mono.Lucene.Net.Store.FSDirectory.GetDirectory(dir, true), new StandardAnalyzer(), true); } catch (UnauthorizedAccessException) { //try in the .config directory try { dir = Path.Combine (SettingsHandler.Path, "search_index"); if (!Directory.Exists (dir)) Directory.CreateDirectory (dir); writer = new IndexWriter(Mono.Lucene.Net.Store.FSDirectory.GetDirectory(dir, true), new StandardAnalyzer(), true); } catch (UnauthorizedAccessException) { Console.WriteLine ("You don't have permissions to write on " + dir); return; } } //Collect all the documents Console.WriteLine ("Collecting and adding documents..."); foreach (HelpSource hs in root.HelpSources) hs.PopulateSearchableIndex (writer); //Optimize and close Console.WriteLine ("Closing..."); writer.Optimize(); writer.Close(); }
public override void PopulateSearchableIndex (IndexWriter writer) { foreach (Node n in Tree.RootNode.Nodes) AddDocuments (writer, n); }
public LogMergePolicy(IndexWriter writer):base(writer) { }
public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge) { InitBlock(enclosingInstance); this.writer = writer; this.startMerge = startMerge; }
public ReaderPool(IndexWriter enclosingInstance) { InitBlock(enclosingInstance); }
/// <summary> Close this index, writing all pending changes to disk. /// /// </summary> /// <throws> IllegalStateException if the index has been closed before already </throws> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public virtual void Close() { lock (directory) { if (!open) throw new System.SystemException("Index is closed already"); if (indexWriter != null) { indexWriter.Close(); indexWriter = null; } else if (indexReader != null) { indexReader.Close(); indexReader = null; } open = false; if (closeDir) { directory.Close(); } closeDir = false; } }
/// <summary> Make sure all changes are written to disk.</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> LockObtainFailedException if another writer </throws> /// <summary> has this index open (<code>write.lock</code> could not /// be obtained) /// </summary> /// <throws> IOException if there is a low-level IO error </throws> public virtual void Flush() { lock (directory) { AssureOpen(); if (indexWriter != null) { indexWriter.Close(); indexWriter = null; CreateIndexWriter(); } else { indexReader.Close(); indexReader = null; CreateIndexReader(); } } }
/// <summary> Close the IndexWriter and open an IndexReader.</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> protected internal virtual void CreateIndexReader() { if (indexReader == null) { if (indexWriter != null) { indexWriter.Close(); indexWriter = null; } indexReader = IndexReader.Open(directory); } }
/// <summary> Close the IndexReader and open an IndexWriter.</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> LockObtainFailedException if another writer </throws> /// <summary> has this index open (<code>write.lock</code> could not /// be obtained) /// </summary> /// <throws> IOException if there is a low-level IO error </throws> protected internal virtual void CreateIndexWriter() { if (indexWriter == null) { if (indexReader != null) { indexReader.Close(); indexReader = null; } indexWriter = new IndexWriter(directory, analyzer, false, new IndexWriter.MaxFieldLength(maxFieldLength)); // IndexModifier cannot use ConcurrentMergeScheduler // because it synchronizes on the directory which can // cause deadlock indexWriter.SetMergeScheduler(new SerialMergeScheduler()); indexWriter.SetInfoStream(infoStream); indexWriter.SetUseCompoundFile(useCompoundFile); if (maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH) indexWriter.SetMaxBufferedDocs(maxBufferedDocs); indexWriter.SetMergeFactor(mergeFactor); } }
/// <summary> Initialize an IndexWriter.</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> LockObtainFailedException if another writer </throws> /// <summary> has this index open (<code>write.lock</code> could not /// be obtained) /// </summary> /// <throws> IOException if there is a low-level IO error </throws> protected internal virtual void Init(Directory directory, Analyzer analyzer, bool create) { this.directory = directory; lock (this.directory) { this.analyzer = analyzer; indexWriter = new IndexWriter(directory, analyzer, create, IndexWriter.MaxFieldLength.LIMITED); open = true; } }
public override void Merge(IndexWriter writer) { // TODO: enable this once we are on JRE 1.5 // assert !Thread.holdsLock(writer); this.writer = writer; InitMergeThreadPriority(); dir = writer.GetDirectory(); // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. if (Verbose()) { Message("now merge"); Message(" index: " + writer.SegString()); } // Iterate, pulling from the IndexWriter's queue of // pending merges, until it's empty: while (true) { // TODO: we could be careful about which merges to do in // the BG (eg maybe the "biggest" ones) vs FG, which // merges to do first (the easiest ones?), etc. MergePolicy.OneMerge merge = writer.GetNextMerge(); if (merge == null) { if (Verbose()) Message(" no more merges pending; now return"); return ; } // We do this w/ the primary thread to keep // deterministic assignment of segment names writer.MergeInit(merge); bool success = false; try { lock (this) { MergeThread merger; while (MergeThreadCount(true) >= maxThreadCount) { if (Verbose()) Message(" too many merge threads running; stalling..."); try { System.Threading.Monitor.Wait(this); } catch (System.Threading.ThreadInterruptedException ie) { // In 3.0 we will change this to throw // InterruptedException instead SupportClass.ThreadClass.Current().Interrupt(); throw new System.SystemException(ie.Message, ie); } } if (Verbose()) Message(" consider merge " + merge.SegString(dir)); // OK to spawn a new merge thread to handle this // merge: merger = GetMergeThread(writer, merge); mergeThreads.Add(merger); if (Verbose()) Message(" launch new thread [" + merger.Name + "]"); merger.Start(); success = true; } } finally { if (!success) { writer.MergeFinish(merge); } } } }
/// <summary>Create and return a new MergeThread </summary> protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { lock (this) { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority(mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + mergeThreadCount++; return thread; } }
private void InitBlock(IndexWriter enclosingInstance) { this.enclosingInstance = enclosingInstance; }
internal ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor):base(writer, infos, termInfosIndexDivisor) { }
public MergePolicy(IndexWriter writer) { this.writer = writer; }
internal DocumentsWriter(Directory directory, IndexWriter writer, IndexingChain indexingChain) { InitBlock(); this.directory = directory; this.writer = writer; this.similarity = writer.GetSimilarity(); flushedDocCount = writer.MaxDoc(); consumer = indexingChain.GetChain(this); if (consumer is DocFieldProcessor) { docFieldProcessor = (DocFieldProcessor) consumer; } }
/// <summary>Run the merges provided by {@link IndexWriter#GetNextMerge()}. </summary> public abstract void Merge(IndexWriter writer);
void AddDocuments (IndexWriter writer, Node node) { string url = node.PublicUrl; Stream file_stream = GetHelpStream (url.Substring (9)); if (file_stream == null) //Error return; XmlDocument xdoc = new XmlDocument (); xdoc.Load (new XmlTextReader (file_stream)); //Obtain the title XmlNode nelem = xdoc.DocumentElement; string title = nelem.Attributes["number"].Value + ": " + nelem.Attributes["title"].Value; //Obtain the text StringBuilder s = new StringBuilder (); GetTextNode (nelem, s); string text = s.ToString (); //Obtain the examples StringBuilder s2 = new StringBuilder (); GetExamples (nelem, s2); string examples = s2.ToString (); //Write to the Lucene Index all the parts SearchableDocument doc = new SearchableDocument (); doc.title = title; doc.hottext = title.Substring (title.IndexOf (':')); doc.url = url; doc.text = text; doc.examples = examples; writer.AddDocument (doc.LuceneDoc); if (node.IsLeaf) return; foreach (Node n in node.Nodes) AddDocuments (writer, n); }
// Used by near real-time search internal DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) { this.directory = writer.GetDirectory(); this.readOnly = true; segmentInfos = infos; segmentInfosStart = (SegmentInfos) infos.Clone(); this.termInfosIndexDivisor = termInfosIndexDivisor; if (!readOnly) { // We assume that this segments_N was previously // properly sync'd: SupportClass.CollectionsHelper.AddAllIfNotContains(synced, infos.Files(directory, true)); } // IndexWriter synchronizes externally before calling // us, which ensures infos will not change; so there's // no need to process segments in reverse order int numSegments = infos.Count; SegmentReader[] readers = new SegmentReader[numSegments]; Directory dir = writer.GetDirectory(); int upto = 0; for (int i = 0; i < numSegments; i++) { bool success = false; try { SegmentInfo info = infos.Info(i); if (info.dir == dir) { readers[upto++] = writer.readerPool.GetReadOnlyClone(info, true, termInfosIndexDivisor); } success = true; } finally { if (!success) { // Close all readers we had opened: for (upto--; upto >= 0; upto--) { try { readers[upto].Close(); } catch (System.Exception ignore) { // keep going - we want to clean up as much as possible } } } } } this.writer = writer; if (upto < readers.Length) { // This means some segments were in a foreign Directory SegmentReader[] newReaders = new SegmentReader[upto]; Array.Copy(readers, 0, newReaders, 0, upto); readers = newReaders; } Initialize(readers); }
// // Create different Documents for adding to Lucene search index // The default action is do nothing. Subclasses should add the docs // public virtual void PopulateSearchableIndex (IndexWriter writer) { return; }
// // Create list of documents for searching // public override void PopulateSearchableIndex (IndexWriter writer) { StringBuilder text; foreach (Node ns_node in Tree.Nodes) { Message (TraceLevel.Info, "\tNamespace: {0} ({1})", ns_node.Caption, ns_node.Nodes.Count); foreach (Node type_node in ns_node.Nodes) { string typename = type_node.Caption.Substring (0, type_node.Caption.IndexOf (' ')); string full = ns_node.Caption + "." + typename; string doc_tag = GetKindFromCaption (type_node.Caption); string url = "T:" + full; string rest; XmlDocument xdoc = GetXmlFromUrl (type_node.URL, out rest); if (xdoc == null) continue; // // For classes, structures or interfaces add a doc for the overview and // add a doc for every constructor, method, event, ... // if (doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface"){ // Adds a doc for every overview of every type SearchableDocument doc = new SearchableDocument (); doc.title = type_node.Caption; doc.hottext = typename; doc.url = url; XmlNode node_sel = xdoc.SelectSingleNode ("/Type/Docs"); text = new StringBuilder (); GetTextFromNode (node_sel, text); doc.text = text.ToString (); text = new StringBuilder (); GetExamples (node_sel, text); doc.examples = text.ToString (); writer.AddDocument (doc.LuceneDoc); //Add docs for contructors, methods, etc. foreach (Node c in type_node.Nodes) { // c = Constructors || Fields || Events || Properties || Methods || Operators if (c.Element == "*") continue; int i = 1; const float innerTypeBoost = 0.2f; foreach (Node nc in c.Nodes) { // Disable constructors indexing as it's often "polluting" search queries // because it has the same hottext than standard types if (c.Caption == "Constructors") continue; //xpath to the docs xml node string xpath; if (c.Caption == "Constructors") xpath = String.Format ("/Type/Members/Member[{0}]/Docs", i++); else if (c.Caption == "Operators") xpath = String.Format ("/Type/Members/Member[@MemberName='op_{0}']/Docs", nc.Caption); else xpath = String.Format ("/Type/Members/Member[@MemberName='{0}']/Docs", nc.Caption); //construct url of the form M:Array.Sort string urlnc; if (c.Caption == "Constructors") urlnc = String.Format ("{0}:{1}.{2}", c.Caption[0], ns_node.Caption, nc.Caption); else urlnc = String.Format ("{0}:{1}.{2}.{3}", c.Caption[0], ns_node.Caption, typename, nc.Caption); //create the doc SearchableDocument doc_nod = new SearchableDocument (); doc_nod.title = LargeName (nc); //dont add the parameters to the hottext int ppos = nc.Caption.IndexOf ('('); if (ppos != -1) doc_nod.hottext = nc.Caption.Substring (0, ppos); else doc_nod.hottext = nc.Caption; doc_nod.url = urlnc; XmlNode xmln = xdoc.SelectSingleNode (xpath); if (xmln == null) { Error ("Problem: {0}, with xpath: {1}", urlnc, xpath); continue; } text = new StringBuilder (); GetTextFromNode (xmln, text); doc_nod.text = text.ToString (); text = new StringBuilder (); GetExamples (xmln, text); doc_nod.examples = text.ToString (); Document lucene_doc = doc_nod.LuceneDoc; lucene_doc.SetBoost (innerTypeBoost); writer.AddDocument (lucene_doc); } } // // Enumerations: add the enumeration values // } else if (doc_tag == "Enumeration"){ XmlNodeList members = xdoc.SelectNodes ("/Type/Members/Member"); if (members == null) continue; text = new StringBuilder (); foreach (XmlNode member_node in members) { string enum_value = member_node.Attributes ["MemberName"].InnerText; text.Append (enum_value); text.Append (" "); GetTextFromNode (member_node["Docs"], text); text.Append ("\n"); } SearchableDocument doc = new SearchableDocument (); text = new StringBuilder (); GetExamples (xdoc.SelectSingleNode ("/Type/Docs"), text); doc.examples = text.ToString (); doc.title = type_node.Caption; doc.hottext = xdoc.DocumentElement.Attributes["Name"].Value; doc.url = url; doc.text = text.ToString(); writer.AddDocument (doc.LuceneDoc); // // Add delegates // } else if (doc_tag == "Delegate"){ SearchableDocument doc = new SearchableDocument (); doc.title = type_node.Caption; doc.hottext = xdoc.DocumentElement.Attributes["Name"].Value; doc.url = url; XmlNode node_sel = xdoc.SelectSingleNode ("/Type/Docs"); text = new StringBuilder (); GetTextFromNode (node_sel, text); doc.text = text.ToString(); text = new StringBuilder (); GetExamples (node_sel, text); doc.examples = text.ToString(); writer.AddDocument (doc.LuceneDoc); } } } }
// // Create list of documents for searching // public override void PopulateSearchableIndex (IndexWriter writer) { StringBuilder text; foreach (Node ns_node in Tree.Nodes) { Message (TraceLevel.Info, "\tNamespace: {0} ({1})", ns_node.Caption, ns_node.Nodes.Count); foreach (Node type_node in ns_node.Nodes) { string typename = type_node.Caption.Substring (0, type_node.Caption.IndexOf (' ')); string full = ns_node.Caption + "." + typename; string doc_tag = GetKindFromCaption (type_node.Caption); string url = "T:" + full; string rest; XmlDocument xdoc = GetXmlFromUrl (type_node.URL, out rest); if (xdoc == null) continue; // // For classes, structures or interfaces add a doc for the overview and // add a doc for every constructor, method, event, ... // if (doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface"){ // Adds a doc for every overview of every type SearchableDocument doc = new SearchableDocument (); doc.title = type_node.Caption; doc.hottext = typename; doc.url = url; doc.fulltitle = full; XmlNode node_sel = xdoc.SelectSingleNode ("/Type/Docs"); text = new StringBuilder (); GetTextFromNode (node_sel, text); doc.text = text.ToString (); text = new StringBuilder (); GetExamples (node_sel, text); doc.examples = text.ToString (); writer.AddDocument (doc.LuceneDoc); var exportParsable = doc_tag == "Class" && (ns_node.Caption.StartsWith ("MonoTouch") || ns_node.Caption.StartsWith ("MonoMac")); //Add docs for contructors, methods, etc. foreach (Node c in type_node.Nodes) { // c = Constructors || Fields || Events || Properties || Methods || Operators if (c.Element == "*") continue; int i = 1; const float innerTypeBoost = 0.2f; foreach (Node nc in c.Nodes) { // Disable constructors indexing as it's often "polluting" search queries // because it has the same hottext than standard types if (c.Caption == "Constructors") continue; //xpath to the docs xml node string xpath; if (c.Caption == "Constructors") xpath = String.Format ("/Type/Members/Member[{0}]/Docs", i++); else if (c.Caption == "Operators") xpath = String.Format ("/Type/Members/Member[@MemberName='op_{0}']/Docs", nc.Caption); else xpath = String.Format ("/Type/Members/Member[@MemberName='{0}']/Docs", nc.Caption); //construct url of the form M:Array.Sort string urlnc; if (c.Caption == "Constructors") urlnc = String.Format ("{0}:{1}.{2}", c.Caption[0], ns_node.Caption, nc.Caption); else urlnc = String.Format ("{0}:{1}.{2}.{3}", c.Caption[0], ns_node.Caption, typename, nc.Caption); //create the doc SearchableDocument doc_nod = new SearchableDocument (); doc_nod.title = LargeName (nc); switch (c.Caption[0]) { case 'M': doc_nod.title += " Method"; break; case 'P': doc_nod.title += " Property"; break; case 'E': doc_nod.title += " Event"; break; case 'O': doc_nod.title += " Operator"; break; default: break; } doc_nod.fulltitle = string.Format ("{0}.{1}::{2}", ns_node.Caption, typename, nc.Caption); //dont add the parameters to the hottext int ppos = nc.Caption.IndexOf ('('); if (ppos != -1) doc_nod.hottext = nc.Caption.Substring (0, ppos); else doc_nod.hottext = nc.Caption; doc_nod.url = urlnc; XmlNode xmln = xdoc.SelectSingleNode (xpath); if (xmln == null) { Error ("Problem: {0}, with xpath: {1}", urlnc, xpath); continue; } text = new StringBuilder (); GetTextFromNode (xmln, text); doc_nod.text = text.ToString (); text = new StringBuilder (); GetExamples (xmln, text); doc_nod.examples = text.ToString (); Document lucene_doc = doc_nod.LuceneDoc; lucene_doc.SetBoost (innerTypeBoost); writer.AddDocument (lucene_doc); // MonoTouch/Monomac specific parsing of [Export] attributes if (exportParsable) { try { var exports = xdoc.SelectNodes (string.Format ("/Type/Members/Member[@MemberName='{0}']/Attributes/Attribute/AttributeName[contains(text(), 'Foundation.Export')]", nc.Caption)); foreach (XmlNode exportNode in exports) { var inner = exportNode.InnerText; var parts = inner.Split ('"'); if (parts.Length != 3) { Console.WriteLine ("Export attribute not found or not usable in {0}", inner); continue; } var export = parts[1]; var export_node = new SearchableDocument (); export_node.title = export + " Export"; export_node.fulltitle = string.Format ("{0}.{1}::{2}", ns_node.Caption, typename, export); export_node.url = urlnc; export_node.hottext = export + ":"; export_node.text = string.Empty; export_node.examples = string.Empty; lucene_doc = export_node.LuceneDoc; lucene_doc.SetBoost (innerTypeBoost); writer.AddDocument (lucene_doc); } } catch (Exception e){ Console.WriteLine ("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e); } } } } // // Enumerations: add the enumeration values // } else if (doc_tag == "Enumeration"){ XmlNodeList members = xdoc.SelectNodes ("/Type/Members/Member"); if (members == null) continue; text = new StringBuilder (); foreach (XmlNode member_node in members) { string enum_value = member_node.Attributes ["MemberName"].InnerText; text.Append (enum_value); text.Append (" "); GetTextFromNode (member_node["Docs"], text); text.Append ("\n"); } SearchableDocument doc = new SearchableDocument (); text = new StringBuilder (); GetExamples (xdoc.SelectSingleNode ("/Type/Docs"), text); doc.examples = text.ToString (); doc.title = type_node.Caption; doc.hottext = xdoc.DocumentElement.Attributes["Name"].Value; doc.fulltitle = full; doc.url = url; doc.text = text.ToString(); writer.AddDocument (doc.LuceneDoc); // // Add delegates // } else if (doc_tag == "Delegate"){ SearchableDocument doc = new SearchableDocument (); doc.title = type_node.Caption; doc.hottext = xdoc.DocumentElement.Attributes["Name"].Value; doc.fulltitle = full; doc.url = url; XmlNode node_sel = xdoc.SelectSingleNode ("/Type/Docs"); text = new StringBuilder (); GetTextFromNode (node_sel, text); doc.text = text.ToString(); text = new StringBuilder (); GetExamples (node_sel, text); doc.examples = text.ToString(); writer.AddDocument (doc.LuceneDoc); } } } }
internal SegmentMerger(IndexWriter writer, System.String name, MergePolicy.OneMerge merge) { InitBlock(); directory = writer.GetDirectory(); segment = name; if (merge != null) { checkAbort = new CheckAbort(merge, directory); } else { checkAbort = new AnonymousClassCheckAbort1(this, null, null); } termIndexInterval = writer.GetTermIndexInterval(); }