/// <summary> /// Runs the indexing process. /// </summary> /// <param name="users">The users.</param> /// <param name="created">if set to <c>true</c> [created].</param> public void CreateOrUpdate(IList<User> users, bool created) { var modifier = new IndexModifier( Setting.UserSearchIndexPath.Value, new StandardAnalyzer(), (!created) ? true : false ); foreach (var user in users) { if (created) { modifier.DeleteDocuments(new Term("id", user.Id.ToString())); } var document = new Document(); UserToDocument(user, document); modifier.AddDocument(document); } modifier.Optimize(); modifier.Close(); }
public void threadproc_update(object obj) { lock (locker) // If a thread is updating the index, no other thread should be doing anything with it. { try { if (searcher != null) { try { searcher.Close(); } catch (Exception e) { } searcher = null; } Lucene.Net.Index.IndexModifier modifier = new Lucene.Net.Index.IndexModifier(DBNLConfigurationManager.LuceneElement.IndexingFolder, analyzer, false); // same as build, but uses "modifier" instead of write. // uses additional "where" clause for bugid int id = (int)obj; modifier.DeleteDocuments(new Lucene.Net.Index.Term("id", Convert.ToString(id))); var item = new ContentService().GetContentById(id); modifier.AddDocument(create_doc( item.ContentId, item.Content1)); modifier.Flush(); modifier.Close(); } catch (Exception e) { } } }
public virtual void TestIndex() { Directory ramDir = new RAMDirectory(); IndexModifier i = new IndexModifier(ramDir, new StandardAnalyzer(), true); i.AddDocument(GetDoc()); Assert.AreEqual(1, i.DocCount()); i.Flush(); i.AddDocument(GetDoc(), new SimpleAnalyzer()); Assert.AreEqual(2, i.DocCount()); i.Optimize(); Assert.AreEqual(2, i.DocCount()); i.Flush(); i.DeleteDocument(0); Assert.AreEqual(1, i.DocCount()); i.Flush(); Assert.AreEqual(1, i.DocCount()); i.AddDocument(GetDoc()); i.AddDocument(GetDoc()); i.Flush(); // depend on merge policy - Assert.AreEqual(3, i.docCount()); i.DeleteDocuments(allDocTerm); Assert.AreEqual(0, i.DocCount()); i.Optimize(); Assert.AreEqual(0, i.DocCount()); // Lucene defaults: Assert.IsNull(i.GetInfoStream()); Assert.IsTrue(i.GetUseCompoundFile()); Assert.AreEqual(IndexWriter.DISABLE_AUTO_FLUSH, i.GetMaxBufferedDocs()); Assert.AreEqual(10000, i.GetMaxFieldLength()); Assert.AreEqual(10, i.GetMergeFactor()); // test setting properties: i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Flush(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); // test setting properties when internally the reader is opened: i.DeleteDocuments(allDocTerm); i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Optimize(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); i.Close(); try { i.DocCount(); Assert.Fail(); } catch (System.SystemException e) { // expected exception } }
public virtual void TestIndex() { Directory ramDir = new RAMDirectory(); IndexModifier i = new IndexModifier(ramDir, new StandardAnalyzer(), true); i.AddDocument(GetDoc()); Assert.AreEqual(1, i.DocCount()); i.Flush(); i.AddDocument(GetDoc(), new SimpleAnalyzer()); Assert.AreEqual(2, i.DocCount()); i.Optimize(); Assert.AreEqual(2, i.DocCount()); i.Flush(); i.DeleteDocument(0); Assert.AreEqual(1, i.DocCount()); i.Flush(); Assert.AreEqual(1, i.DocCount()); i.AddDocument(GetDoc()); i.AddDocument(GetDoc()); i.Flush(); // depend on merge policy - Assert.AreEqual(3, i.docCount()); i.DeleteDocuments(allDocTerm); Assert.AreEqual(0, i.DocCount()); i.Optimize(); Assert.AreEqual(0, i.DocCount()); // Lucene defaults: Assert.IsNull(i.GetInfoStream()); Assert.IsTrue(i.GetUseCompoundFile()); Assert.AreEqual(IndexWriter.DISABLE_AUTO_FLUSH, i.GetMaxBufferedDocs()); Assert.AreEqual(10000, i.GetMaxFieldLength()); Assert.AreEqual(10, i.GetMergeFactor()); // test setting properties: i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Flush(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); // test setting properties when internally the reader is opened: i.DeleteDocuments(allDocTerm); i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Optimize(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); i.Close(); try { i.DocCount(); Assert.Fail(); } catch (System.SystemException e) { // expected exception } }
override public void Run() { long endTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) + 1000 * TEST_SECONDS; try { while ((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) < endTime) { int rand = random.Next(101); if (rand < 5) { index.Optimize(); } else if (rand < 60) { Document doc = GetDocument(); index.AddDocument(doc); idStack.Add(doc.Get("id")); added++; } else { // we just delete the last document added and remove it // from the id stack so that it won't be removed twice: System.String delId = null; try { delId = idStack[idStack.Count - 1] as System.String; idStack.RemoveAt(idStack.Count - 1); } catch (System.ArgumentOutOfRangeException e) { continue; } Term delTerm = new Term("id", System.Int32.Parse(delId).ToString()); int delCount = index.DeleteDocuments(delTerm); if (delCount != 1) { throw new System.SystemException("Internal error: " + threadNumber + " deleted " + delCount + " documents, term=" + delTerm); } deleted++; } if (maxWait > 0) { rand = random.Next(maxWait); //System.out.println("waiting " + rand + "ms"); try { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * rand)); } catch (System.Threading.ThreadInterruptedException ie) { SupportClass.ThreadClass.Current().Interrupt(); throw new System.SystemException("", ie); } } } } catch (System.IO.IOException e) { throw new System.SystemException("", e); } }
/// <summary> /// Loops thro a list of stories and adds them to the index. If the crawl is an incremental /// update then first the story is removed then added again. /// </summary> /// <param name="modifier">IndexModifer used to update the index</param> /// <param name="isIncrementalCrawl">bool indicating if the stories should /// be removed from the existing index before being added again.</param> /// <param name="stories">StoryCollection containing the stories to add/update /// in the index</param> private void AddStoriesToIndex(IndexModifier modifier, bool isIncrementalCrawl, StoryCollection stories) { if (isIncrementalCrawl) { //remove the stories from the index that have been updated Log.DebugFormat("Updating index, removing {0} stories", stories.Count); foreach (Story s in stories) { Term existingItem = new Term("id", s.StoryID.ToString()); int j = modifier.DeleteDocuments(existingItem); } } //add the new documents Log.DebugFormat("Adding batch of {0} stories to the index", stories.Count); foreach (Story story in stories) { //spam stories shouldnt be added to the index if (story.IsSpam) continue; Document doc = new Document(); doc.Add(new Field("url", story.Url, Field.Store.NO, Field.Index.TOKENIZED)); doc.Add(new Field("title", story.Title, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES)); doc.Add(new Field("description", story.Description, Field.Store.NO, Field.Index.TOKENIZED)); doc.Add(new Field("users", GetUserWhoKickedSearchString(story), Field.Store.NO, Field.Index.TOKENIZED)); doc.Add(new Field("category", story.Category.Name, Field.Store.NO, Field.Index.TOKENIZED)); doc.Add(new Field("tags", GetStoryTags(story), Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES)); doc.Add(new Field("id", story.StoryID.ToString(), Field.Store.YES, Field.Index.UN_TOKENIZED)); doc.Add(new Field("kickCount", story.KickCount.ToString(), Field.Store.NO, Field.Index.UN_TOKENIZED)); doc.Add(new Field("dateAdded", DateField.DateToString(story.CreatedOn), Field.Store.NO, Field.Index.UN_TOKENIZED)); modifier.AddDocument(doc); Log.DebugFormat("StoryId {0} added to index", story.StoryID); } }
public void CreateIndexIfNeeded(List<MsDocEntryPoint> documentation, DateTime? lastModified) { lock (SynchLock) { if (IsIndexingNeeded(lastModified) || !IsOperational) { try { //Drop all index if (searcher != null) searcher.Close(); if (_directory != null) _directory.Close(); //Delete dir Directory.Delete(_indexDirectory, true); _directory = FSDirectory.GetDirectory(_indexDirectory, true); //Reopen directory var indexModifier = new IndexModifier(_directory, new StandardAnalyzer(), true); foreach (var entryPoint in documentation) { var pointDoc = new Document(); //Id keys pointDoc.Add(new Field("point", entryPoint.Name, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES)); if (!string.IsNullOrEmpty(entryPoint.Summary)) pointDoc.Add(new Field("summary", entryPoint.Summary, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(entryPoint.Example)) pointDoc.Add(new Field("example", entryPoint.Example, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); foreach (var pointMethod in entryPoint.Methods) { var doc = new Document(); //Id keys doc.Add(new Field("point", entryPoint.Name, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES)); doc.Add(new Field("path", pointMethod.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES)); doc.Add(new Field("url", pointMethod.Path, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Notes)) doc.Add(new Field("notes", pointMethod.Notes, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Remarks)) doc.Add(new Field("remarks", pointMethod.Remarks, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Example)) doc.Add(new Field("examlpe", pointMethod.Example, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Returns)) doc.Add(new Field("returns", pointMethod.Returns, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.ShortName)) doc.Add(new Field("short", pointMethod.ShortName, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Summary)) doc.Add(new Field("summary", pointMethod.Summary, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); foreach (var param in pointMethod.Params) { if (!string.IsNullOrEmpty(param.Description)) doc.Add(new Field("param-" + param.Name + "-description", param.Description, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Remarks)) doc.Add(new Field("param-" + param.Name + "-remarks", param.Remarks, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); } indexModifier.AddDocument(doc); } } indexModifier.Optimize(); indexModifier.Close(); searcher = new IndexSearcher(_directory); IsOperational = true; } catch (Exception) { IsOperational = false; } } } }