/// <summary> /// Deletes the index. /// </summary> /// <param name="indexPath">The index path.</param> /// <param name="docList">The doc list.</param> /// <returns></returns> public bool DeleteIndex(string indexPath, List <string> docList) { bool result = false; try { using (Lucene.Net.Store.Directory directory = FSDirectory.Open(new DirectoryInfo(indexPath))) { Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29); var parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, RegularExpression, analyzer); IndexModifier modifier = new Lucene.Net.Index.IndexModifier(indexPath, analyzer, false); foreach (string doc in docList) { modifier.DeleteDocuments(new Lucene.Net.Index.Term(FilePath, doc)); } modifier.Flush(); modifier.Close(); } } catch (Exception ex) { // Swallow Exceptions here } return(result); }
/// <summary> /// Runs the indexing process. /// </summary> /// <param name="users">The users.</param> /// <param name="created">if set to <c>true</c> [created].</param> public void CreateOrUpdate(IList<User> users, bool created) { var modifier = new IndexModifier( Setting.UserSearchIndexPath.Value, new StandardAnalyzer(), (!created) ? true : false ); foreach (var user in users) { if (created) { modifier.DeleteDocuments(new Term("id", user.Id.ToString())); } var document = new Document(); UserToDocument(user, document); modifier.AddDocument(document); } modifier.Optimize(); modifier.Close(); }
private void TestIndexInternal(int maxWait) { bool create = true; //Directory rd = new RAMDirectory(); // work on disk to make sure potential lock problems are tested: System.String tempDir = System.IO.Path.GetTempPath(); if (tempDir == null) { throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test"); } System.IO.FileInfo indexDir = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex")); Directory rd = FSDirectory.Open(indexDir); IndexThread.id = 0; IndexThread.idStack.Clear(); IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create); IndexThread thread1 = new IndexThread(index, maxWait, 1); thread1.Start(); IndexThread thread2 = new IndexThread(index, maxWait, 2); thread2.Start(); while (thread1.IsAlive || thread2.IsAlive) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 100)); } index.Optimize(); int added = thread1.added + thread2.added; int deleted = thread1.deleted + thread2.deleted; Assert.AreEqual(added - deleted, index.DocCount()); index.Close(); try { index.Close(); Assert.Fail(); } catch (System.SystemException e) { // expected exception } RmDir(indexDir); }
/// <summary> /// Clears the index. /// </summary> /// <param name="reCreate">if set to <c>true</c> [re create].</param> private static void ClearIndex(bool reCreate) { if (!Directory.Exists(LuceneFolder)) Directory.CreateDirectory(LuceneFolder); if (reCreate) { IndexModifier modifier = new IndexModifier(LuceneFolder, new StandardAnalyzer(), true); modifier.Close(); } }
public void threadproc_update(object obj) { lock (locker) // If a thread is updating the index, no other thread should be doing anything with it. { try { if (searcher != null) { try { searcher.Close(); } catch (Exception e) { } searcher = null; } Lucene.Net.Index.IndexModifier modifier = new Lucene.Net.Index.IndexModifier(DBNLConfigurationManager.LuceneElement.IndexingFolder, analyzer, false); // same as build, but uses "modifier" instead of write. // uses additional "where" clause for bugid int id = (int)obj; modifier.DeleteDocuments(new Lucene.Net.Index.Term("id", Convert.ToString(id))); var item = new ContentService().GetContentById(id); modifier.AddDocument(create_doc( item.ContentId, item.Content1)); modifier.Flush(); modifier.Close(); } catch (Exception e) { } } }
public virtual void TestIndex() { Directory ramDir = new RAMDirectory(); IndexModifier i = new IndexModifier(ramDir, new StandardAnalyzer(), true); i.AddDocument(GetDoc()); Assert.AreEqual(1, i.DocCount()); i.Flush(); i.AddDocument(GetDoc(), new SimpleAnalyzer()); Assert.AreEqual(2, i.DocCount()); i.Optimize(); Assert.AreEqual(2, i.DocCount()); i.Flush(); i.DeleteDocument(0); Assert.AreEqual(1, i.DocCount()); i.Flush(); Assert.AreEqual(1, i.DocCount()); i.AddDocument(GetDoc()); i.AddDocument(GetDoc()); i.Flush(); // depend on merge policy - Assert.AreEqual(3, i.docCount()); i.DeleteDocuments(allDocTerm); Assert.AreEqual(0, i.DocCount()); i.Optimize(); Assert.AreEqual(0, i.DocCount()); // Lucene defaults: Assert.IsNull(i.GetInfoStream()); Assert.IsTrue(i.GetUseCompoundFile()); Assert.AreEqual(IndexWriter.DISABLE_AUTO_FLUSH, i.GetMaxBufferedDocs()); Assert.AreEqual(10000, i.GetMaxFieldLength()); Assert.AreEqual(10, i.GetMergeFactor()); // test setting properties: i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Flush(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); // test setting properties when internally the reader is opened: i.DeleteDocuments(allDocTerm); i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Optimize(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); i.Close(); try { i.DocCount(); Assert.Fail(); } catch (System.SystemException e) { // expected exception } }
private void TestIndexInternal(int maxWait) { bool create = true; //Directory rd = new RAMDirectory(); // work on disk to make sure potential lock problems are tested: System.String tempDir = System.IO.Path.GetTempPath(); if (tempDir == null) throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test"); System.IO.FileInfo indexDir = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex")); Directory rd = FSDirectory.Open(indexDir); IndexThread.id = 0; IndexThread.idStack.Clear(); IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create); IndexThread thread1 = new IndexThread(index, maxWait, 1); thread1.Start(); IndexThread thread2 = new IndexThread(index, maxWait, 2); thread2.Start(); while (thread1.IsAlive || thread2.IsAlive) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 100)); } index.Optimize(); int added = thread1.added + thread2.added; int deleted = thread1.deleted + thread2.deleted; Assert.AreEqual(added - deleted, index.DocCount()); index.Close(); try { index.Close(); Assert.Fail(); } catch (System.SystemException e) { // expected exception } RmDir(indexDir); }
public virtual void TestIndex() { Directory ramDir = new RAMDirectory(); IndexModifier i = new IndexModifier(ramDir, new StandardAnalyzer(), true); i.AddDocument(GetDoc()); Assert.AreEqual(1, i.DocCount()); i.Flush(); i.AddDocument(GetDoc(), new SimpleAnalyzer()); Assert.AreEqual(2, i.DocCount()); i.Optimize(); Assert.AreEqual(2, i.DocCount()); i.Flush(); i.DeleteDocument(0); Assert.AreEqual(1, i.DocCount()); i.Flush(); Assert.AreEqual(1, i.DocCount()); i.AddDocument(GetDoc()); i.AddDocument(GetDoc()); i.Flush(); // depend on merge policy - Assert.AreEqual(3, i.docCount()); i.DeleteDocuments(allDocTerm); Assert.AreEqual(0, i.DocCount()); i.Optimize(); Assert.AreEqual(0, i.DocCount()); // Lucene defaults: Assert.IsNull(i.GetInfoStream()); Assert.IsTrue(i.GetUseCompoundFile()); Assert.AreEqual(IndexWriter.DISABLE_AUTO_FLUSH, i.GetMaxBufferedDocs()); Assert.AreEqual(10000, i.GetMaxFieldLength()); Assert.AreEqual(10, i.GetMergeFactor()); // test setting properties: i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Flush(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); // test setting properties when internally the reader is opened: i.DeleteDocuments(allDocTerm); i.SetMaxBufferedDocs(100); i.SetMergeFactor(25); i.SetMaxFieldLength(250000); i.AddDocument(GetDoc()); i.SetUseCompoundFile(false); i.Optimize(); Assert.AreEqual(100, i.GetMaxBufferedDocs()); Assert.AreEqual(25, i.GetMergeFactor()); Assert.AreEqual(250000, i.GetMaxFieldLength()); Assert.IsFalse(i.GetUseCompoundFile()); i.Close(); try { i.DocCount(); Assert.Fail(); } catch (System.SystemException e) { // expected exception } }
/// <summary> /// Creates/Updates the index for a given host. If the index already exists then we'll update it other /// wise we create a new index for the host. Each host index is store in its own folder off the base directory. /// </summary> /// <param name="hostId"></param> private void GenerateIndex(int hostId, DateTime lastUpdateTime) { Log.InfoFormat("Starting index generation for HostID: {0}", hostId); IndexModifier modifier = null; bool isIncrementalCrawl = false; int storiesToIndexCount = 0; int pageSize; try { bool indexExists = IndexExists(hostId); if (indexExists) { //logic to version the lucene index to allow //use to replace with a new version should we need to //this should be kind of value stored in settings table. //check if the version of the lucene index is the latest version indexExists = IsLuceneIndexCorrectVersion(hostId); if (!indexExists) { Log.Debug("Lucene index exists but is older version, need to overwrite with new document fields"); } } StoryCollection stories = null; if (!indexExists) { //the index doesnt exist so we are going to do a full index of //all stories in the database Log.InfoFormat("Creating a new index HostID: {0}", hostId); isIncrementalCrawl = false; storiesToIndexCount = Story.GetAllStoriesCount(hostId); } else { if (!HostCrawlSuccessful(hostId)) { //force the last update time to a low value to get all the records //since we need to recrawl the index fully lastUpdateTime = DateTime.Parse("1/1/1975"); Log.InfoFormat("Last crawl didnt complete successfully, attempting a full crawl"); } else Log.InfoFormat("Updating existing index"); isIncrementalCrawl = true; storiesToIndexCount = Story.GetUpdatedStoriesCount(hostId, lastUpdateTime); Log.InfoFormat("Found: {0} stories to add to index since last update at: {1}", storiesToIndexCount, lastUpdateTime); } if (storiesToIndexCount == 0) { Log.InfoFormat("Nothing todo, no new stories to crawl for HostID: {0}", hostId); isUpdateRunning = false; return; } modifier = new IndexModifier(IndexHostPath(hostId), new DnkAnalyzer(), !indexExists); modifier.SetMaxBufferedDocs(50); modifier.SetMergeFactor(150); SearchSettings searchSettings = new SearchSettings(); pageSize = searchSettings.StoriesPageSize; int pageTotal = CalculateNumberOfPages(storiesToIndexCount, pageSize); for (int i = 1; i <= pageTotal; i++) { if (isIncrementalCrawl) stories = Story.GetUpdatedStories(hostId, lastUpdateTime, i, pageSize); else stories = Story.GetAllStories(hostId, i, pageSize); AddStoriesToIndex(modifier, isIncrementalCrawl, stories); } modifier.Optimize(); Log.InfoFormat("index optimized for HostID:{0}", hostId); modifier.Close(); Log.InfoFormat("Index Modifier closed for Host:{0}", hostId); modifier = null; //we completed ok RecordHostCrawlSuccess(hostId, true); } catch (Exception ex) { RecordHostCrawlSuccess(hostId, false); Log.ErrorFormat("Error occurred while adding items to the index HostID:{0}, message: {1}",hostId, ex.Message); } finally { //attempt to close the modifier if it still exists if (modifier != null) { try { modifier.Close(); modifier = null; Log.InfoFormat("Able to close the Index Modifier in the final block, HostID:{0}", hostId); } catch (Exception ex) { Log.ErrorFormat("Unable to close the modifer in the final block HostID:{0} Message:{1}", hostId, ex.Message); } } } }
public void CreateIndexIfNeeded(List<MsDocEntryPoint> documentation, DateTime? lastModified) { lock (SynchLock) { if (IsIndexingNeeded(lastModified) || !IsOperational) { try { //Drop all index if (searcher != null) searcher.Close(); if (_directory != null) _directory.Close(); //Delete dir Directory.Delete(_indexDirectory, true); _directory = FSDirectory.GetDirectory(_indexDirectory, true); //Reopen directory var indexModifier = new IndexModifier(_directory, new StandardAnalyzer(), true); foreach (var entryPoint in documentation) { var pointDoc = new Document(); //Id keys pointDoc.Add(new Field("point", entryPoint.Name, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES)); if (!string.IsNullOrEmpty(entryPoint.Summary)) pointDoc.Add(new Field("summary", entryPoint.Summary, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(entryPoint.Example)) pointDoc.Add(new Field("example", entryPoint.Example, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); foreach (var pointMethod in entryPoint.Methods) { var doc = new Document(); //Id keys doc.Add(new Field("point", entryPoint.Name, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES)); doc.Add(new Field("path", pointMethod.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES)); doc.Add(new Field("url", pointMethod.Path, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Notes)) doc.Add(new Field("notes", pointMethod.Notes, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Remarks)) doc.Add(new Field("remarks", pointMethod.Remarks, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Example)) doc.Add(new Field("examlpe", pointMethod.Example, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Returns)) doc.Add(new Field("returns", pointMethod.Returns, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.ShortName)) doc.Add(new Field("short", pointMethod.ShortName, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Summary)) doc.Add(new Field("summary", pointMethod.Summary, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); foreach (var param in pointMethod.Params) { if (!string.IsNullOrEmpty(param.Description)) doc.Add(new Field("param-" + param.Name + "-description", param.Description, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); if (!string.IsNullOrEmpty(pointMethod.Remarks)) doc.Add(new Field("param-" + param.Name + "-remarks", param.Remarks, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); } indexModifier.AddDocument(doc); } } indexModifier.Optimize(); indexModifier.Close(); searcher = new IndexSearcher(_directory); IsOperational = true; } catch (Exception) { IsOperational = false; } } } }
/// <summary> /// </summary> protected override void DoIndexNode(Node node, bool reindexAfterRemovingOld) { lock(this) { foreach (string langKey in luceneDirectoryIndexedByLanguageCodeThenView.Keys) { foreach (string viewKey in luceneDirectoryIndexedByLanguageCodeThenView[langKey].Keys) { Directory d = luceneDirectoryIndexedByLanguageCodeThenView[langKey][viewKey]; IndexModifier modifier = new IndexModifier(d, LuceneLibrarySearchCriteriaAdaptor.ANALYZER, false); try { foreach (int i in GetDocumentIdsForElementId(node.Id)) { Document doc = GetDocument(i); if (MakePath(node) == doc.GetField(LuceneNodeIndexer.PATH_FIELD).StringValue() && node.Order.ToString() == doc.GetField(LuceneNodeIndexer.ORDER_FIELD).StringValue()) { Debug.WriteLine( string.Format("DoIndexNode: Removing document {0} for element id {1}", i, node.Id)); modifier.DeleteDocument(i); } } } finally { Debug.WriteLine("DoIndexNode: Closing index reader"); modifier.Close(); } if (!reindexAfterRemovingOld) return; indexWriter = new IndexWriter(d, LuceneLibrarySearchCriteriaAdaptor.ANALYZER, false); LuceneNodeIndexer indexer = new LuceneNodeIndexer(indexWriter, langKey); indexer.AddToIndex(node, node.PreferredLabel); } } } }