public void Dispose() { if (m_MapIndex != null) { m_MapIndex.Close(); } else { MapStream?.Close(); } DataStream?.Close(); IndexReader?.Close(); }
private List <SearchResult> search(string keyword, string shipword) { string indexPath = "E:/Index"; //Directory directory = FSDirectory.Open(new File(indexPath)); FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NoLockFactory()); /* bool isUpdate = IndexReader.IndexExists(indexPath); * if (isUpdate) * { * if (IndexWriter.IsLocked(indexPath)) * { * IndexWriter.Unlock(indexPath); * } * }*/ // IndexReader reader = IndexReader.Open(directory, true); IndexReader reader = DirectoryReader.Open(directory); IndexSearcher search = new IndexSearcher(reader); TopDocs topdocs = null; /* QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29,"content", new PanGuAnalyzer()); * Query query = parser.Parse(keyword); * TopDocs topdocs = search.Search(query, 1000); */ // 认为地理名词与航母至少有一个不为空 string temp = null; if ((keyword == "") || (shipword == "")) { temp = keyword + shipword; QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, "content", new PanGuAnalyzer()); Query query = parser.Parse(temp); topdocs = search.Search(query, 1000); } else if ((keyword != "") && (shipword != "")) { //多种field查询 string[] queryString = { kw, shipword }; //地理位置 航母号 时间 string[] fields = { "content", "content" }; BooleanClause.Occur[] flags = { BooleanClause.Occur.MUST, BooleanClause.Occur.MUST }; Query multiQuery = MultiFieldQueryParser.Parse(Lucene.Net.Util.Version.LUCENE_29, queryString, fields, flags, new PanGuAnalyzer()); topdocs = search.Search(multiQuery, 1000); } // Query muly = QueryParser.Parse(queryString, fields,flags, new PanGuAnalyzer()); ScoreDoc[] scoreDocs = topdocs.scoreDocs; List <SearchResult> list = new List <SearchResult>(); logger.Debug("查询结果总数---" + topdocs.totalHits + " 最大的评分--" + topdocs.GetMaxScore()); count = topdocs.totalHits; // Console.WriteLine("查询结果总数---" + topdocs.totalHits + " 最大的评分--" + topdocs.GetMaxScore()); for (int i = 0; i < scoreDocs.Length; i++) { int doc = scoreDocs[i].doc; Document document = search.Doc(doc); // Console.WriteLine("id--" + scoreDocs[i].doc + "---scors--" + scoreDocs[i].score + "---uri--" + document.Get("link")); string number = scoreDocs[i].doc.ToString(); string score = scoreDocs[i].score.ToString(); string uri = document.Get("link"); string date = document.Get("publishtime").ToString(); string title = "标题:" + document.Get("title").ToString() + document.Get("publishtime").ToString(); // SearchResult searcher = new SearchResult() { Number = number, Score = score, BodyPreview = Preview(body, kw) }; SearchResult searcher = new SearchResult() { Number = number, Score = score, Uri = uri, Title = title, Date = date }; list.Add(searcher); } reader.Close(); return(list); }
private void CloseInternalReader(bool trace, IndexReader subReader, bool finalClose) { ReaderData readerData; // TODO: can we avoid the lock? lock (semaphoreIndexReaderLock) { readerData = searchIndexReaderSemaphores[subReader]; } if (readerData == null) { log.Error("Trying to close a Lucene IndexReader not present: " + subReader.Directory()); // TODO: Should we try to close? return; } // Acquire the locks in the same order as everywhere else object directoryProviderLock = perDirectoryProviderManipulationLocks[readerData.Provider]; bool closeReader = false; lock (directoryProviderLock) { IndexReader reader; bool isActive = activeSearchIndexReaders.TryGetValue(readerData.Provider, out reader) && reader == subReader; if (trace) { log.Info("IndexReader not active: " + subReader); } lock (semaphoreIndexReaderLock) { readerData = searchIndexReaderSemaphores[subReader]; if (readerData == null) { log.Error("Trying to close a Lucene IndexReader not present: " + subReader.Directory()); // TODO: Should we try to close? return; } //final close, the semaphore should be at 0 already if (!finalClose) { readerData.Semaphore--; if (trace) { log.Info("Semaphore decreased to: " + readerData.Semaphore + " for " + subReader); } } if (readerData.Semaphore < 0) { log.Error("Semaphore negative: " + subReader.Directory()); } if (!isActive && readerData.Semaphore == 0) { searchIndexReaderSemaphores.Remove(subReader); closeReader = true; } else { closeReader = false; } } } if (closeReader) { if (trace) { log.Info("Closing IndexReader: " + subReader); } try { subReader.Close(); } catch (IOException e) { log.Warn("Unable to close Lucene IndexReader", e); } } }
private IndexReader ReplaceActiveReader(IndexReader outOfDateReader, object directoryProviderLock, IDirectoryProvider directoryProvider, IndexReader[] readers) { bool trace = log.IsInfoEnabled; IndexReader oldReader; bool closeOldReader = false; bool closeOutOfDateReader = false; IndexReader reader; /** * Since out of lock protection, can have multiple readers created in // * not worse than NotShared and limit the locking time, hence scalability */ try { reader = IndexReader.Open(directoryProvider.Directory); } catch (IOException e) { throw new SearchException("Unable to open Lucene IndexReader", e); } lock (directoryProviderLock) { // Since not protected by a lock, other ones can have been added oldReader = activeSearchIndexReaders[directoryProvider] = reader; lock (semaphoreIndexReaderLock) { searchIndexReaderSemaphores[reader] = new ReaderData(1, directoryProvider); if (trace) { log.Info("Semaphore: 1 for " + reader); } if (outOfDateReader != null) { ReaderData readerData; searchIndexReaderSemaphores.TryGetValue(outOfDateReader, out readerData); if (readerData == null) { closeOutOfDateReader = false; //already removed by another prevous thread } else if (readerData.Semaphore == 0) { searchIndexReaderSemaphores.Remove(outOfDateReader); closeOutOfDateReader = true; } else { closeOutOfDateReader = false; } } if (oldReader != null && oldReader != outOfDateReader) { ReaderData readerData = searchIndexReaderSemaphores[oldReader]; if (readerData == null) { log.Warn("Semaphore should not be null"); closeOldReader = true; //TODO should be true or false? } else if (readerData.Semaphore == 0) { searchIndexReaderSemaphores.Remove(oldReader); closeOldReader = true; } else { closeOldReader = false; } } } } if (closeOutOfDateReader) { if (trace) { log.Info("Closing out of date IndexReader " + outOfDateReader); } try { outOfDateReader.Close(); } catch (IOException e) { ReaderProviderHelper.Clean(readers); throw new SearchException("Unable to close Lucene IndexReader", e); } } if (closeOldReader) { if (trace) { log.Info("Closing old IndexReader " + oldReader); } try { oldReader.Close(); } catch (IOException e) { ReaderProviderHelper.Clean(readers); throw new SearchException("Unable to close Lucene IndexReader", e); } } return(reader); }
private void ValidateSearcher(bool forceReopen) { EnsureIndex(); if (!forceReopen) { if (_reader == null) { lock (_locker) { //double check if (_reader == null) { try { //get a reader - could be NRT or based on directly depending on how this was constructed _reader = _nrtWriter == null ? OpenNewReader() : _nrtWriter.GetReader(); _searcher = new IndexSearcher(_reader); //track it! OpenReaderTracker.Current.AddOpenReader(_reader); } catch (IOException ex) { throw new ApplicationException("Could not create an index searcher with the supplied lucene directory", ex); } } } } else { switch (_reader.GetReaderStatus()) { case ReaderStatus.Current: break; case ReaderStatus.Closed: lock (_locker) { //get a reader - could be NRT or based on directly depending on how this was constructed _reader = _nrtWriter == null ? OpenNewReader() : _nrtWriter.GetReader(); _searcher = new IndexSearcher(_reader); //track it! OpenReaderTracker.Current.AddOpenReader(_reader); } break; case ReaderStatus.NotCurrent: lock (_locker) { //yes, this is actually the way the Lucene wants you to work... //normally, i would have thought just calling Reopen() on the underlying reader would suffice... but it doesn't. //here's references: // http://stackoverflow.com/questions/1323779/lucene-indexreader-reopen-doesnt-seem-to-work-correctly // http://gist.github.com/173978 //Also note that when a new reader is returned from Reopen() the old reader is not actually closed - // but more importantly the old reader might still be in use from another thread! So we can't just // close it here because that would cause a YSOD: Lucene.Net.Store.AlreadyClosedException: this IndexReader is closed // since another thread might be using it. I'm 'hoping' that the GC will just take care of the left over reader's that might // be currently being used in a search, otherwise there's really no way to now when it's safe to close the reader. var newReader = _reader.Reopen(); if (newReader != _reader) { //if it's changed, then re-assign, note: the above, before we used to close the old one here // but that will cause problems since the old reader might be in use on another thread. _reader = newReader; _searcher = new IndexSearcher(_reader); //track it! OpenReaderTracker.Current.AddOpenReader(_reader); //get rid of old ones (anything a minute or older) OpenReaderTracker.Current.CloseStaleReaders(GetLuceneDirectory(), TimeSpan.FromMinutes(1)); } } break; } } } else { if (_reader != null) { lock (_locker) { //double check if (_reader != null) { try { _searcher.Close(); _reader.Close(); } catch (IOException ex) { //this will happen if it's already closed ( i think ) Trace.TraceError("Examine: error occurred closing index searcher. {0}", ex); } finally { //set to null in case another call to this method has passed the first lock and is checking for null _searcher = null; _reader = null; } try { //get a reader - could be NRT or based on directly depending on how this was constructed _reader = _nrtWriter == null ? OpenNewReader() : _nrtWriter.GetReader(); _searcher = new IndexSearcher(_reader); } catch (IOException ex) { throw new ApplicationException("Could not create an index searcher with the supplied lucene directory", ex); } } } } } }
public void InternalSharding() { ISession s = OpenSession(); ITransaction tx = s.BeginTransaction(); Animal a = new Animal(); a.Id = 1; a.Name = "Elephant"; s.Persist(a); a = new Animal(); a.Id = 2; a.Name = "Bear"; s.Persist(a); tx.Commit(); s.Clear(); IndexReader reader = IndexReader.Open(new FileInfo(BaseIndexDir.FullName + "\\Animal00")); try { int num = reader.NumDocs(); Assert.AreEqual(1, num); } finally { reader.Close(); } reader = IndexReader.Open(new FileInfo(BaseIndexDir.FullName + "\\Animal.1")); try { int num = reader.NumDocs(); Assert.AreEqual(1, num); } finally { reader.Close(); } tx = s.BeginTransaction(); a = (Animal)s.Get(typeof(Animal), 1); a.Name = "Mouse"; tx.Commit(); s.Clear(); reader = IndexReader.Open(new FileInfo(BaseIndexDir.FullName + "\\Animal.1")); try { int num = reader.NumDocs(); Assert.AreEqual(1, num); TermDocs docs = reader.TermDocs(new Term("name", "mouse")); Assert.IsTrue(docs.Next()); Document doc = reader.Document(docs.Doc()); Assert.IsFalse(docs.Next()); } finally { reader.Close(); } tx = s.BeginTransaction(); IFullTextSession fts = Search.CreateFullTextSession(s); QueryParser parser = new QueryParser("id", new StopAnalyzer()); IList results = fts.CreateFullTextQuery(parser.Parse("name:mouse OR name:bear")).List(); Assert.AreEqual(2, results.Count, "Either double insert, single update, or query fails with shards"); // cleanup s.Delete("from System.Object"); tx.Commit(); s.Close(); }
public static TermInfo[] GetHighFreqTerms(Directory dir, Hashtable junkWords, int numTerms, String[] fields) { if (dir == null || fields == null) { return(new TermInfo[0]); } IndexReader reader = IndexReader.Open(dir, true); TermInfoQueue tiq = new TermInfoQueue(numTerms); TermEnum terms = reader.Terms(); int minFreq = 0; while (terms.Next()) { String field = terms.Term().Field(); if (fields != null && fields.Length > 0) { bool skip = true; for (int i = 0; i < fields.Length; i++) { if (field.Equals(fields[i])) { skip = false; break; } } if (skip) { continue; } } if (junkWords != null && junkWords[terms.Term().Text()] != null) { continue; } if (terms.DocFreq() > minFreq) { TermInfo top = (TermInfo)tiq.Add(new TermInfo(terms.Term(), terms.DocFreq())); if (tiq.Size() >= numTerms) // if tiq overfull { tiq.Pop(); // remove lowest in tiq minFreq = top.DocFreq; // reset minFreq } } } TermInfo[] res = new TermInfo[tiq.Size()]; for (int i = 0; i < res.Length; i++) { res[res.Length - i - 1] = (TermInfo)tiq.Pop(); } reader.Close(); return(res); }
// tests storing "id" and "field2" fields as pulsing codec, // whose term sort is backwards unicode code point, and // storing "field1" as a custom entirely-in-RAM codec public virtual void TestPerFieldCodec() { int NUM_DOCS = atLeast(173); if (VERBOSE) { Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS); } BaseDirectoryWrapper dir = newDirectory(); dir.CheckIndexOnClose = false; // we use a custom codec provider IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(new CustomPerFieldCodec()).setMergePolicy(newLogMergePolicy(3))); Document doc = new Document(); // uses default codec: doc.Add(newTextField("field1", "this field uses the standard codec as the test", Field.Store.NO)); // uses pulsing codec: Field field2 = newTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO); doc.Add(field2); Field idField = newStringField("id", "", Field.Store.NO); doc.Add(idField); for (int i = 0; i < NUM_DOCS; i++) { idField.StringValue = "" + i; w.addDocument(doc); if ((i + 1) % 10 == 0) { w.commit(); } } if (VERBOSE) { Console.WriteLine("TEST: now delete id=77"); } w.deleteDocuments(new Term("id", "77")); IndexReader r = DirectoryReader.Open(w, true); Assert.AreEqual(NUM_DOCS - 1, r.NumDocs()); IndexSearcher s = newSearcher(r); Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); r.Close(); if (VERBOSE) { Console.WriteLine("\nTEST: now delete 2nd doc"); } w.deleteDocuments(new Term("id", "44")); if (VERBOSE) { Console.WriteLine("\nTEST: now force merge"); } w.forceMerge(1); if (VERBOSE) { Console.WriteLine("\nTEST: now open reader"); } r = DirectoryReader.Open(w, true); Assert.AreEqual(NUM_DOCS - 2, r.MaxDoc()); Assert.AreEqual(NUM_DOCS - 2, r.NumDocs()); s = newSearcher(r); Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); Assert.AreEqual(1, s.Search(new TermQuery(new Term("id", "76")), 1).TotalHits); Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "77")), 1).TotalHits); Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "44")), 1).TotalHits); if (VERBOSE) { Console.WriteLine("\nTEST: now close NRT reader"); } r.Close(); w.Dispose(); dir.Dispose(); }
public override void TearDown() { reader.Close(); base.TearDown(); }
void IDisposable.Dispose() { GC.SuppressFinalize(this); _idx.Close(); }
private void ProcessQueue(DataTable q, string indexPath) { rowsProcessed = 0; rowsToProcess = q.Rows.Count; // first process deletes with reader try { IndexReader reader = IndexReader.Open(indexPath); foreach (DataRow row in q.Rows) { Term term = new Term("Key", row["ItemKey"].ToString()); try { reader.DeleteDocuments(term); log.Debug("reader.DeleteDocuments(term) for Key " + row["ItemKey"].ToString()); } catch (Exception ge) { // TODO: monitor what real exceptions if any occur and then // change this catch to catch only the expected ones // instead of non specific exception log.Error(ge); } bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]); if (removeOnly) { Int64 rowId = Convert.ToInt64(row["RowId"]); IndexingQueue.Delete(rowId); } if (DateTime.UtcNow > nextStatusUpdateTime) { // don't mark as complete because there may be more qu items //for different index paths in a multi site installation bool markAsComplete = false; ReportStatus(markAsComplete); } } reader.Close(); } catch (IOException ex) { log.Info("IndexWriter swallowed exception this is not unexpected if building or rebuilding the search index ", ex); errorCount += 1; } catch (TypeInitializationException ex) { log.Info("IndexWriter swallowed exception ", ex); errorCount += 1; } // next add items with writer IndexWriter indexWriter = GetWriter(indexPath); if (indexWriter == null) { log.Error("failed to get IndexWriter for path: " + indexPath); errorCount += 1; return; } foreach (DataRow row in q.Rows) { bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]); if (!removeOnly) { try { IndexItem indexItem = (IndexItem)SerializationHelper.DeserializeFromString(typeof(IndexItem), row["SerializedItem"].ToString()); Document doc = GetDocument(indexItem); WriteToIndex(doc, indexWriter); log.Debug("called WriteToIndex(doc, indexWriter) for key " + indexItem.Key); Int64 rowId = Convert.ToInt64(row["RowId"]); IndexingQueue.Delete(rowId); } catch (Exception ex) { log.Error(ex); } } if (DateTime.UtcNow > nextStatusUpdateTime) { // don't mark as complete because there may be more qu items //for different index paths in a multi site installation bool markAsComplete = false; ReportStatus(markAsComplete); } } try { indexWriter.Optimize(); } catch (IOException ex) { log.Error(ex); } try { indexWriter.Close(); } catch (IOException ex) { log.Error(ex); } }
public void _Search() { //string request = (searchParams as string); string old_request = ""; string new_request = ""; while (true) { lock (_request) { new_request = _request; } if (new_request != old_request) { old_request = new_request; if (new_request.Length != 0) { IndexReader reader = null; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); string index_path = Owl.Properties.Settings.Default.IndexPath; try { reader = IndexReader.Open(index_path); } catch { status.SearchStatus = string.Format("Problems while opening Index: has it been created in {0} ?", Owl.Properties.Settings.Default.IndexPath); } int nb_docs = 0; int found_docs = 0; if (reader != null) { try { Searcher searcher = new IndexSearcher(reader); Analyzer analyzer = new StandardAnalyzer(); //QueryParser parser = new QueryParser("contents", analyzer); MultiFieldQueryParser parser = new MultiFieldQueryParser(new string[] { "contents", "path" }, analyzer); Query query = parser.Parse(new_request); SimpleCall sc = delegate() { resultItems.Clear(); }; Application.Current.Dispatcher.Invoke(DispatcherPriority.Background, sc); //Hits hits = searcher.Search(query); TopDocs docs = searcher.Search(query, null, 100); int num_doc = 1; foreach (ScoreDoc score_doc in docs.scoreDocs) { if (HasRequestChanged(new_request)) { break; } Document doc = searcher.Doc(score_doc.doc); System.String path = doc.Get("path"); // SimpleCall sc2 = delegate() { resultItems.Add(new Result(string.Format("{0} - {2} ({1})%\n{3}", num_doc++, (int)((score_doc.score * 100) / docs.GetMaxScore()), System.IO.Path.GetFileName(path), System.IO.Path.GetDirectoryName(path)), path)); }; Application.Current.Dispatcher.Invoke(DispatcherPriority.Background, sc2); // } found_docs = docs.scoreDocs.Length; nb_docs = reader.NumDocs(); searcher.Close(); } //catch (TokenMgrError) //{ } catch (Exception e) { status.SearchStatus = string.Format("Problems with request {0} ", new_request); Log.Error(e); } finally { reader.Close(); stopWatch.Stop(); } } //--- status.SearchStatus = string.Format("{0} results for '{3}' in {1} docs (took {2} ms)", found_docs, nb_docs, stopWatch.ElapsedMilliseconds, new_request); } } else { _mre.Reset(); _mre.WaitOne(); } //Thread.Sleep(250); } }
public void Close() { _reader.Close(); }
internal void StartCrawler() { // install the scheduler in a separate thread new Thread(delegate() { #if Log Console.WriteLine("Initiating Resource Scheduler in file system crawler"); #endif Scheduler(); }).Start(); //Start the crawler in the current thread #if Log Console.WriteLine("Initiating file system Crawler"); #endif long start = DateTime.Now.Ticks; foreach (string drive in GlobalData.Drives) { DriveInfo di = new DriveInfo(drive); if (di.IsReady && di.DriveType == DriveType.Fixed && GlobalData.RunCrawler) { //check whether the index directory is clean or not. if not delete all files in it if (Directory.Exists(GlobalData.IndexRootPath + drive[0])) { Directory.Delete(GlobalData.IndexRootPath + drive[0], true); } Directory.CreateDirectory(GlobalData.IndexRootPath + drive[0]); //Initialize the indexer GlobalData.Indexer = new EDSIndexer(GlobalData.IndexRootPath + drive[0]); //Start File System Crawler Crawler(drive); //Optimize the indexes GlobalData.Indexer.Optimize(); GlobalData.Indexer.Close(); } } #if Log Console.WriteLine("Completed File System Crawling"); #endif if (GlobalData.RunCrawler) //only when the crawler finished its work do the following stuff.....otherwise if crawler has been terminated by closing the application donot do the following stuff { GlobalData.IndexingCompleted = true; long end = DateTime.Now.Ticks; DateTime dt = new DateTime(end - start); GlobalData.lIndexingStatus.Text = "Indexing: Completed 100% in " + (dt.Hour != 0 ? dt.Hour + "h:" : "") + (dt.Minute != 0 ? dt.Minute + "m:" : "") + (dt.Second != 0 ? dt.Second + "s: " : "") + (dt.Millisecond != 0 ? dt.Millisecond + "ms" : ""); //update the statusbar indexed documents count int docCount = 0; foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath)) { IndexReader ir = IndexReader.Open(dir); docCount += ir.NumDocs(); ir.Close(); } if (GlobalData.MainStatusStrip != null && GlobalData.MainStatusStrip.Items.Count != 0) { GlobalData.MainStatusStrip.Items["tsslDocCount"].Text = docCount + " documents indexed "; } //Finally start file system monitor for further file system updates if (GlobalData.RunMonitor) { #if Log Console.WriteLine("Starting File System Monitor"); #endif FileSystemMonitor fsw = FileSystemMonitor.GetInstance(); foreach (string drive in Environment.GetLogicalDrives()) { fsw.AddMonitor(drive); } fsw.StartAllMonitors(); } } //after completion of crawling stop the scheduler by setting RunScheduler => false; GlobalData.RunScheduler = false; }
public List <Post> Similar(int postid, int itemsToReturn) { List <Post> TList = new List <Post>(); int docId = -1; IndexSearcher searcher = null; IndexReader reader = null; if (rd == null) { BuildIndex(); } lck.AcquireReaderLock(ReaderTimeOut); try { Analyzer analyzer = GetAnalyzer(); QueryParser parser = GetQueryParser(analyzer); parser.SetDefaultOperator(QueryParser.AND_OPERATOR); Query q = parser.Parse("postid:" + postid); searcher = new IndexSearcher(rd, true); //TODO #pragma warning disable CS0618 // Type or member is obsolete Hits hits = searcher.Search(q); #pragma warning restore CS0618 // Type or member is obsolete if (hits != null && hits.Length() > 0) { docId = hits.Id(0); } if (docId > -1) { reader = IndexReader.Open(rd, true); TermFreqVector tfv = reader.GetTermFreqVector(docId, "exact"); BooleanQuery booleanQuery = new BooleanQuery(); for (int j = 0; j < tfv.Size(); j++) { TermQuery tq = new TermQuery(new Term("exact", tfv.GetTerms()[j])); booleanQuery.Add(tq, BooleanClause.Occur.SHOULD); } //TODO #pragma warning disable CS0618 // Type or member is obsolete Hits similarhits = searcher.Search(booleanQuery, Sort.RELEVANCE); #pragma warning restore CS0618 // Type or member is obsolete for (int i = 0; i < similarhits.Length(); i++) { Document doc = similarhits.Doc(i); if (similarhits.Id(i) != docId) { TList.Add(CreateFromDocument(doc, analyzer, null)); } if (TList.Count >= itemsToReturn) { break; } } } } catch (Exception) { } finally { if (searcher != null) { searcher.Close(); } if (reader != null) { reader.Close(); } lck.ReleaseReaderLock(); } return(TList); }
public void End(bool shouldClose) { if (!_is_started) { return; } if (!shouldClose) { return; } //build 2del file list if (!_job_status.Cancelled) { TermEnum term_enum = _index_reader.Terms(); Term path_term = new Term("path"); int nb_terms = 0; while (term_enum.SkipTo(path_term)) //skip to new term equal or *ABOVE* "path:" !!! { Term term = term_enum.Term(); if (term.Field() != path_term.Field()) { break; } if (!File.Exists(term.Text())) { _del_file_list.Add(term.Text()); } if (_job_status.Cancelled) { break; } nb_terms++; } term_enum.Close(); Logger.Log.Info("update: deletion: {0} analyzed terms, found {1} vanished files.", nb_terms, _del_file_list.Count); } _index_searcher.Close(); _index_reader.Close(); //--- deleting deprecated if ((_del_file_list.Count > 0) && (!_job_status.Cancelled)) { Stopwatch watch = new Stopwatch(); watch.Start(); int num_file = 0; int nb_files = _del_file_list.Count; IndexWriter writer = new IndexWriter(_index_path, _default_analyzer, false); foreach (string path in _del_file_list) { if (((num_file++) % 101) == 1) { int progress = ((((num_file++) + 1)) * 100) / nb_files; _job_status.Progress = progress; _job_status.Description = String.Format("upd: removing (from index) file {0}/{1} - {2}", num_file, _del_file_list.Count, StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000))); } if (_job_status.Cancelled) { break; } writer.DeleteDocuments(new Term("path", path)); } writer.Commit(); writer.Close(); watch.Stop(); } //adding new files if ((_add_file_list.Count > 0) && (!_job_status.Cancelled)) { Stopwatch watch = new Stopwatch(); watch.Start(); IndexWriter writer = null; try { writer = new IndexWriter(_index_path, _default_analyzer, false, new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); int num_file = 0; int nb_files = _add_file_list.Count; foreach (BasicFileInfo fi in _add_file_list) { if (((num_file++) % 101) == 1) { int progress = ((((num_file++) + 1)) * 100) / nb_files; _job_status.Progress = progress; _job_status.Description = String.Format("upd: indexing new file {0}/{1} - {2}", num_file, _add_file_list.Count, StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000))); } if (_job_status.Cancelled) { break; } writer.AddDocument(_doc_factory.CreateFromPath(fi.FilePath, fi.LastModification)); if (num_file % 20 == 0) { writer.Commit(); } } writer.Commit(); } catch (System.Exception ex) { Log.Error(ex); } finally { if (writer != null) { writer.Close(); writer = null; } } watch.Stop(); } //updating modified files if ((_upd_file_list.Count > 0) && (!_job_status.Cancelled)) { Stopwatch watch = new Stopwatch(); watch.Start(); int num_file = 0; int nb_files = _upd_file_list.Count; IndexWriter writer = null; try { writer = new IndexWriter(_index_path, _default_analyzer, false, new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); foreach (BasicFileInfo fi in _upd_file_list) { if (((num_file++) % 101) == 1) { int progress = ((((num_file++) + 1)) * 100) / nb_files; _job_status.Progress = progress; _job_status.Description = String.Format("upd: modified file {0}/{1} - {2}", num_file, _upd_file_list.Count, StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000))); } if (_job_status.Cancelled) { break; } writer.UpdateDocument(new Term("path", fi.FilePath), _doc_factory.CreateFromPath(fi.FilePath, fi.LastModification)); } writer.Commit(); //LittleBeagle.Properties.Settings.Default.NbIndexedFiles = num_file; } catch (System.Exception ex) { Log.Error(ex); } finally { if (writer != null) { writer.Close(); writer = null; } } watch.Stop(); } }
void IndexGatheredFiles(object state) { try { #region Deleting files from the indexes //Iterate over all the directories present in the %appdata%\EDS\indexes folder if (GlobalData.DeletedFiles.Count > 0) { foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath)) { //Deletion of files from indexes IndexReader reader = IndexReader.Open(dir); lock (GlobalData.DeletedFiles) { //foreach (string path in GlobalData.DeletedFiles) while (GlobalData.DeletedFiles.Count != 0) { reader.DeleteDocuments(new Term("path", GlobalData.DeletedFiles[0])); GlobalData.lIndexingStatus.Text = GlobalData.DeletedFiles[0] + " is deleted from indexes"; GlobalData.DeletedFiles.RemoveAt(0); } } reader.Close(); } } #endregion #region indexing newly created files //Indexing of new Files if (GlobalData.CreatedFiles.Count > 0) { lock (GlobalData.CreatedFiles) { // foreach (string path in GlobalData.CreatedFiles) while (GlobalData.CreatedFiles.Count != 0) { string path = GlobalData.CreatedFiles[0]; //Get the respective content handler. If no content handler is present return the default handler IEDSParser parser = GlobalData.Parsers.ContainsKey(Path.GetExtension(path).ToLower()) ? GlobalData.Parsers[Path.GetExtension(path).ToLower()] : GlobalData.Parsers["*.*"]; if (parser != null) //For some nasty reason the file got deleted or doesnot exist { StringDictionary properties = parser.GetProperties(path); if (properties != null) { //Initialize the indexer EDSIndexer indexer = new EDSIndexer(GlobalData.IndexRootPath + path[0]); indexer.Index(properties); indexer.Close(); } } GlobalData.lIndexingStatus.Text = GlobalData.CreatedFiles[0] + " is newly indexed"; GlobalData.CreatedFiles.RemoveAt(0); } } } #endregion #region Updating already indexed files lock (GlobalData.ChangedFiles) { //foreach (string path in GlobalData.ChangedFiles) while (GlobalData.ChangedFiles.Count != 0) { string path = GlobalData.ChangedFiles[0]; //first delete that document foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath)) { //Deletion of files from indexes IndexReader reader = IndexReader.Open(dir); reader.DeleteDocuments(new Term("path", path)); bool deleted = reader.HasDeletions(); reader.Close(); if (deleted) { break; } } //reindex the document //Get the respective content handler. If no content handler is present return the default handler IEDSParser parser = GlobalData.Parsers.ContainsKey(Path.GetExtension(path).ToLower()) ? GlobalData.Parsers[Path.GetExtension(path).ToLower()] : GlobalData.Parsers["*.*"]; if (parser != null) //For some nasty reason the file got deleted or doesnot exist { StringDictionary properties = parser.GetProperties(path); if (properties != null) { //Initialize the indexer EDSIndexer indexer = new EDSIndexer(GlobalData.IndexRootPath + path[0]); indexer.Index(properties); indexer.Close(); } } GlobalData.lIndexingStatus.Text = path + " is updated in the indexes"; GlobalData.ChangedFiles.RemoveAt(0); } } #endregion #region Updating renamed files lock (GlobalData.RenamedFiles) { //foreach (string path in GlobalData.ChangedFiles) while (GlobalData.RenamedFiles.Count != 0) { string path = GlobalData.RenamedFiles[0].OldFullPath; //first delete the old document foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath)) { //Deletion of files from indexes IndexReader reader = IndexReader.Open(dir); reader.DeleteDocuments(new Term("path", path)); bool deleted = reader.HasDeletions(); reader.Close(); if (deleted) { break; } } //reindex the new document //Get the respective content handler. If no content handler is present return the default handler path = GlobalData.RenamedFiles[0].FullPath; IEDSParser parser = GlobalData.Parsers.ContainsKey(Path.GetExtension(path).ToLower()) ? GlobalData.Parsers[Path.GetExtension(path).ToLower()] : GlobalData.Parsers["*.*"]; if (parser != null) //For some nasty reason the file got deleted or doesnot exist { StringDictionary properties = parser.GetProperties(path); if (properties != null) { //Initialize the indexer EDSIndexer indexer = new EDSIndexer(GlobalData.IndexRootPath + path[0]); indexer.Index(properties); indexer.Close(); } } GlobalData.lIndexingStatus.Text = "Renamed file " + path + " is updated in the indexes"; GlobalData.RenamedFiles.RemoveAt(0); } } #endregion } catch (Exception ex) { #if Log Console.WriteLine(ex); #endif } }
public void TearDown() { reader.Close(); searcher.Close(); directory.Close(); }
protected internal virtual IndexReader DoReopen(bool doClone) { EnsureOpen(); bool reopened = false; IList <IndexReader> newReaders = new List <IndexReader>(); bool success = false; try { foreach (var oldReader in readers) { IndexReader newReader = null; if (doClone) { newReader = (IndexReader)oldReader.Clone(); } else { newReader = oldReader.Reopen(); } newReaders.Add(newReader); // if at least one of the subreaders was updated we remember that // and return a new ParallelReader if (newReader != oldReader) { reopened = true; } } success = true; } finally { if (!success && reopened) { for (int i = 0; i < newReaders.Count; i++) { IndexReader r = newReaders[i]; if (r != readers[i]) { try { r.Close(); } catch (System.IO.IOException) { // keep going - we want to clean up as much as possible } } } } } if (reopened) { List <bool> newDecrefOnClose = new List <bool>(); ParallelReader pr = new ParallelReader(); for (int i = 0; i < readers.Count; i++) { IndexReader oldReader = readers[i]; IndexReader newReader = newReaders[i]; if (newReader == oldReader) { newDecrefOnClose.Add(true); newReader.IncRef(); } else { // this is a new subreader instance, so on close() we don't // decRef but close it newDecrefOnClose.Add(false); } pr.Add(newReader, !storedFieldReaders.Contains(oldReader)); } pr.decrefOnClose = newDecrefOnClose; pr.incRefReaders = incRefReaders; return(pr); } else { // No subreader was refreshed return(this); } }
public override void TearDown() { reader.Close(); dir2.Close(); }
private void CloseIndexReader(IndexReader myReader) { myReader.Close(); myReader.Dispose(); }