public virtual void TestTermVectorsFieldOrder() { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); doc.Add(new Field("c", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.Add(new Field("a", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.Add(new Field("b", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.Add(new Field("x", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); writer.AddDocument(doc); writer.Close(); IndexReader reader = IndexReader.Open(dir); TermFreqVector[] v = reader.GetTermFreqVectors(0); Assert.AreEqual(4, v.Length); System.String[] expectedFields = new System.String[] { "a", "b", "c", "x" }; int[] expectedPositions = new int[] { 1, 2, 0 }; for (int i = 0; i < v.Length; i++) { TermPositionVector posVec = (TermPositionVector)v[i]; Assert.AreEqual(expectedFields[i], posVec.GetField()); System.String[] terms = posVec.GetTerms(); Assert.AreEqual(3, terms.Length); Assert.AreEqual("content", terms[0]); Assert.AreEqual("here", terms[1]); Assert.AreEqual("some", terms[2]); for (int j = 0; j < 3; j++) { int[] positions = posVec.GetTermPositions(j); Assert.AreEqual(1, positions.Length); Assert.AreEqual(expectedPositions[j], positions[0]); } } }
public virtual void TestAfterClose() { Directory dir1 = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer.SetInfoStream(infoStream); // create the index CreateIndexNoClose(false, "test", writer); IndexReader r = writer.GetReader(); writer.Close(); _TestUtil.CheckIndex(dir1); // reader should remain usable even after IndexWriter is closed: Assert.AreEqual(100, r.NumDocs()); Query q = new TermQuery(new Term("indexname", "test")); Assert.AreEqual(100, new IndexSearcher(r).Search(q, 10).totalHits); try { r.Reopen(); Assert.Fail("failed to hit AlreadyClosedException"); } catch (AlreadyClosedException ace) { // expected } r.Close(); dir1.Close(); }
public virtual void TestCloneWriteableToReadOnly() { Directory dir1 = new MockRAMDirectory(); TestIndexReaderReopen.CreateIndex(dir1, true); IndexReader reader = IndexReader.Open(dir1, false); IndexReader readOnlyReader = reader.Clone(true); if (!IsReadOnly(readOnlyReader)) { Assert.Fail("reader isn't read only"); } if (DeleteWorked(1, readOnlyReader)) { Assert.Fail("deleting from the original should not have worked"); } // this readonly reader shouldn't have a write lock if (readOnlyReader.hasChanges_ForNUnit) { Assert.Fail("readOnlyReader has a write lock"); } reader.Close(); readOnlyReader.Close(); dir1.Close(); }
public virtual void TestLucene() { int num = 100; Directory indexA = new MockRAMDirectory(); Directory indexB = new MockRAMDirectory(); FillIndex(indexA, 0, num); Assert.IsFalse(VerifyIndex(indexA, 0), "Index a is invalid"); FillIndex(indexB, num, num); Assert.IsFalse(VerifyIndex(indexB, num), "Index b is invalid"); Directory merged = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); writer.MergeFactor = 2; writer.AddIndexesNoOptimize(new []{indexA, indexB}); writer.Optimize(); writer.Close(); var fail = VerifyIndex(merged, 0); merged.Close(); Assert.IsFalse(fail, "The merged index is invalid"); }
public virtual void TestNullOrSubScorer() { Directory dir = new MockRAMDirectory(); IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); Document doc = new Document(); doc.Add(new Field("field", "a b c d", Field.Store.NO, Field.Index.ANALYZED)); w.AddDocument(doc); IndexReader r = w.GetReader(); IndexSearcher s = new IndexSearcher(r); BooleanQuery q = new BooleanQuery(); q.Add(new TermQuery(new Term("field", "a")), Occur.SHOULD); // LUCENE-2617: make sure that a term not in the index still contributes to the score via coord factor float score = s.Search(q, 10).MaxScore; Query subQuery = new TermQuery(new Term("field", "not_in_index")); subQuery.Boost = 0; q.Add(subQuery, Occur.SHOULD); float score2 = s.Search(q, 10).MaxScore; Assert.AreEqual(score * .5, score2, 1e-6); // LUCENE-2617: make sure that a clause not in the index still contributes to the score via coord factor BooleanQuery qq = (BooleanQuery)q.Clone(); PhraseQuery phrase = new PhraseQuery(); phrase.Add(new Term("field", "not_in_index")); phrase.Add(new Term("field", "another_not_in_index")); phrase.Boost = 0; qq.Add(phrase, Occur.SHOULD); score2 = s.Search(qq, 10).MaxScore; Assert.AreEqual(score * (1.0 / 3), score2, 1e-6); // now test BooleanScorer2 subQuery = new TermQuery(new Term("field", "b")); subQuery.Boost = 0; q.Add(subQuery, Occur.MUST); score2 = s.Search(q, 10).MaxScore; Assert.AreEqual(score * (2.0 / 3), score2, 1e-6); // PhraseQuery w/ no terms added returns a null scorer PhraseQuery pq = new PhraseQuery(); q.Add(pq, Occur.SHOULD); Assert.AreEqual(1, s.Search(q, 10).TotalHits); // A required clause which returns null scorer should return null scorer to // IndexSearcher. q = new BooleanQuery(); pq = new PhraseQuery(); q.Add(new TermQuery(new Term("field", "a")), Occur.SHOULD); q.Add(pq, Occur.MUST); Assert.AreEqual(0, s.Search(q, 10).TotalHits); DisjunctionMaxQuery dmq = new DisjunctionMaxQuery(1.0f); dmq.Add(new TermQuery(new Term("field", "a"))); dmq.Add(pq); Assert.AreEqual(1, s.Search(dmq, 10).TotalHits); r.Close(); w.Close(); dir.Close(); }
private void InitBlock(int num, Lucene.Net.Index.IndexWriter writer, Lucene.Net.Store.MockRAMDirectory ramDir, TestRAMDirectory enclosingInstance) { this.num = num; this.writer = writer; this.ramDir = ramDir; this.enclosingInstance = enclosingInstance; }
public virtual void TestNoExtraFiles() { RAMDirectory directory = new MockRAMDirectory(); for (int pass = 0; pass < 2; pass++) { bool autoCommit = pass == 0; IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true); for (int iter = 0; iter < 7; iter++) { ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); for (int j = 0; j < 21; j++) { Document doc = new Document(); doc.Add(new Field("content", "a b c", Field.Store.NO, Field.Index.ANALYZED)); writer.AddDocument(doc); } writer.Close(); TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles autoCommit=" + autoCommit); // Reopen writer = new IndexWriter(directory, autoCommit, ANALYZER, false); } writer.Close(); } directory.Close(); }
public virtual void TestNoExtraFiles() { RAMDirectory directory = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); for (int iter = 0; iter < 7; iter++) { ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); for (int j = 0; j < 21; j++) { Document doc = new Document(); doc.Add(new Field("content", "a b c", Field.Store.NO, Field.Index.ANALYZED)); writer.AddDocument(doc); } writer.Close(); TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles"); // Reopen writer = new IndexWriter(directory, ANALYZER, false, IndexWriter.MaxFieldLength.UNLIMITED); } writer.Close(); directory.Close(); }
public virtual void TestExpungeDeletes() { Directory dir = new MockRAMDirectory(); IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED, null); Document doc = new Document(); doc.Add(new Field("field", "a b c", Field.Store.NO, Field.Index.ANALYZED)); Field id = new Field("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED); doc.Add(id); id.SetValue("0"); w.AddDocument(doc, null); id.SetValue("1"); w.AddDocument(doc, null); w.DeleteDocuments(null, new Term("id", "0")); IndexReader r = w.GetReader(null); w.ExpungeDeletes(null); w.Close(); r.Close(); r = IndexReader.Open(dir, true, null); Assert.AreEqual(1, r.NumDocs()); Assert.IsFalse(r.HasDeletions); r.Close(); dir.Close(); }
public virtual void TestSubclassConcurrentMergeScheduler() { MockRAMDirectory dir = new MockRAMDirectory(); dir.FailOn(new FailOnlyOnMerge()); Document doc = new Document(); Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.Add(idField); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); MyMergeScheduler ms = new MyMergeScheduler(this); writer.SetMergeScheduler(ms); writer.SetMaxBufferedDocs(2); writer.SetRAMBufferSizeMB(Lucene.Net.Index.IndexWriter.DISABLE_AUTO_FLUSH); for (int i = 0; i < 20; i++) { writer.AddDocument(doc); } ms.Sync(); writer.Close(); Assert.IsTrue(mergeThreadCreated); Assert.IsTrue(mergeCalled); Assert.IsTrue(excCalled); dir.Close(); Assert.IsTrue(ConcurrentMergeScheduler.AnyUnhandledExceptions()); }
public virtual void TestLucene() { int num = 100; Directory indexA = new MockRAMDirectory(); Directory indexB = new MockRAMDirectory(); FillIndex(indexA, 0, num); Assert.IsFalse(VerifyIndex(indexA, 0), "Index a is invalid"); FillIndex(indexB, num, num); Assert.IsFalse(VerifyIndex(indexB, num), "Index b is invalid"); Directory merged = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED, null); writer.MergeFactor = 2; writer.AddIndexesNoOptimize(null, new [] { indexA, indexB }); writer.Optimize(null); writer.Close(); var fail = VerifyIndex(merged, 0); merged.Close(); Assert.IsFalse(fail, "The merged index is invalid"); }
public void TestDeletesNumDocs() { Directory dir = new MockRAMDirectory(); IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); doc.Add(new Field("field", "a b c", Field.Store.NO, Field.Index.ANALYZED)); Field id = new Field("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED); doc.Add(id); id.SetValue("0"); w.AddDocument(doc); id.SetValue("1"); w.AddDocument(doc); IndexReader r = w.GetReader(); Assert.AreEqual(2, r.NumDocs()); r.Close(); w.DeleteDocuments(new Term("id", "0")); r = w.GetReader(); Assert.AreEqual(1, r.NumDocs()); r.Close(); w.DeleteDocuments(new Term("id", "1")); r = w.GetReader(); Assert.AreEqual(0, r.NumDocs()); r.Close(); w.Close(); dir.Close(); }
override public void Run() { try { Directory[] dirs = new Directory[Enclosing_Instance.numDirs]; for (int k = 0; k < Enclosing_Instance.numDirs; k++) { dirs[k] = new MockRAMDirectory(Enclosing_Instance.addDir); } //int j = 0; //while (true) { // System.out.println(Thread.currentThread().getName() + ": iter // j=" + j); for (int x = 0; x < numIter; x++) { // only do addIndexesNoOptimize Enclosing_Instance.DoBody(x, dirs); } //if (numIter > 0 && j == numIter) // break; //doBody(j++, dirs); //doBody(5, dirs); //} } catch (System.Exception t) { Enclosing_Instance.Handle(t); } }
public virtual void TestNPESpanQuery() { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(new System.Collections.Hashtable(0)), IndexWriter.MaxFieldLength.LIMITED); // Add documents AddDoc(writer, "1", "the big dogs went running to the market"); AddDoc(writer, "2", "the cat chased the mouse, then the cat ate the mouse quickly"); // Commit writer.Close(); // Get searcher IndexReader reader = IndexReader.Open(dir); IndexSearcher searcher = new IndexSearcher(reader); // Control (make sure docs indexed) Assert.AreEqual(2, HitCount(searcher, "the")); Assert.AreEqual(1, HitCount(searcher, "cat")); Assert.AreEqual(1, HitCount(searcher, "dogs")); Assert.AreEqual(0, HitCount(searcher, "rabbit")); // This throws exception (it shouldn't) Assert.AreEqual(1, searcher.Search(CreateSpan(0, true, new SpanQuery[] { CreateSpan(4, false, "chased", "cat"), CreateSpan("ate") }), 10).TotalHits); reader.Close(); dir.Close(); }
public virtual void TestAddIndexesAndDoDeletesThreads() { int numIter = 5; int numDirs = 3; Directory mainDir = new MockRAMDirectory(); IndexWriter mainWriter = new IndexWriter(mainDir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); mainWriter.SetInfoStream(infoStream); AddDirectoriesThreads addDirThreads = new AddDirectoriesThreads(this, numIter, mainWriter); addDirThreads.LaunchThreads(numDirs); addDirThreads.JoinThreads(); //Assert.AreEqual(100 + numDirs * (3 * numIter / 4) * addDirThreads.NUM_THREADS // * addDirThreads.NUM_INIT_DOCS, addDirThreads.mainWriter.numDocs()); Assert.AreEqual(addDirThreads.count.IntValue(), addDirThreads.mainWriter.NumDocs()); addDirThreads.Close(true); Assert.IsTrue(addDirThreads.failures.Count == 0); _TestUtil.CheckIndex(mainDir); IndexReader reader = IndexReader.Open(mainDir); Assert.AreEqual(addDirThreads.count.IntValue(), reader.NumDocs()); //Assert.AreEqual(100 + numDirs * (3 * numIter / 4) * addDirThreads.NUM_THREADS // * addDirThreads.NUM_INIT_DOCS, reader.numDocs()); reader.Close(); addDirThreads.CloseDir(); mainDir.Close(); }
public virtual void TestCompressionTools() { Fieldable binaryFldCompressed = new Field("binaryCompressed", CompressionTools.Compress(System.Text.UTF8Encoding.UTF8.GetBytes(binaryValCompressed)), Field.Store.YES); Fieldable stringFldCompressed = new Field("stringCompressed", CompressionTools.CompressString(binaryValCompressed), Field.Store.YES); Document doc = new Document(); doc.Add(binaryFldCompressed); doc.Add(stringFldCompressed); /** add the doc to a ram index */ MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.AddDocument(doc); writer.Close(); /** open a reader and fetch the document */ IndexReader reader = IndexReader.Open(dir); Document docFromReader = reader.Document(0); Assert.IsTrue(docFromReader != null); /** fetch the binary compressed field and compare it's content with the original one */ System.String binaryFldCompressedTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(CompressionTools.Decompress(docFromReader.GetBinaryValue("binaryCompressed")))); Assert.IsTrue(binaryFldCompressedTest.Equals(binaryValCompressed)); Assert.IsTrue(CompressionTools.DecompressString(docFromReader.GetBinaryValue("stringCompressed")).Equals(binaryValCompressed)); reader.Close(); dir.Close(); }
public virtual void TestAddIndexes2() { bool optimize = false; Directory dir1 = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer.SetInfoStream(infoStream); // create a 2nd index Directory dir2 = new MockRAMDirectory(); IndexWriter writer2 = new IndexWriter(dir2, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer2.SetInfoStream(infoStream); CreateIndexNoClose(!optimize, "index2", writer2); writer2.Close(); writer.AddIndexesNoOptimize(new Directory[] { dir2 }); writer.AddIndexesNoOptimize(new Directory[] { dir2 }); writer.AddIndexesNoOptimize(new Directory[] { dir2 }); writer.AddIndexesNoOptimize(new Directory[] { dir2 }); writer.AddIndexesNoOptimize(new Directory[] { dir2 }); IndexReader r1 = writer.GetReader(); Assert.AreEqual(500, r1.MaxDoc()); r1.Close(); writer.Close(); dir1.Close(); }
public virtual void TestNullOrSubScorer() { Directory dir = new MockRAMDirectory(); IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); Document doc = new Document(); doc.Add(new Field("field", "a b c d", Field.Store.NO, Field.Index.ANALYZED)); w.AddDocument(doc); IndexReader r = w.GetReader(); IndexSearcher s = new IndexSearcher(r); BooleanQuery q = new BooleanQuery(); q.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD); // PhraseQuery w/ no terms added returns a null scorer PhraseQuery pq = new PhraseQuery(); q.Add(pq, BooleanClause.Occur.SHOULD); Assert.AreEqual(1, s.Search(q, 10).TotalHits); // A required clause which returns null scorer should return null scorer to // IndexSearcher. q = new BooleanQuery(); pq = new PhraseQuery(); q.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD); q.Add(pq, BooleanClause.Occur.MUST); Assert.AreEqual(0, s.Search(q, 10).TotalHits); DisjunctionMaxQuery dmq = new DisjunctionMaxQuery(1.0f); dmq.Add(new TermQuery(new Term("field", "a"))); dmq.Add(pq); Assert.AreEqual(1, s.Search(dmq, 10).TotalHits); r.Close(); w.Close(); dir.Close(); }
public virtual void TestThreadedOptimize_Renamed() { Directory directory = new MockRAMDirectory(); runTest(directory, false, new SerialMergeScheduler()); runTest(directory, true, new SerialMergeScheduler()); runTest(directory, false, new ConcurrentMergeScheduler()); runTest(directory, true, new ConcurrentMergeScheduler()); directory.Close(); System.String tempDir = SupportClass.AppSettings.Get("tempDir", ""); if (tempDir == null) { throw new System.IO.IOException("tempDir undefined, cannot run test"); } System.String dirName = tempDir + "/luceneTestThreadedOptimize"; directory = FSDirectory.Open(new System.IO.FileInfo(dirName)); runTest(directory, false, new SerialMergeScheduler()); runTest(directory, true, new SerialMergeScheduler()); runTest(directory, false, new ConcurrentMergeScheduler()); runTest(directory, true, new ConcurrentMergeScheduler()); directory.Close(); _TestUtil.RmDir(dirName); }
public virtual void TestRandomExceptionsThreads() { random = new Random((int)(DateTime.Now.Ticks & 0x7fffffff)); MockRAMDirectory dir = new MockRAMDirectory(); MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); ((ConcurrentMergeScheduler)writer.MergeScheduler).SetSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.SetRAMBufferSizeMB(0.2); if (DEBUG) { System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; writer.SetInfoStream(temp_writer, null); } int NUM_THREADS = 4; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(this, i, writer); threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Join(); } for (int i = 0; i < NUM_THREADS; i++) { Assert.IsNull(threads[i].failure, "thread " + threads[i].Name + ": hit unexpected failure"); } writer.Commit(null); try { writer.Close(); } catch (System.Exception t) { System.Console.Out.WriteLine("exception during close:"); System.Console.Out.WriteLine(t.StackTrace); writer.Rollback(null); } // Confirm that when doc hits exception partway through tokenization, it's deleted: IndexReader r2 = IndexReader.Open((Directory)dir, true, null); int count = r2.DocFreq(new Term("content4", "aaa"), null); int count2 = r2.DocFreq(new Term("content4", "ddd"), null); Assert.AreEqual(count, count2); r2.Close(); _TestUtil.CheckIndex(dir); }
public virtual void TestIncompatibleIndexes() { // two documents: Directory dir1 = GetDir1(); // one document only: Directory dir2 = new MockRAMDirectory(); IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); Document d3 = new Document(); d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED)); w2.AddDocument(d3); w2.Close(); ParallelReader pr = new ParallelReader(); pr.Add(IndexReader.Open(dir1)); try { pr.Add(IndexReader.Open(dir2)); Assert.Fail("didn't get exptected exception: indexes don't have same number of documents"); } catch (System.ArgumentException e) { // expected exception } }
public virtual void TestCloneWithSetNorm() { Directory dir1 = new MockRAMDirectory(); TestIndexReaderReopen.CreateIndex(dir1, false); IndexReader orig = IndexReader.Open(dir1, false); orig.SetNorm(1, "field1", 17.0f); byte encoded = Similarity.EncodeNorm(17.0f); Assert.AreEqual(encoded, orig.Norms("field1")[1]); // the cloned segmentreader should have 2 references, 1 to itself, and 1 to // the original segmentreader IndexReader clonedReader = (IndexReader)orig.Clone(); orig.Close(); clonedReader.Close(); IndexReader r = IndexReader.Open(dir1, false); Assert.AreEqual(encoded, r.Norms("field1")[1]); r.Close(); dir1.Close(); }
public virtual void TestLucene() { int num = 100; Directory indexA = new MockRAMDirectory(); Directory indexB = new MockRAMDirectory(); FillIndex(indexA, 0, num); bool fail = VerifyIndex(indexA, 0); if (fail) { Assert.Fail("Index a is invalid"); } FillIndex(indexB, num, num); fail = VerifyIndex(indexB, num); if (fail) { Assert.Fail("Index b is invalid"); } Directory merged = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMergeFactor(2); writer.AddIndexes(new Directory[]{indexA, indexB}); writer.Close(); fail = VerifyIndex(merged, 0); merged.Close(); Assert.IsFalse(fail, "The merged index is invalid"); }
public virtual void TestNoPrxFile() { Directory ram = new MockRAMDirectory(); Analyzer analyzer = new StandardAnalyzer(); IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(3); writer.SetMergeFactor(2); writer.SetUseCompoundFile(false); Document d = new Document(); Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED); f1.SetOmitTermFreqAndPositions(true); d.Add(f1); for (int i = 0; i < 30; i++) { writer.AddDocument(d); } writer.Commit(); AssertNoPrx(ram); // force merge writer.Optimize(); // flush writer.Close(); AssertNoPrx(ram); _TestUtil.CheckIndex(ram); ram.Close(); }
public override void Eval(MockRAMDirectory dir) { if (!failed) { failed = true; throw new System.IO.IOException("fail in add doc"); } }
public virtual void TestMixedMerge() { Directory ram = new MockRAMDirectory(); Analyzer analyzer = new StandardAnalyzer(); IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(3); writer.SetMergeFactor(2); Document d = new Document(); // this field will have Tf Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED); d.Add(f1); // this field will NOT have Tf Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED); f2.SetOmitTermFreqAndPositions(true); d.Add(f2); for (int i = 0; i < 30; i++) { writer.AddDocument(d); } // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger // keep things constant d = new Document(); // Reverese f1.SetOmitTermFreqAndPositions(true); d.Add(f1); f2.SetOmitTermFreqAndPositions(false); d.Add(f2); for (int i = 0; i < 30; i++) { writer.AddDocument(d); } // force merge writer.Optimize(); // flush writer.Close(); _TestUtil.CheckIndex(ram); SegmentReader reader = SegmentReader.GetOnlySegmentReader(ram); FieldInfos fi = reader.FieldInfos(); Assert.IsTrue(fi.FieldInfo("f1").omitTermFreqAndPositions_ForNUnit, "OmitTermFreqAndPositions field bit should be set."); Assert.IsTrue(fi.FieldInfo("f2").omitTermFreqAndPositions_ForNUnit, "OmitTermFreqAndPositions field bit should be set."); reader.Close(); ram.Close(); }
private void Crash(IndexWriter writer) { MockRAMDirectory dir = (MockRAMDirectory)writer.GetDirectory(); ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)writer.GetMergeScheduler(); dir.Crash(); cms.Sync(); dir.ClearCrash(); }
public virtual void TestDeleteFromIndexWriter() { bool optimize = true; Directory dir1 = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED, null); writer.ReaderTermsIndexDivisor = 2; writer.SetInfoStream(infoStream, null); // create the index CreateIndexNoClose(!optimize, "index1", writer); writer.Flush(false, true, true, null); // get a reader IndexReader r1 = writer.GetReader(null); System.String id10 = r1.Document(10, null).GetField("id").StringValue(null); // deleted IW docs should not show up in the next getReader writer.DeleteDocuments(null, new Term("id", id10)); IndexReader r2 = writer.GetReader(null); Assert.AreEqual(1, Count(new Term("id", id10), r1)); Assert.AreEqual(0, Count(new Term("id", id10), r2)); System.String id50 = r1.Document(50, null).GetField("id").StringValue(null); Assert.AreEqual(1, Count(new Term("id", id50), r1)); writer.DeleteDocuments(null, new Term("id", id50)); IndexReader r3 = writer.GetReader(null); Assert.AreEqual(0, Count(new Term("id", id10), r3)); Assert.AreEqual(0, Count(new Term("id", id50), r3)); System.String id75 = r1.Document(75, null).GetField("id").StringValue(null); writer.DeleteDocuments(null, new TermQuery(new Term("id", id75))); IndexReader r4 = writer.GetReader(null); Assert.AreEqual(1, Count(new Term("id", id75), r3)); Assert.AreEqual(0, Count(new Term("id", id75), r4)); r1.Close(); r2.Close(); r3.Close(); r4.Close(); writer.Close(); // reopen the writer to verify the delete made it to the directory writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED, null); writer.SetInfoStream(infoStream, null); IndexReader w2r1 = writer.GetReader(null); Assert.AreEqual(0, Count(new Term("id", id10), w2r1)); w2r1.Close(); writer.Close(); dir1.Close(); }
/// <summary> Tests creating a segment, then check to insure the segment can be seen via /// IW.getReader /// </summary> public virtual void DoTestIndexWriterReopenSegment(bool optimize) { Directory dir1 = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer.SetInfoStream(infoStream); IndexReader r1 = writer.GetReader(); Assert.AreEqual(0, r1.MaxDoc()); CreateIndexNoClose(false, "index1", writer); writer.Flush(!optimize, true, true); IndexReader iwr1 = writer.GetReader(); Assert.AreEqual(100, iwr1.MaxDoc()); IndexReader r2 = writer.GetReader(); Assert.AreEqual(r2.MaxDoc(), 100); // add 100 documents for (int x = 10000; x < 10000 + 100; x++) { Document d = CreateDocument(x, "index1", 5); writer.AddDocument(d); } writer.Flush(false, true, true); // verify the reader was reopened internally IndexReader iwr2 = writer.GetReader(); Assert.IsTrue(iwr2 != r1); Assert.AreEqual(200, iwr2.MaxDoc()); // should have flushed out a segment IndexReader r3 = writer.GetReader(); Assert.IsTrue(r2 != r3); Assert.AreEqual(200, r3.MaxDoc()); // dec ref the readers rather than close them because // closing flushes changes to the writer r1.Close(); iwr1.Close(); r2.Close(); r3.Close(); iwr2.Close(); writer.Close(); // test whether the changes made it to the directory writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); IndexReader w2r1 = writer.GetReader(); // insure the deletes were actually flushed to the directory Assert.AreEqual(200, w2r1.MaxDoc()); w2r1.Close(); writer.Close(); dir1.Close(); }
public virtual void TestFlushExceptions() { MockRAMDirectory directory = new MockRAMDirectory(); FailOnlyOnFlush failure = new FailOnlyOnFlush(); directory.FailOn(failure); IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); Document doc = new Document(); Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.Add(idField); int extraCount = 0; for (int i = 0; i < 10; i++) { for (int j = 0; j < 20; j++) { idField.SetValue(System.Convert.ToString(i * 20 + j)); writer.AddDocument(doc); } while (true) { // must cycle here because sometimes the merge flushes // the doc we just added and so there's nothing to // flush, and we don't hit the exception writer.AddDocument(doc); failure.SetDoFail(); try { writer.Flush(true, false, true); if (failure.hitExc) { Assert.Fail("failed to hit IOException"); } extraCount++; } catch (System.IO.IOException ioe) { failure.ClearDoFail(); break; } } } writer.Close(); IndexReader reader = IndexReader.Open(directory, true); Assert.AreEqual(200 + extraCount, reader.NumDocs()); reader.Close(); directory.Close(); }
public virtual void TestDuringAddDelete() { Directory dir1 = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer.SetInfoStream(infoStream); writer.SetMergeFactor(2); // create the index CreateIndexNoClose(false, "test", writer); writer.Commit(); IndexReader r = writer.GetReader(); int NUM_THREAD = 5; float SECONDS = 3; long endTime = (long)((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) + 1000.0 * SECONDS); System.Collections.IList excs = (System.Collections.IList)System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(new System.Collections.ArrayList())); SupportClass.ThreadClass[] threads = new SupportClass.ThreadClass[NUM_THREAD]; for (int i = 0; i < NUM_THREAD; i++) { threads[i] = new AnonymousClassThread1(endTime, writer, excs, this); threads[i].IsBackground = true; threads[i].Start(); } int sum = 0; while ((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) < endTime) { IndexReader r2 = r.Reopen(); if (r2 != r) { r.Close(); r = r2; } Query q = new TermQuery(new Term("indexname", "test")); sum += new IndexSearcher(r).Search(q, 10).totalHits; } for (int i = 0; i < NUM_THREAD; i++) { threads[i].Join(); } Assert.IsTrue(sum > 0); Assert.AreEqual(0, excs.Count); writer.Close(); _TestUtil.CheckIndex(dir1); r.Close(); dir1.Close(); }
public virtual void TestCrashWhileIndexing() { IndexWriter writer = InitIndex(); MockRAMDirectory dir = (MockRAMDirectory)writer.GetDirectory(); Crash(writer); IndexReader reader = IndexReader.Open(dir); Assert.IsTrue(reader.NumDocs() < 157); }
public virtual void TestFilterIndexReader_Renamed() { RAMDirectory directory = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); Document d1 = new Document(); d1.Add(new Field("default", "one two", Field.Store.YES, Field.Index.ANALYZED)); writer.AddDocument(d1); Document d2 = new Document(); d2.Add(new Field("default", "one three", Field.Store.YES, Field.Index.ANALYZED)); writer.AddDocument(d2); Document d3 = new Document(); d3.Add(new Field("default", "two four", Field.Store.YES, Field.Index.ANALYZED)); writer.AddDocument(d3); writer.Close(); IndexReader reader = new TestReader(IndexReader.Open(directory)); Assert.IsTrue(reader.IsOptimized()); TermEnum terms = reader.Terms(); while (terms.Next()) { Assert.IsTrue(terms.Term().Text().IndexOf('e') != -1); } terms.Close(); TermPositions positions = reader.TermPositions(new Term("default", "one")); while (positions.Next()) { Assert.IsTrue((positions.Doc() % 2) == 1); } int NUM_DOCS = 3; TermDocs td = reader.TermDocs(null); for (int i = 0; i < NUM_DOCS; i++) { Assert.IsTrue(td.Next()); Assert.AreEqual(i, td.Doc()); Assert.AreEqual(1, td.Freq()); } td.Close(); reader.Close(); directory.Close(); }
public virtual void TestNoWaitClose() { RAMDirectory directory = new MockRAMDirectory(); Document doc = new Document(); Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.Add(idField); for (int pass = 0; pass < 2; pass++) { bool autoCommit = pass == 0; IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true); for (int iter = 0; iter < 10; iter++) { ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); writer.SetMergeFactor(100); for (int j = 0; j < 201; j++) { idField.SetValue(System.Convert.ToString(iter * 201 + j)); writer.AddDocument(doc); } int delID = iter * 201; for (int j = 0; j < 20; j++) { writer.DeleteDocuments(new Term("id", System.Convert.ToString(delID))); delID += 5; } // Force a bunch of merge threads to kick off so we // stress out aborting them on close: writer.SetMergeFactor(3); writer.AddDocument(doc); writer.Flush(); writer.Close(false); IndexReader reader = IndexReader.Open(directory); Assert.AreEqual((1 + iter) * 182, reader.NumDocs()); reader.Close(); // Reopen writer = new IndexWriter(directory, autoCommit, ANALYZER, false); } writer.Close(); } directory.Close(); }
public override void Eval(MockRAMDirectory dir) { if (doFail) { System.Diagnostics.StackTrace st = new System.Diagnostics.StackTrace(); foreach (System.Diagnostics.StackFrame f in st.GetFrames()) { if ("DoFlush" == f.GetMethod().Name) throw new System.IO.IOException("now failing during flush"); } } }
public virtual void TestFlushExceptions() { MockRAMDirectory directory = new MockRAMDirectory(); FailOnlyOnFlush failure = new FailOnlyOnFlush(); directory.FailOn(failure); IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); Document doc = new Document(); Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.Add(idField); int extraCount = 0; for (int i = 0; i < 10; i++) { for (int j = 0; j < 20; j++) { idField.SetValue(System.Convert.ToString(i*20 + j)); writer.AddDocument(doc); } while (true) { // must cycle here because sometimes the merge flushes // the doc we just added and so there's nothing to // flush, and we don't hit the exception writer.AddDocument(doc); failure.SetDoFail(); try { writer.Flush(true, false, true); if (failure.hitExc) Assert.Fail("failed to hit IOException"); extraCount++; } catch (System.IO.IOException ioe) { failure.ClearDoFail(); break; } } } writer.Close(); IndexReader reader = IndexReader.Open(directory, true); Assert.AreEqual(200 + extraCount, reader.NumDocs()); reader.Close(); directory.Close(); }
public virtual void TestCommitUserData() { RAMDirectory d = new MockRAMDirectory(); System.Collections.Generic.IDictionary<string, string> commitUserData = new System.Collections.Generic.Dictionary<string,string>(); commitUserData["foo"] = "fighters"; // set up writer IndexWriter writer = new IndexWriter(d, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(2); for (int i = 0; i < 27; i++) AddDocumentWithFields(writer); writer.Close(); IndexReader r = IndexReader.Open(d, false); r.DeleteDocument(5); r.Flush(commitUserData); r.Close(); SegmentInfos sis = new SegmentInfos(); sis.Read(d); IndexReader r2 = IndexReader.Open(d, false); IndexCommit c = r.IndexCommit; Assert.AreEqual(c.UserData, commitUserData); Assert.AreEqual(sis.GetCurrentSegmentFileName(), c.SegmentsFileName); Assert.IsTrue(c.Equals(r.IndexCommit)); // Change the index writer = new IndexWriter(d, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(2); for (int i = 0; i < 7; i++) AddDocumentWithFields(writer); writer.Close(); IndexReader r3 = r2.Reopen(); Assert.IsFalse(c.Equals(r3.IndexCommit)); Assert.IsFalse(r2.IndexCommit.IsOptimized); r3.Close(); writer = new IndexWriter(d, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.LIMITED); writer.Optimize(); writer.Close(); r3 = r2.Reopen(); Assert.IsTrue(r3.IndexCommit.IsOptimized); r2.Close(); r3.Close(); d.Close(); }
public virtual void TestCloneNoChangesStillReadOnly() { Directory dir1 = new MockRAMDirectory(); TestIndexReaderReopen.CreateIndex(dir1, true); IndexReader r1 = IndexReader.Open(dir1, false); IndexReader r2 = r1.Clone(false); Assert.IsTrue(DeleteWorked(1, r2), "deleting from the cloned should have worked"); r1.Close(); r2.Close(); dir1.Close(); }
public virtual void TestCloneWriteToOrig() { Directory dir1 = new MockRAMDirectory(); TestIndexReaderReopen.CreateIndex(dir1, true); IndexReader r1 = IndexReader.Open(dir1, false); IndexReader r2 = r1.Clone(false); Assert.IsTrue(DeleteWorked(1, r1), "deleting from the original should have worked"); r1.Close(); r2.Close(); dir1.Close(); }
public virtual void TestCloneReadOnlySegmentReader() { Directory dir1 = new MockRAMDirectory(); TestIndexReaderReopen.CreateIndex(dir1, false); IndexReader reader = IndexReader.Open(dir1, false); IndexReader readOnlyReader = reader.Clone(true); Assert.IsTrue(IsReadOnly(readOnlyReader), "reader isn't read only"); Assert.IsFalse(DeleteWorked(1, readOnlyReader), "deleting from the original should not have worked"); reader.Close(); readOnlyReader.Close(); dir1.Close(); }
public override void Eval(MockRAMDirectory dir) { if (doFail) { System.Diagnostics.StackTrace trace = new System.Diagnostics.StackTrace(); for (int i = 0; i < trace.FrameCount; i++) { System.Diagnostics.StackFrame sf = trace.GetFrame(i); if ("DoFlush".Equals(sf.GetMethod().Name)) { //new RuntimeException().printStackTrace(System.out); throw new System.IO.IOException("now failing during flush"); } } } }
public virtual void TestPrevTermAtEnd() { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); AddDoc(writer, "aaa bbb"); writer.Close(); IndexReader reader = IndexReader.Open(dir); SegmentTermEnum termEnum = (SegmentTermEnum) reader.Terms(); Assert.IsTrue(termEnum.Next()); Assert.AreEqual("aaa", termEnum.Term().Text()); Assert.IsTrue(termEnum.Next()); Assert.AreEqual("aaa", termEnum.Prev().Text()); Assert.AreEqual("bbb", termEnum.Term().Text()); Assert.IsFalse(termEnum.Next()); Assert.AreEqual("bbb", termEnum.Prev().Text()); }
private IndexWriter InitIndex(MockRAMDirectory dir) { dir.SetLockFactory(NoLockFactory.Instance); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); //writer.setMaxBufferedDocs(2); writer.SetMaxBufferedDocs(10); ((ConcurrentMergeScheduler) writer.MergeScheduler).SetSuppressExceptions(); Document doc = new Document(); doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED)); doc.Add(new Field("id", "0", Field.Store.YES, Field.Index.ANALYZED)); for (int i = 0; i < 157; i++) writer.AddDocument(doc); return writer; }
public override void Eval(MockRAMDirectory dir) { if (doFail && !(Thread.CurrentThread.Name ?? "").Contains("Merge Thread")) { System.Diagnostics.StackTrace trace = new System.Diagnostics.StackTrace(); for (int i = 0; i < trace.FrameCount; i++) { System.Diagnostics.StackFrame sf = trace.GetFrame(i); if ("DoFlush".Equals(sf.GetMethod().Name)) { hitExc = true; //Console.WriteLine(trace); throw new System.IO.IOException("now failing during flush"); } } } }
public virtual void TestBinaryFieldInIndex() { IFieldable binaryFldStored = new Field("binaryStored", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValStored), Field.Store.YES); IFieldable stringFldStored = new Field("stringStored", binaryValStored, Field.Store.YES, Field.Index.NO, Field.TermVector.NO); // binary fields with store off are not allowed Assert.Throws<ArgumentException>( () => new Field("fail", System.Text.Encoding.UTF8.GetBytes(binaryValStored), Field.Store.NO)); Document doc = new Document(); doc.Add(binaryFldStored); doc.Add(stringFldStored); /** test for field count */ Assert.AreEqual(2, doc.fields_ForNUnit.Count); /** add the doc to a ram index */ MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); writer.AddDocument(doc); writer.Close(); /** open a reader and fetch the document */ IndexReader reader = IndexReader.Open(dir, false); Document docFromReader = reader.Document(0); Assert.IsTrue(docFromReader != null); /** fetch the binary stored field and compare it's content with the original one */ System.String binaryFldStoredTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(docFromReader.GetBinaryValue("binaryStored"))); Assert.IsTrue(binaryFldStoredTest.Equals(binaryValStored)); /** fetch the string field and compare it's content with the original one */ System.String stringFldStoredTest = docFromReader.Get("stringStored"); Assert.IsTrue(stringFldStoredTest.Equals(binaryValStored)); /** delete the document from index */ reader.DeleteDocument(0); Assert.AreEqual(0, reader.NumDocs()); reader.Close(); dir.Close(); }
public virtual void TestFlushExceptions() { MockRAMDirectory directory = new MockRAMDirectory(); FailOnlyOnFlush failure = new FailOnlyOnFlush(); directory.FailOn(failure); IndexWriter writer = new IndexWriter(directory, true, ANALYZER, true); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); Document doc = new Document(); Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.Add(idField); for (int i = 0; i < 10; i++) { for (int j = 0; j < 20; j++) { idField.SetValue(System.Convert.ToString(i * 20 + j)); writer.AddDocument(doc); } writer.AddDocument(doc); failure.SetDoFail(); try { writer.Flush(); Assert.Fail("failed to hit IOException"); } catch (System.IO.IOException ioe) { failure.ClearDoFail(); } } writer.Close(); IndexReader reader = IndexReader.Open(directory); Assert.AreEqual(200, reader.NumDocs()); reader.Close(); directory.Close(); }
public virtual void TestDeletedDocs() { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); writer.SetMaxBufferedDocs(2); Document doc = new Document(); doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Close(); IndexReader reader = IndexReader.Open(dir); reader.DeleteDocument(5); reader.Close(); CheckIndex.out_Renamed = new System.IO.StringWriter(); bool condition = CheckIndex.Check(dir, false); String message = CheckIndex.out_Renamed.ToString(); Assert.IsTrue(condition, message); }
public virtual void TestIndexing() { Directory mainDir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(mainDir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer.UseCompoundFile = false; IndexReader reader = writer.GetReader(); // start pooling readers reader.Close(); writer.MergeFactor = 2; writer.SetMaxBufferedDocs(10); RunThread[] indexThreads = new RunThread[4]; for (int x = 0; x < indexThreads.Length; x++) { indexThreads[x] = new RunThread(this, x % 2, writer); indexThreads[x].Name = "Thread " + x; indexThreads[x].Start(); } long startTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); long duration = 5 * 1000; while (((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) - startTime) < duration) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 100)); } int delCount = 0; int addCount = 0; for (int x = 0; x < indexThreads.Length; x++) { indexThreads[x].run_Renamed_Field = false; Assert.IsTrue(indexThreads[x].ex == null); addCount += indexThreads[x].addCount; delCount += indexThreads[x].delCount; } for (int x = 0; x < indexThreads.Length; x++) { indexThreads[x].Join(); } //System.out.println("addCount:"+addCount); //System.out.println("delCount:"+delCount); writer.Close(); mainDir.Close(); }
public virtual void TestMissingTerms() { System.String fieldName = "field1"; MockRAMDirectory rd = new MockRAMDirectory(); IndexWriter w = new IndexWriter(rd, new KeywordAnalyzer(), MaxFieldLength.UNLIMITED); for (int i = 0; i < 100; i++) { Document doc = new Document(); int term = i * 10; //terms are units of 10; doc.Add(new Field(fieldName, "" + term, Field.Store.YES, Field.Index.NOT_ANALYZED)); w.AddDocument(doc); } w.Close(); IndexReader reader = IndexReader.Open(rd, true); IndexSearcher searcher = new IndexSearcher(reader); int numDocs = reader.NumDocs(); ScoreDoc[] results; MatchAllDocsQuery q = new MatchAllDocsQuery(); System.Collections.ArrayList terms = new System.Collections.ArrayList(); terms.Add("5"); results = searcher.Search(q, new FieldCacheTermsFilter(fieldName, (System.String[]) terms.ToArray(typeof(System.String))), numDocs).ScoreDocs; Assert.AreEqual(0, results.Length, "Must match nothing"); terms = new System.Collections.ArrayList(); terms.Add("10"); results = searcher.Search(q, new FieldCacheTermsFilter(fieldName, (System.String[])terms.ToArray(typeof(System.String))), numDocs).ScoreDocs; Assert.AreEqual(1, results.Length, "Must match 1"); terms = new System.Collections.ArrayList(); terms.Add("10"); terms.Add("20"); results = searcher.Search(q, new FieldCacheTermsFilter(fieldName, (System.String[]) terms.ToArray(typeof(System.String))), numDocs).ScoreDocs; Assert.AreEqual(2, results.Length, "Must match 2"); reader.Close(); rd.Close(); }
public override void Eval(MockRAMDirectory dir) { if (sawMaybe && !failed) { bool seen = false; System.Diagnostics.StackFrame[] frames = new System.Diagnostics.StackTrace().GetFrames(); for (int i = 0; i < frames.Length; i++) { System.String methodName = frames[i].GetMethod().Name; if ("ApplyDeletes".Equals(methodName)) { seen = true; break; } } if (!seen) { // Only fail once we are no longer in applyDeletes failed = true; throw new System.IO.IOException("fail after applyDeletes"); } } if (!failed) { System.Diagnostics.StackFrame[] frames = new System.Diagnostics.StackTrace().GetFrames(); for (int i = 0; i < frames.Length; i++) { System.String methodName = frames[i].GetMethod().Name; if ("ApplyDeletes".Equals(methodName)) { sawMaybe = true; break; } } } }
public override void Eval(MockRAMDirectory dir) { if (sawMaybe && !failed) { bool seen = false; System.Diagnostics.StackTrace trace = new System.Diagnostics.StackTrace(); for (int i = 0; i < trace.FrameCount; i++) { System.Diagnostics.StackFrame sf = trace.GetFrame(i); if ("ApplyDeletes".Equals(sf.GetMethod().Name)) { seen = true; break; } } if (!seen) { // Only fail once we are no longer in applyDeletes failed = true; throw new System.IO.IOException("fail after applyDeletes"); } } if (!failed) { System.Diagnostics.StackTrace trace = new System.Diagnostics.StackTrace(); for (int i = 0; i < trace.FrameCount; i++) { System.Diagnostics.StackFrame sf = trace.GetFrame(i); if ("ApplyDeletes".Equals(sf.GetMethod().Name)) { sawMaybe = true; break; } } } }
public override /*virtual*/ void Eval(MockRAMDirectory dir) { System.Diagnostics.StackTrace trace = new System.Diagnostics.StackTrace(); for (int i = 0; i < trace.FrameCount; i++) { System.Diagnostics.StackFrame sf = trace.GetFrame(i); if ("DoMerge".Equals(sf.GetMethod().Name)) throw new System.IO.IOException("now failing during merge"); } }
public virtual void TestSubclassConcurrentMergeScheduler() { MockRAMDirectory dir = new MockRAMDirectory(); dir.FailOn(new FailOnlyOnMerge()); Document doc = new Document(); Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.Add(idField); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); MyMergeScheduler ms = new MyMergeScheduler(this); writer.SetMergeScheduler(ms); writer.SetMaxBufferedDocs(2); writer.SetRAMBufferSizeMB(Lucene.Net.Index.IndexWriter.DISABLE_AUTO_FLUSH); for (int i = 0; i < 20; i++) writer.AddDocument(doc); ms.Sync(); writer.Close(); Assert.IsTrue(mergeThreadCreated); Assert.IsTrue(mergeCalled); Assert.IsTrue(excCalled); dir.Close(); Assert.IsTrue(ConcurrentMergeScheduler.AnyUnhandledExceptions()); }
public virtual void TestDeletedDocs() { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(2); Document doc = new Document(); doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Close(); IndexReader reader = IndexReader.Open(dir, false); reader.DeleteDocument(5); reader.Close(); System.IO.MemoryStream bos = new System.IO.MemoryStream(1024); CheckIndex checker = new CheckIndex(dir); checker.SetInfoStream(new System.IO.StreamWriter(bos)); //checker.setInfoStream(System.out); CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method(); if (indexStatus.clean == false) { System.Console.Out.WriteLine("CheckIndex failed"); char[] tmpChar; byte[] tmpByte; tmpByte = bos.GetBuffer(); tmpChar = new char[bos.Length]; System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length); System.Console.Out.WriteLine(new System.String(tmpChar)); Assert.Fail(); } CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus) indexStatus.segmentInfos[0]; Assert.IsTrue(seg.openReaderPassed); Assert.IsNotNull(seg.diagnostics); Assert.IsNotNull(seg.fieldNormStatus); Assert.IsNull(seg.fieldNormStatus.error); Assert.AreEqual(1, seg.fieldNormStatus.totFields); Assert.IsNotNull(seg.termIndexStatus); Assert.IsNull(seg.termIndexStatus.error); Assert.AreEqual(1, seg.termIndexStatus.termCount); Assert.AreEqual(19, seg.termIndexStatus.totFreq); Assert.AreEqual(18, seg.termIndexStatus.totPos); Assert.IsNotNull(seg.storedFieldStatus); Assert.IsNull(seg.storedFieldStatus.error); Assert.AreEqual(18, seg.storedFieldStatus.docCount); Assert.AreEqual(18, seg.storedFieldStatus.totFields); Assert.IsNotNull(seg.termVectorStatus); Assert.IsNull(seg.termVectorStatus.error); Assert.AreEqual(18, seg.termVectorStatus.docCount); Assert.AreEqual(18, seg.termVectorStatus.totVectors); Assert.IsTrue(seg.diagnostics.Count > 0); List<string> onlySegments = new List<string>(); onlySegments.Add("_0"); Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true); }
public virtual void TestTermVectorsFieldOrder() { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); doc.Add(new Field("c", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.Add(new Field("a", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.Add(new Field("b", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.Add(new Field("x", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); writer.AddDocument(doc); writer.Close(); IndexReader reader = IndexReader.Open(dir, true); ITermFreqVector[] v = reader.GetTermFreqVectors(0); Assert.AreEqual(4, v.Length); System.String[] expectedFields = new System.String[]{"a", "b", "c", "x"}; int[] expectedPositions = new int[]{1, 2, 0}; for (int i = 0; i < v.Length; i++) { TermPositionVector posVec = (TermPositionVector) v[i]; Assert.AreEqual(expectedFields[i], posVec.Field); System.String[] terms = posVec.GetTerms(); Assert.AreEqual(3, terms.Length); Assert.AreEqual("content", terms[0]); Assert.AreEqual("here", terms[1]); Assert.AreEqual("some", terms[2]); for (int j = 0; j < 3; j++) { int[] positions = posVec.GetTermPositions(j); Assert.AreEqual(1, positions.Length); Assert.AreEqual(expectedPositions[j], positions[0]); } } }
public virtual void TestTermQueryMultiSearcherExplain() { // creating two directories for indices Directory indexStoreA = new MockRAMDirectory(); Directory indexStoreB = new MockRAMDirectory(); Document lDoc = new Document(); lDoc.Add(new Field("handle", "1 2", Field.Store.YES, Field.Index.ANALYZED)); Document lDoc2 = new Document(); lDoc2.Add(new Field("handle", "1 2", Field.Store.YES, Field.Index.ANALYZED)); Document lDoc3 = new Document(); lDoc3.Add(new Field("handle", "1 2", Field.Store.YES, Field.Index.ANALYZED)); IndexWriter writerA = new IndexWriter(indexStoreA, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); IndexWriter writerB = new IndexWriter(indexStoreB, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); writerA.AddDocument(lDoc); writerA.AddDocument(lDoc2); writerA.Optimize(); writerA.Close(); writerB.AddDocument(lDoc3); writerB.Close(); QueryParser parser = new QueryParser(Util.Version.LUCENE_CURRENT, "fulltext", new StandardAnalyzer(Util.Version.LUCENE_CURRENT)); Query query = parser.Parse("handle:1"); Searcher[] searchers = new Searcher[2]; searchers[0] = new IndexSearcher(indexStoreB, true); searchers[1] = new IndexSearcher(indexStoreA, true); Searcher mSearcher = new MultiSearcher(searchers); ScoreDoc[] hits = mSearcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length); Explanation explain = mSearcher.Explain(query, hits[0].Doc); System.String exp = explain.ToString(0); Assert.IsTrue(exp.IndexOf("maxDocs=3") > - 1, exp); Assert.IsTrue(exp.IndexOf("docFreq=3") > - 1, exp); query = parser.Parse("handle:\"1 2\""); hits = mSearcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length); explain = mSearcher.Explain(query, hits[0].Doc); exp = explain.ToString(0); Assert.IsTrue(exp.IndexOf("1=3") > - 1, exp); Assert.IsTrue(exp.IndexOf("2=3") > - 1, exp); query = new SpanNearQuery(new SpanQuery[]{new SpanTermQuery(new Term("handle", "1")), new SpanTermQuery(new Term("handle", "2"))}, 0, true); hits = mSearcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length); explain = mSearcher.Explain(query, hits[0].Doc); exp = explain.ToString(0); Assert.IsTrue(exp.IndexOf("1=3") > - 1, exp); Assert.IsTrue(exp.IndexOf("2=3") > - 1, exp); mSearcher.Close(); }
public virtual void TestKnownSetOfDocuments() { System.String test1 = "eating chocolate in a computer lab"; //6 terms System.String test2 = "computer in a computer lab"; //5 terms System.String test3 = "a chocolate lab grows old"; //5 terms System.String test4 = "eating chocolate with a chocolate lab in an old chocolate colored computer lab"; //13 terms System.Collections.IDictionary test4Map = new System.Collections.Hashtable(); test4Map["chocolate"] = 3; test4Map["lab"] = 2; test4Map["eating"] = 1; test4Map["computer"] = 1; test4Map["with"] = 1; test4Map["a"] = 1; test4Map["colored"] = 1; test4Map["in"] = 1; test4Map["an"] = 1; test4Map["computer"] = 1; test4Map["old"] = 1; Document testDoc1 = new Document(); SetupDoc(testDoc1, test1); Document testDoc2 = new Document(); SetupDoc(testDoc2, test2); Document testDoc3 = new Document(); SetupDoc(testDoc3, test3); Document testDoc4 = new Document(); SetupDoc(testDoc4, test4); Directory dir = new MockRAMDirectory(); try { IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); Assert.IsTrue(writer != null); writer.AddDocument(testDoc1); writer.AddDocument(testDoc2); writer.AddDocument(testDoc3); writer.AddDocument(testDoc4); writer.Close(); IndexSearcher knownSearcher = new IndexSearcher(dir, true); TermEnum termEnum = knownSearcher.reader_ForNUnit.Terms(); TermDocs termDocs = knownSearcher.reader_ForNUnit.TermDocs(); //System.out.println("Terms: " + termEnum.size() + " Orig Len: " + termArray.length); Similarity sim = knownSearcher.Similarity; while (termEnum.Next() == true) { Term term = termEnum.Term; //System.out.println("Term: " + term); termDocs.Seek(term); while (termDocs.Next()) { int docId = termDocs.Doc; int freq = termDocs.Freq; //System.out.println("Doc Id: " + docId + " freq " + freq); ITermFreqVector vector = knownSearcher.reader_ForNUnit.GetTermFreqVector(docId, "field"); float tf = sim.Tf(freq); float idf = sim.Idf(knownSearcher.DocFreq(term), knownSearcher.MaxDoc); //float qNorm = sim.queryNorm() //This is fine since we don't have stop words float lNorm = sim.LengthNorm("field", vector.GetTerms().Length); //float coord = sim.coord() //System.out.println("TF: " + tf + " IDF: " + idf + " LenNorm: " + lNorm); Assert.IsTrue(vector != null); System.String[] vTerms = vector.GetTerms(); int[] freqs = vector.GetTermFrequencies(); for (int i = 0; i < vTerms.Length; i++) { if (term.Text.Equals(vTerms[i])) { Assert.IsTrue(freqs[i] == freq); } } } //System.out.println("--------"); } Query query = new TermQuery(new Term("field", "chocolate")); ScoreDoc[] hits = knownSearcher.Search(query, null, 1000).ScoreDocs; //doc 3 should be the first hit b/c it is the shortest match Assert.IsTrue(hits.Length == 3); float score = hits[0].Score; /*System.out.println("Hit 0: " + hits.id(0) + " Score: " + hits.score(0) + " String: " + hits.doc(0).toString()); System.out.println("Explain: " + knownSearcher.explain(query, hits.id(0))); System.out.println("Hit 1: " + hits.id(1) + " Score: " + hits.score(1) + " String: " + hits.doc(1).toString()); System.out.println("Explain: " + knownSearcher.explain(query, hits.id(1))); System.out.println("Hit 2: " + hits.id(2) + " Score: " + hits.score(2) + " String: " + hits.doc(2).toString()); System.out.println("Explain: " + knownSearcher.explain(query, hits.id(2)));*/ Assert.IsTrue(hits[0].Doc == 2); Assert.IsTrue(hits[1].Doc == 3); Assert.IsTrue(hits[2].Doc == 0); ITermFreqVector vector2 = knownSearcher.reader_ForNUnit.GetTermFreqVector(hits[1].Doc, "field"); Assert.IsTrue(vector2 != null); //System.out.println("Vector: " + vector); System.String[] terms = vector2.GetTerms(); int[] freqs2 = vector2.GetTermFrequencies(); Assert.IsTrue(terms != null && terms.Length == 10); for (int i = 0; i < terms.Length; i++) { System.String term = terms[i]; //System.out.println("Term: " + term); int freq = freqs2[i]; Assert.IsTrue(test4.IndexOf(term) != - 1); System.Int32 freqInt = -1; try { freqInt = (System.Int32) test4Map[term]; } catch (Exception) { Assert.IsTrue(false); } Assert.IsTrue(freqInt == freq); } SortedTermVectorMapper mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator()); knownSearcher.reader_ForNUnit.GetTermFreqVector(hits[1].Doc, mapper); var vectorEntrySet = mapper.TermVectorEntrySet; Assert.IsTrue(vectorEntrySet.Count == 10, "mapper.getTermVectorEntrySet() Size: " + vectorEntrySet.Count + " is not: " + 10); TermVectorEntry last = null; foreach(TermVectorEntry tve in vectorEntrySet) { if (tve != null && last != null) { Assert.IsTrue(last.Frequency >= tve.Frequency, "terms are not properly sorted"); System.Int32 expectedFreq = (System.Int32) test4Map[tve.Term]; //we expect double the expectedFreq, since there are two fields with the exact same text and we are collapsing all fields Assert.IsTrue(tve.Frequency == 2 * expectedFreq, "Frequency is not correct:"); } last = tve; } FieldSortedTermVectorMapper fieldMapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator()); knownSearcher.reader_ForNUnit.GetTermFreqVector(hits[1].Doc, fieldMapper); var map = fieldMapper.FieldToTerms; Assert.IsTrue(map.Count == 2, "map Size: " + map.Count + " is not: " + 2); vectorEntrySet = map["field"]; Assert.IsTrue(vectorEntrySet != null, "vectorEntrySet is null and it shouldn't be"); Assert.IsTrue(vectorEntrySet.Count == 10, "vectorEntrySet Size: " + vectorEntrySet.Count + " is not: " + 10); knownSearcher.Close(); } catch (System.IO.IOException e) { System.Console.Error.WriteLine(e.StackTrace); Assert.IsTrue(false); } }
public virtual void TestSnapshotDeletionPolicy_Renamed() { System.IO.FileInfo dir = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), INDEX_PATH)); try { // Sometimes past test leaves the dir _TestUtil.RmDir(dir); Directory fsDir = FSDirectory.Open(dir); RunTest(fsDir); fsDir.Close(); } finally { _TestUtil.RmDir(dir); } MockRAMDirectory dir2 = new MockRAMDirectory(); RunTest(dir2); dir2.Close(); }
public virtual void TestNPESpanQuery() { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(new System.Collections.Hashtable(0)), IndexWriter.MaxFieldLength.LIMITED); // Add documents AddDoc(writer, "1", "the big dogs went running to the market"); AddDoc(writer, "2", "the cat chased the mouse, then the cat ate the mouse quickly"); // Commit writer.Close(); // Get searcher IndexReader reader = IndexReader.Open(dir); IndexSearcher searcher = new IndexSearcher(reader); // Control (make sure docs indexed) Assert.AreEqual(2, HitCount(searcher, "the")); Assert.AreEqual(1, HitCount(searcher, "cat")); Assert.AreEqual(1, HitCount(searcher, "dogs")); Assert.AreEqual(0, HitCount(searcher, "rabbit")); // This throws exception (it shouldn't) Assert.AreEqual(1, searcher.Search(CreateSpan(0, true, new SpanQuery[]{CreateSpan(4, false, "chased", "cat"), CreateSpan("ate")}), 10).totalHits); reader.Close(); dir.Close(); }
public virtual void TestReuseAcrossWriters() { Directory dir = new MockRAMDirectory(); SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); IndexWriter writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp); // Force frequent commits writer.SetMaxBufferedDocs(2); Document doc = new Document(); doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for (int i = 0; i < 7; i++) writer.AddDocument(doc); IndexCommit cp = (IndexCommit) dp.Snapshot(); CopyFiles(dir, cp); writer.Close(); CopyFiles(dir, cp); writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp); CopyFiles(dir, cp); for (int i = 0; i < 7; i++) writer.AddDocument(doc); CopyFiles(dir, cp); writer.Close(); CopyFiles(dir, cp); dp.Release(); writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp); writer.Close(); try { CopyFiles(dir, cp); Assert.Fail("did not hit expected IOException"); } catch (System.IO.IOException ioe) { // expected } dir.Close(); }