Close() private method

private Close ( ) : void
return void
Esempio n. 1
0
 public override void  TearDown()
 {
     base.TearDown();
     readerA.Close();
     readerB.Close();
     readerX.Close();
 }
Esempio n. 2
0
        private static RAMDirectory MakeEmptyIndex(int numDeletedDocs)
        {
            RAMDirectory d = new RAMDirectory();
            IndexWriter  w = new IndexWriter(d, new WhitespaceAnalyzer(), true, MaxFieldLength.LIMITED, null);

            for (int i = 0; i < numDeletedDocs; i++)
            {
                w.AddDocument(new Document(), null);
            }
            w.Commit(null);
            w.DeleteDocuments(null, new MatchAllDocsQuery());
            w.Commit(null);

            if (0 < numDeletedDocs)
            {
                Assert.IsTrue(w.HasDeletions(null), "writer has no deletions");
            }

            Assert.AreEqual(numDeletedDocs, w.MaxDoc(), "writer is missing some deleted docs");
            Assert.AreEqual(0, w.NumDocs(null), "writer has non-deleted docs");
            w.Close();
            IndexReader r = IndexReader.Open((Directory)d, true, null);

            Assert.AreEqual(numDeletedDocs, r.NumDeletedDocs, "reader has wrong number of deleted docs");
            r.Close();
            return(d);
        }
        public virtual void  Test()
        {
            IndexReader reader = null;

            try
            {
                reader = IndexReader.Open((Directory)directory, true, null);
                for (int i = 1; i <= numThreads; i++)
                {
                    TestTermPositionVectors(reader, i);
                }
            }
            catch (System.IO.IOException ioe)
            {
                Assert.Fail(ioe.Message);
            }
            finally
            {
                if (reader != null)
                {
                    try
                    {
                        /* close the opened reader */
                        reader.Close();
                    }
                    catch (System.IO.IOException ioe)
                    {
                        System.Console.Error.WriteLine(ioe.StackTrace);
                    }
                }
            }
        }
Esempio n. 4
0
        public virtual void  TestRAMDirectory_Renamed()
        {
            Directory        dir    = FSDirectory.Open(indexDir);
            MockRAMDirectory ramDir = new MockRAMDirectory(dir);

            // close the underlaying directory
            dir.Close();

            // Check size
            Assert.AreEqual(ramDir.SizeInBytes(), ramDir.GetRecomputedSizeInBytes());

            // open reader to test document count
            IndexReader reader = IndexReader.Open((Directory)ramDir, true, null);

            Assert.AreEqual(docsToAdd, reader.NumDocs());

            // open search zo check if all doc's are there
            IndexSearcher searcher = new IndexSearcher(reader);

            // search for all documents
            for (int i = 0; i < docsToAdd; i++)
            {
                Document doc = searcher.Doc(i, null);
                Assert.IsTrue(doc.GetField("content") != null);
            }

            // cleanup
            reader.Close();
            searcher.Close();
        }
Esempio n. 5
0
        public virtual void  TestVerifyIndex()
        {
            IndexReader reader = IndexReader.Open(mDirectory, true, null);

            Assert.AreEqual(8, reader.NumDocs());
            reader.Close();
        }
Esempio n. 6
0
        public virtual void  TestCompressionTools()
        {
            IFieldable binaryFldCompressed = new Field("binaryCompressed", CompressionTools.Compress(System.Text.UTF8Encoding.UTF8.GetBytes(binaryValCompressed)), Field.Store.YES);
            IFieldable stringFldCompressed = new Field("stringCompressed", CompressionTools.CompressString(binaryValCompressed), Field.Store.YES);

            Document doc = new Document();

            doc.Add(binaryFldCompressed);
            doc.Add(stringFldCompressed);

            /* add the doc to a ram index */
            MockRAMDirectory dir    = new MockRAMDirectory();
            IndexWriter      writer = new IndexWriter(dir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED, null);

            writer.AddDocument(doc, null);
            writer.Close();

            /* open a reader and fetch the document */
            IndexReader reader        = IndexReader.Open((Directory)dir, false, null);
            Document    docFromReader = reader.Document(0, null);

            Assert.IsTrue(docFromReader != null);

            /* fetch the binary compressed field and compare it's content with the original one */
            System.String binaryFldCompressedTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(CompressionTools.Decompress(docFromReader.GetBinaryValue("binaryCompressed", null))));
            Assert.IsTrue(binaryFldCompressedTest.Equals(binaryValCompressed));
            Assert.IsTrue(CompressionTools.DecompressString(docFromReader.GetBinaryValue("stringCompressed", null)).Equals(binaryValCompressed));

            reader.Close();
            dir.Close();
        }
Esempio n. 7
0
        private static void doSpellCheckerIndexing(string LuceneIndexDir, string SpellCheckerIndexDir)
        {
            try
            {
                // http://lucene.apache.org/java/2_2_0/api/org/apache/lucene/search/spell/SpellChecker.html
                FSDirectory spellCheckerIndexDir = FSDirectory.GetDirectory(SpellCheckerIndexDir, false);
                FSDirectory indexDir             = FSDirectory.GetDirectory(LuceneIndexDir, false);

                SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(spellCheckerIndexDir);
                spellchecker.ClearIndex();
                // SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker (global::Lucene.Net.Store.Directory SpellChecker(spellIndexDirectory);

                IndexReader r = IndexReader.Open(indexDir);
                try
                {
                    // To index a field of a user index:
                    Dictionary dict = new SpellChecker.Net.Search.Spell.LuceneDictionary(r, "title");

                    spellchecker.IndexDictionary(dict);
                }
                finally
                {
                    r.Close();
                }
            }
            catch (Exception ex)
            {
                Console.Write("Could not create spell-checking index" + ex.Message);
            }
        }
Esempio n. 8
0
        public virtual void  TestNPESpanQuery()
        {
            Directory   dir    = new MockRAMDirectory();
            IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT, Support.Compatibility.SetFactory.CreateHashSet <string>()), IndexWriter.MaxFieldLength.LIMITED, null);

            // Add documents
            AddDoc(writer, "1", "the big dogs went running to the market");
            AddDoc(writer, "2", "the cat chased the mouse, then the cat ate the mouse quickly");

            // Commit
            writer.Close();

            // Get searcher
            IndexReader   reader   = IndexReader.Open(dir, true, null);
            IndexSearcher searcher = new IndexSearcher(reader);

            // Control (make sure docs indexed)
            Assert.AreEqual(2, HitCount(searcher, "the"));
            Assert.AreEqual(1, HitCount(searcher, "cat"));
            Assert.AreEqual(1, HitCount(searcher, "dogs"));
            Assert.AreEqual(0, HitCount(searcher, "rabbit"));

            // This throws exception (it shouldn't)
            Assert.AreEqual(1, searcher.Search(CreateSpan(0, true, new SpanQuery[] { CreateSpan(4, false, "chased", "cat"), CreateSpan("ate") }), 10, null).TotalHits);
            reader.Close();
            dir.Close();
        }
 public void CloseReader(IndexReader reader)
 {
     try
     {
         reader.Close();
     }
     catch (System.IO.IOException ex)
     {
         //TODO: extract subReaders and close each one individually
         ReaderProviderHelper.Clean(reader);
         new SearchException("Unable to close multiReader", ex);
     }
 }
Esempio n. 10
0
        public virtual void  TestSetBufferSize()
        {
            System.IO.DirectoryInfo indexDir = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testSetBufferSize"));
            MockFSDirectory         dir      = new MockFSDirectory(indexDir, NewRandom());

            try
            {
                IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);
                writer.UseCompoundFile = false;
                for (int i = 0; i < 37; i++)
                {
                    Document doc = new Document();
                    doc.Add(new Field("content", "aaa bbb ccc ddd" + i, Field.Store.YES, Field.Index.ANALYZED));
                    doc.Add(new Field("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
                    writer.AddDocument(doc, null);
                }
                writer.Close();

                dir.allIndexInputs.Clear();

                IndexReader reader = IndexReader.Open((Directory)dir, false, null);
                Term        aaa    = new Term("content", "aaa");
                Term        bbb    = new Term("content", "bbb");
                Term        ccc    = new Term("content", "ccc");
                Assert.AreEqual(37, reader.DocFreq(ccc, null));
                reader.DeleteDocument(0, null);
                Assert.AreEqual(37, reader.DocFreq(aaa, null));
                dir.tweakBufferSizes();
                reader.DeleteDocument(4, null);
                Assert.AreEqual(reader.DocFreq(bbb, null), 37);
                dir.tweakBufferSizes();

                IndexSearcher searcher = new IndexSearcher(reader);
                ScoreDoc[]    hits     = searcher.Search(new TermQuery(bbb), null, 1000, null).ScoreDocs;
                dir.tweakBufferSizes();
                Assert.AreEqual(35, hits.Length);
                dir.tweakBufferSizes();
                hits = searcher.Search(new TermQuery(new Term("id", "33")), null, 1000, null).ScoreDocs;
                dir.tweakBufferSizes();
                Assert.AreEqual(1, hits.Length);
                hits = searcher.Search(new TermQuery(aaa), null, 1000, null).ScoreDocs;
                dir.tweakBufferSizes();
                Assert.AreEqual(35, hits.Length);
                searcher.Close();
                reader.Close();
            }
            finally
            {
                _TestUtil.RmDir(indexDir);
            }
        }
Esempio n. 11
0
        /***
         * Understands the lucene query syntax
         */
        public List <Utilities.Language.TextIndexing.IndexResult> GetDocumentsWithQuery(string query)
        {
            List <Utilities.Language.TextIndexing.IndexResult> fingerprints = new List <Utilities.Language.TextIndexing.IndexResult>();
            HashSet <string> fingerprints_already_seen = new HashSet <string>();

            try
            {
                using (Lucene.Net.Index.IndexReader index_reader = Lucene.Net.Index.IndexReader.Open(LIBRARY_INDEX_BASE_PATH, true))
                {
                    using (Lucene.Net.Search.IndexSearcher index_searcher = new Lucene.Net.Search.IndexSearcher(index_reader))
                    {
                        Lucene.Net.QueryParsers.QueryParser query_parser = new Lucene.Net.QueryParsers.QueryParser(Version.LUCENE_29, "content", analyzer);

                        Lucene.Net.Search.Query query_object = query_parser.Parse(query);
                        Lucene.Net.Search.Hits  hits         = index_searcher.Search(query_object);

                        var i = hits.Iterator();
                        while (i.MoveNext())
                        {
                            Lucene.Net.Search.Hit hit = (Lucene.Net.Search.Hit)i.Current;
                            string fingerprint        = hit.Get("fingerprint");
                            string page = hit.Get("page");

                            if (!fingerprints_already_seen.Contains(fingerprint))
                            {
                                fingerprints_already_seen.Add(fingerprint);

                                IndexResult index_result = new IndexResult {
                                    fingerprint = fingerprint, score = hit.GetScore()
                                };
                                fingerprints.Add(index_result);
                            }
                        }

                        // Close the index
                        index_searcher.Close();
                    }
                    index_reader.Close();
                }
            }
            catch (Exception ex)
            {
                Logging.Warn(ex, "GetDocumentsWithQuery: There was a problem opening the index file for searching.");
            }

            return(fingerprints);
        }
Esempio n. 12
0
        } // doIndex

        private static void removeAllDuplicateAndDeletedFiles(IndexableFileInfo[] fileInfos, string LuceneIndexDir, IndexCreationMode indexCreationMode)
        {
            if (indexCreationMode != IndexCreationMode.AppendToExistingIndex)
            {
                return;
            }

            IndexReader reader = IndexReader.Open(LuceneIndexDir);

            try
            {
                int numDocs = reader.NumDocs();
                for (int i = 0; i < numDocs; i++)
                {
                    Document docToCheck         = reader.Document(i);
                    bool     removeDocFromIndex = true;
                    string   filenameField      = docToCheck.GetField("filename").StringValue();
                    string   lastModified       = (docToCheck.GetField("LastModified").StringValue());

                    foreach (IndexableFileInfo fi in fileInfos)
                    {
                        if (String.Compare(fi.Filename, filenameField, true) == 0 && DateTools.DateToString(fi.LastModified, DateTools.Resolution.SECOND) == lastModified)
                        {
                            removeDocFromIndex = false;
                            break;
                        }
                    } // foreach

                    if (removeDocFromIndex)
                    {
                        reader.DeleteDocument(i);
                        if (!reader.HasDeletions())
                        {
                            throw new Exception("error: deletion failed!!");
                        }
                    }
                } // for each lucene doc
            }
            finally
            {
                reader.Close();
            }
            LuceneIndexer indexer = new LuceneIndexer(LuceneIndexDir, indexCreationMode); // open up the index again

            indexer.CloseIndexWriter(OptimizeMode.DoOptimization);                        // just to optimize the index (which removes deleted items).
        }
Esempio n. 13
0
        public virtual void  TestBinaryFieldInIndex()
        {
            IFieldable binaryFldStored = new Field("binaryStored", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValStored), Field.Store.YES);
            IFieldable stringFldStored = new Field("stringStored", binaryValStored, Field.Store.YES, Field.Index.NO, Field.TermVector.NO);

            // binary fields with store off are not allowed
            Assert.Throws <ArgumentException>(
                () => new Field("fail", System.Text.Encoding.UTF8.GetBytes(binaryValStored), Field.Store.NO));

            Document doc = new Document();

            doc.Add(binaryFldStored);

            doc.Add(stringFldStored);

            /* test for field count */
            Assert.AreEqual(2, doc.fields_ForNUnit.Count);

            /* add the doc to a ram index */
            MockRAMDirectory dir    = new MockRAMDirectory();
            IndexWriter      writer = new IndexWriter(dir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED, null);

            writer.AddDocument(doc, null);
            writer.Close();

            /* open a reader and fetch the document */
            IndexReader reader        = IndexReader.Open((Directory)dir, false, null);
            Document    docFromReader = reader.Document(0, null);

            Assert.IsTrue(docFromReader != null);

            /* fetch the binary stored field and compare it's content with the original one */
            System.String binaryFldStoredTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(docFromReader.GetBinaryValue("binaryStored", null)));
            Assert.IsTrue(binaryFldStoredTest.Equals(binaryValStored));

            /* fetch the string field and compare it's content with the original one */
            System.String stringFldStoredTest = docFromReader.Get("stringStored", null);
            Assert.IsTrue(stringFldStoredTest.Equals(binaryValStored));

            /* delete the document from index */
            reader.DeleteDocument(0, null);
            Assert.AreEqual(0, reader.NumDocs());

            reader.Close();
            dir.Close();
        }
        public void TestFieldNonExistent()
        {
            try
            {
                indexReader = IndexReader.Open(store, true);

                ld = new LuceneDictionary(indexReader, "nonexistent_field");
                it = ld.GetWordsIterator();

                AssertFalse("More elements than expected", it.HasNext());
                AssertTrue("Nonexistent element is really null", it.Next() == null);
            }
            finally
            {
                if (indexReader != null) { indexReader.Close(); }
            }
        }
Esempio n. 15
0
        protected override void Dispose(bool disposing)
        {
            if (isDisposed)
            {
                return;
            }

            if (disposing)
            {
                if (closeReader)
                {
                    reader.Close();
                }
            }

            isDisposed = true;
        }
Esempio n. 16
0
        public virtual void  TestCaching()
        {
            Directory   dir    = new RAMDirectory();
            IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), IndexWriter.MaxFieldLength.LIMITED, null);
            Document    doc    = new Document();
            TokenStream stream = new AnonymousClassTokenStream(this);

            stream = new CachingTokenFilter(stream);

            doc.Add(new Field("preanalyzed", stream, TermVector.NO));

            // 1) we consume all tokens twice before we add the doc to the index
            checkTokens(stream);
            stream.Reset();
            checkTokens(stream);

            // 2) now add the document to the index and verify if all tokens are indexed
            //    don't reset the stream here, the DocumentWriter should do that implicitly
            writer.AddDocument(doc, null);
            writer.Close();

            IndexReader   reader        = IndexReader.Open(dir, true, null);
            TermPositions termPositions = reader.TermPositions(new Term("preanalyzed", "term1"), null);

            Assert.IsTrue(termPositions.Next(null));
            Assert.AreEqual(1, termPositions.Freq);
            Assert.AreEqual(0, termPositions.NextPosition(null));

            termPositions.Seek(new Term("preanalyzed", "term2"), null);
            Assert.IsTrue(termPositions.Next(null));
            Assert.AreEqual(2, termPositions.Freq);
            Assert.AreEqual(1, termPositions.NextPosition(null));
            Assert.AreEqual(3, termPositions.NextPosition(null));

            termPositions.Seek(new Term("preanalyzed", "term3"), null);
            Assert.IsTrue(termPositions.Next(null));
            Assert.AreEqual(1, termPositions.Freq);
            Assert.AreEqual(2, termPositions.NextPosition(null));
            reader.Close();

            // 3) reset stream and consume tokens again
            stream.Reset();
            checkTokens(stream);
        }
        public virtual void  TestMissingTerms()
        {
            System.String    fieldName = "field1";
            MockRAMDirectory rd        = new MockRAMDirectory();
            IndexWriter      w         = new IndexWriter(rd, new KeywordAnalyzer(), MaxFieldLength.UNLIMITED, null);

            for (int i = 0; i < 100; i++)
            {
                Document doc  = new Document();
                int      term = i * 10;            //terms are units of 10;
                doc.Add(new Field(fieldName, "" + term, Field.Store.YES, Field.Index.NOT_ANALYZED));
                w.AddDocument(doc, null);
            }
            w.Close();

            IndexReader   reader   = IndexReader.Open((Directory)rd, true, null);
            IndexSearcher searcher = new IndexSearcher(reader);
            int           numDocs  = reader.NumDocs();

            ScoreDoc[]        results;
            MatchAllDocsQuery q = new MatchAllDocsQuery();

            System.Collections.ArrayList terms = new System.Collections.ArrayList();
            terms.Add("5");
            results = searcher.Search(q, new FieldCacheTermsFilter(fieldName, (System.String[])terms.ToArray(typeof(System.String))), numDocs, null).ScoreDocs;
            Assert.AreEqual(0, results.Length, "Must match nothing");

            terms = new System.Collections.ArrayList();
            terms.Add("10");
            results = searcher.Search(q, new FieldCacheTermsFilter(fieldName, (System.String[])terms.ToArray(typeof(System.String))), numDocs, null).ScoreDocs;
            Assert.AreEqual(1, results.Length, "Must match 1");

            terms = new System.Collections.ArrayList();
            terms.Add("10");
            terms.Add("20");
            results = searcher.Search(q, new FieldCacheTermsFilter(fieldName, (System.String[])terms.ToArray(typeof(System.String))), numDocs, null).ScoreDocs;
            Assert.AreEqual(2, results.Length, "Must match 2");

            reader.Close();
            rd.Close();
        }
Esempio n. 18
0
        public virtual void  TestFilterWorks()
        {
            Directory   dir    = new RAMDirectory();
            IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);

            for (int i = 0; i < 500; i++)
            {
                Document document = new Document();
                document.Add(new Field("field", English.IntToEnglish(i) + " equals " + English.IntToEnglish(i), Field.Store.NO, Field.Index.ANALYZED));
                writer.AddDocument(document, null);
            }
            writer.Close();

            IndexReader reader = IndexReader.Open(dir, true, null);

            SpanTermQuery    query    = new SpanTermQuery(new Term("field", English.IntToEnglish(10).Trim()));
            SpanQueryFilter  filter   = new SpanQueryFilter(query);
            SpanFilterResult result   = filter.BitSpans(reader, null);
            DocIdSet         docIdSet = result.DocIdSet;

            Assert.IsTrue(docIdSet != null, "docIdSet is null and it shouldn't be");
            AssertContainsDocId("docIdSet doesn't contain docId 10", docIdSet, 10);
            var spans = result.Positions;

            Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
            int size = GetDocIdSetSize(docIdSet);

            Assert.IsTrue(spans.Count == size, "spans Size: " + spans.Count + " is not: " + size);
            for (System.Collections.IEnumerator iterator = spans.GetEnumerator(); iterator.MoveNext();)
            {
                SpanFilterResult.PositionInfo info = (SpanFilterResult.PositionInfo)iterator.Current;
                Assert.IsTrue(info != null, "info is null and it shouldn't be");
                //The doc should indicate the bit is on
                AssertContainsDocId("docIdSet doesn't contain docId " + info.Doc, docIdSet, info.Doc);
                //There should be two positions in each
                Assert.IsTrue(info.Positions.Count == 2, "info.getPositions() Size: " + info.Positions.Count + " is not: " + 2);
            }
            reader.Close();
        }
Esempio n. 19
0
        public virtual void  TestSetNorm_Renamed()
        {
            RAMDirectory store  = new RAMDirectory();
            IndexWriter  writer = new IndexWriter(store, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);

            // add the same document four times
            IFieldable f1 = new Field("field", "word", Field.Store.YES, Field.Index.ANALYZED);
            Document   d1 = new Document();

            d1.Add(f1);
            writer.AddDocument(d1, null);
            writer.AddDocument(d1, null);
            writer.AddDocument(d1, null);
            writer.AddDocument(d1, null);
            writer.Close();

            // reset the boost of each instance of this document
            IndexReader reader = IndexReader.Open((Directory)store, false, null);

            reader.SetNorm(0, "field", 1.0f, null);
            reader.SetNorm(1, "field", 2.0f, null);
            reader.SetNorm(2, "field", 4.0f, null);
            reader.SetNorm(3, "field", 16.0f, null);
            reader.Close();

            // check that searches are ordered by this boost
            float[] scores = new float[4];

            new IndexSearcher(store, true, null).Search(new TermQuery(new Term("field", "word")), new AnonymousClassCollector(scores, this), null);

            float lastScore = 0.0f;

            for (int i = 0; i < 4; i++)
            {
                Assert.IsTrue(scores[i] > lastScore);
                lastScore = scores[i];
            }
        }
Esempio n. 20
0
        public virtual void  TestKeepLastNDeletionPolicyWithCreates()
        {
            int N = 10;

            for (int pass = 0; pass < 4; pass++)
            {
                bool autoCommit      = pass < 2;
                bool useCompoundFile = (pass % 2) > 0;

                KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);

                Directory   dir    = new RAMDirectory();
                IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
                writer.SetMaxBufferedDocs(10);
                writer.SetUseCompoundFile(useCompoundFile);
                writer.Close();
                Term  searchTerm = new Term("content", "aaa");
                Query query      = new TermQuery(searchTerm);

                for (int i = 0; i < N + 1; i++)
                {
                    writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
                    writer.SetMaxBufferedDocs(10);
                    writer.SetUseCompoundFile(useCompoundFile);
                    for (int j = 0; j < 17; j++)
                    {
                        AddDoc(writer);
                    }
                    // this is a commit when autoCommit=false:
                    writer.Close();
                    IndexReader reader = IndexReader.Open(dir, policy);
                    reader.DeleteDocument(3);
                    reader.SetNorm(5, "content", 2.0F);
                    IndexSearcher searcher = new IndexSearcher(reader);
                    ScoreDoc[]    hits     = searcher.Search(query, null, 1000).scoreDocs;
                    Assert.AreEqual(16, hits.Length);
                    // this is a commit when autoCommit=false:
                    reader.Close();
                    searcher.Close();

                    writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
                    // This will not commit: there are no changes
                    // pending because we opened for "create":
                    writer.Close();
                }

                Assert.AreEqual(1 + 3 * (N + 1), policy.numOnInit);
                if (!autoCommit)
                {
                    Assert.AreEqual(3 * (N + 1), policy.numOnCommit);
                }

                IndexSearcher searcher2 = new IndexSearcher(dir);
                ScoreDoc[]    hits2     = searcher2.Search(query, null, 1000).scoreDocs;
                Assert.AreEqual(0, hits2.Length);

                // Simplistic check: just verify only the past N segments_N's still
                // exist, and, I can open a reader on each:
                long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);

                dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
                int expectedCount = 0;

                for (int i = 0; i < N + 1; i++)
                {
                    try
                    {
                        IndexReader reader = IndexReader.Open(dir);

                        // Work backwards in commits on what the expected
                        // count should be.  Only check this in the
                        // autoCommit false case:
                        if (!autoCommit)
                        {
                            searcher2 = new IndexSearcher(reader);
                            hits2     = searcher2.Search(query, null, 1000).scoreDocs;
                            Assert.AreEqual(expectedCount, hits2.Length);
                            searcher2.Close();
                            if (expectedCount == 0)
                            {
                                expectedCount = 16;
                            }
                            else if (expectedCount == 16)
                            {
                                expectedCount = 17;
                            }
                            else if (expectedCount == 17)
                            {
                                expectedCount = 0;
                            }
                        }
                        reader.Close();
                        if (i == N)
                        {
                            Assert.Fail("should have failed on commits before last " + N);
                        }
                    }
                    catch (System.IO.IOException e)
                    {
                        if (i != N)
                        {
                            throw e;
                        }
                    }
                    if (i < N)
                    {
                        dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
                    }
                    gen--;
                }

                dir.Close();
            }
        }
Esempio n. 21
0
        public virtual void  TestKeepLastNDeletionPolicy()
        {
            int N = 5;

            for (int pass = 0; pass < 4; pass++)
            {
                bool autoCommit      = pass < 2;
                bool useCompoundFile = (pass % 2) > 0;

                Directory dir = new RAMDirectory();

                KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);

                for (int j = 0; j < N + 1; j++)
                {
                    IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
                    writer.SetMaxBufferedDocs(10);
                    writer.SetUseCompoundFile(useCompoundFile);
                    for (int i = 0; i < 17; i++)
                    {
                        AddDoc(writer);
                    }
                    writer.Optimize();
                    writer.Close();
                }

                Assert.IsTrue(policy.numDelete > 0);
                Assert.AreEqual(N + 1, policy.numOnInit);
                if (autoCommit)
                {
                    Assert.IsTrue(policy.numOnCommit > 1);
                }
                else
                {
                    Assert.AreEqual(N + 1, policy.numOnCommit);
                }

                // Simplistic check: just verify only the past N segments_N's still
                // exist, and, I can open a reader on each:
                dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
                long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
                for (int i = 0; i < N + 1; i++)
                {
                    try
                    {
                        IndexReader reader = IndexReader.Open(dir);
                        reader.Close();
                        if (i == N)
                        {
                            Assert.Fail("should have failed on commits prior to last " + N);
                        }
                    }
                    catch (System.IO.IOException e)
                    {
                        if (i != N)
                        {
                            throw e;
                        }
                    }
                    if (i < N)
                    {
                        dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
                    }
                    gen--;
                }

                dir.Close();
            }
        }
Esempio n. 22
0
        public virtual void  TestOpenPriorSnapshot()
        {
            // Never deletes a commit
            KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);

            Directory dir = new MockRAMDirectory();

            policy.dir = dir;

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);

            writer.SetMaxBufferedDocs(2);
            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer);
                if ((1 + i) % 2 == 0)
                {
                    writer.Commit();
                }
            }
            writer.Close();

            System.Collections.ICollection commits = IndexReader.ListCommits(dir);
            Assert.AreEqual(6, commits.Count);
            IndexCommit lastCommit = null;

            System.Collections.IEnumerator it = commits.GetEnumerator();
            while (it.MoveNext())
            {
                IndexCommit commit = (IndexCommit)it.Current;
                if (lastCommit == null || commit.GetGeneration() > lastCommit.GetGeneration())
                {
                    lastCommit = commit;
                }
            }
            Assert.IsTrue(lastCommit != null);

            // Now add 1 doc and optimize
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
            AddDoc(writer);
            Assert.AreEqual(11, writer.NumDocs());
            writer.Optimize();
            writer.Close();

            Assert.AreEqual(7, IndexReader.ListCommits(dir).Count);

            // Now open writer on the commit just before optimize:
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED, lastCommit);
            Assert.AreEqual(10, writer.NumDocs());

            // Should undo our rollback:
            writer.Rollback();

            IndexReader r = IndexReader.Open(dir);

            // Still optimized, still 11 docs
            Assert.IsTrue(r.IsOptimized());
            Assert.AreEqual(11, r.NumDocs());
            r.Close();

            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED, lastCommit);
            Assert.AreEqual(10, writer.NumDocs());
            // Commits the rollback:
            writer.Close();

            // Now 8 because we made another commit
            Assert.AreEqual(8, IndexReader.ListCommits(dir).Count);

            r = IndexReader.Open(dir);
            // Not optimized because we rolled it back, and now only
            // 10 docs
            Assert.IsTrue(!r.IsOptimized());
            Assert.AreEqual(10, r.NumDocs());
            r.Close();

            // Reoptimize
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
            writer.Optimize();
            writer.Close();

            r = IndexReader.Open(dir);
            Assert.IsTrue(r.IsOptimized());
            Assert.AreEqual(10, r.NumDocs());
            r.Close();

            // Now open writer on the commit just before optimize,
            // but this time keeping only the last commit:
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), new KeepOnlyLastCommitDeletionPolicy(), IndexWriter.MaxFieldLength.LIMITED, lastCommit);
            Assert.AreEqual(10, writer.NumDocs());

            // Reader still sees optimized index, because writer
            // opened on the prior commit has not yet committed:
            r = IndexReader.Open(dir);
            Assert.IsTrue(r.IsOptimized());
            Assert.AreEqual(10, r.NumDocs());
            r.Close();

            writer.Close();

            // Now reader sees unoptimized index:
            r = IndexReader.Open(dir);
            Assert.IsTrue(!r.IsOptimized());
            Assert.AreEqual(10, r.NumDocs());
            r.Close();

            dir.Close();
        }
Esempio n. 23
0
        public virtual void  TestDeletedDocs()
        {
            MockRAMDirectory dir    = new MockRAMDirectory();
            IndexWriter      writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);

            writer.SetMaxBufferedDocs(2);
            Document doc = new Document();

            doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
            for (int i = 0; i < 19; i++)
            {
                writer.AddDocument(doc, null);
            }
            writer.Close();
            IndexReader reader = IndexReader.Open((Directory)dir, false, null);

            reader.DeleteDocument(5, null);
            reader.Close();

            System.IO.MemoryStream bos     = new System.IO.MemoryStream(1024);
            CheckIndex             checker = new CheckIndex(dir);

            checker.SetInfoStream(new System.IO.StreamWriter(bos));
            //checker.setInfoStream(System.out);
            CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method(null);
            if (indexStatus.clean == false)
            {
                System.Console.Out.WriteLine("CheckIndex failed");

                using (var sr = new StreamReader(bos))
                {
                    System.Console.Out.WriteLine(sr.ReadToEnd());
                    Assert.Fail();
                }
            }

            CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus)indexStatus.segmentInfos[0];
            Assert.IsTrue(seg.openReaderPassed);

            Assert.IsNotNull(seg.diagnostics);

            Assert.IsNotNull(seg.fieldNormStatus);
            Assert.IsNull(seg.fieldNormStatus.error);
            Assert.AreEqual(1, seg.fieldNormStatus.totFields);

            Assert.IsNotNull(seg.termIndexStatus);
            Assert.IsNull(seg.termIndexStatus.error);
            Assert.AreEqual(1, seg.termIndexStatus.termCount);
            Assert.AreEqual(19, seg.termIndexStatus.totFreq);
            Assert.AreEqual(18, seg.termIndexStatus.totPos);

            Assert.IsNotNull(seg.storedFieldStatus);
            Assert.IsNull(seg.storedFieldStatus.error);
            Assert.AreEqual(18, seg.storedFieldStatus.docCount);
            Assert.AreEqual(18, seg.storedFieldStatus.totFields);

            Assert.IsNotNull(seg.termVectorStatus);
            Assert.IsNull(seg.termVectorStatus.error);
            Assert.AreEqual(18, seg.termVectorStatus.docCount);
            Assert.AreEqual(18, seg.termVectorStatus.totVectors);

            Assert.IsTrue(seg.diagnostics.Count > 0);
            List <string> onlySegments = new List <string>();

            onlySegments.Add("_0");

            Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments, null).clean == true);
        }
 public void TestSpellchecker()
 {
     SpellChecker.Net.Search.Spell.SpellChecker sc = new SpellChecker.Net.Search.Spell.SpellChecker(new RAMDirectory());
     indexReader = IndexReader.Open(store);
     sc.IndexDictionary(new LuceneDictionary(indexReader, "contents"));
     String[] suggestions = sc.SuggestSimilar("Tam", 1);
     AssertEquals(1, suggestions.Length);
     AssertEquals("Tom", suggestions[0]);
     suggestions = sc.SuggestSimilar("Jarry", 1);
     AssertEquals(1, suggestions.Length);
     AssertEquals("Jerry", suggestions[0]);
     indexReader.Close();
 }
        public void TestFieldContents_1()
        {
            try
            {
                indexReader = IndexReader.Open(store);

                ld = new LuceneDictionary(indexReader, "contents");
                it = ld.GetWordsIterator();

                AssertTrue("First element doesn't exist.", it.HasNext());
                AssertTrue("First element isn't correct", it.Next().Equals("Jerry"));
                AssertTrue("Second element doesn't exist.", it.HasNext());
                AssertTrue("Second element isn't correct", it.Next().Equals("Tom"));
                AssertFalse("More elements than expected", it.HasNext());
                AssertTrue("Nonexistent element is really null", it.Next() == null);

                ld = new LuceneDictionary(indexReader, "contents");
                it = ld.GetWordsIterator();

                int counter = 2;
                while (it.HasNext())
                {
                    it.Next();
                    counter--;
                }

                AssertTrue("Number of words incorrect", counter == 0);
            }
            finally
            {
                if (indexReader != null) { indexReader.Close(); }
            }
        }
Esempio n. 26
0
        /* Open pre-lockless index, add docs, do a delete &
         * setNorm, and search */
        public virtual void  ChangeIndexWithAdds(System.String dirName)
        {
            System.String origDirName = dirName;
            dirName = FullDir(dirName);

            Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo(dirName));

            // open writer
            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);

            // add 10 docs
            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer, 35 + i);
            }

            // make sure writer sees right total -- writer seems not to know about deletes in .del?
            int expected;

            if (Compare(origDirName, "24") < 0)
            {
                expected = 45;
            }
            else
            {
                expected = 46;
            }
            Assert.AreEqual(expected, writer.MaxDoc(), "wrong doc count");
            writer.Close();

            // make sure searching sees right # hits
            IndexSearcher searcher = new IndexSearcher(dir, true);

            ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
            Document   d    = searcher.Doc(hits[0].Doc);

            Assert.AreEqual("21", d.Get("id"), "wrong first document");
            TestHits(hits, 44, searcher.IndexReader);
            searcher.Close();

            // make sure we can do delete & setNorm against this
            // pre-lockless segment:
            IndexReader reader     = IndexReader.Open(dir, false);
            Term        searchTerm = new Term("id", "6");
            int         delCount   = reader.DeleteDocuments(searchTerm);

            Assert.AreEqual(1, delCount, "wrong delete count");
            reader.SetNorm(22, "content", (float)2.0);
            reader.Close();

            // make sure they "took":
            searcher = new IndexSearcher(dir, true);
            hits     = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
            Assert.AreEqual(43, hits.Length, "wrong number of hits");
            d = searcher.Doc(hits[0].Doc);
            Assert.AreEqual("22", d.Get("id"), "wrong first document");
            TestHits(hits, 43, searcher.IndexReader);
            searcher.Close();

            // optimize
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
            writer.Optimize();
            writer.Close();

            searcher = new IndexSearcher(dir, true);
            hits     = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
            Assert.AreEqual(43, hits.Length, "wrong number of hits");
            d = searcher.Doc(hits[0].Doc);
            TestHits(hits, 43, searcher.IndexReader);
            Assert.AreEqual("22", d.Get("id"), "wrong first document");
            searcher.Close();

            dir.Close();
        }
Esempio n. 27
0
 protected internal override void  DoClose()
 {
     in_Renamed.Close();
 }
Esempio n. 28
0
 private void CloseIndexReader(IndexReader myReader)
 {
     myReader.Close();
     myReader.Dispose();
 }
        private void CloseInternalReader(bool trace, IndexReader subReader, bool finalClose)
        {
            ReaderData readerData;
            // TODO: can we avoid the lock?
            lock (semaphoreIndexReaderLock)
            {
                readerData = searchIndexReaderSemaphores[subReader];
            }

            if (readerData == null)
            {
                log.Error("Trying to close a Lucene IndexReader not present: " + subReader.Directory());
                // TODO: Should we try to close?
                return;
            }

            // Acquire the locks in the same order as everywhere else
            object directoryProviderLock = perDirectoryProviderManipulationLocks[readerData.Provider];
            bool closeReader = false;
            lock (directoryProviderLock)
            {
                IndexReader reader;
                bool isActive = activeSearchIndexReaders.TryGetValue(readerData.Provider, out reader)
                    && reader == subReader;
                if (trace) log.Info("IndexReader not active: " + subReader);
                lock (semaphoreIndexReaderLock)
                {
                    readerData = searchIndexReaderSemaphores[subReader];
                    if (readerData == null)
                    {
                        log.Error("Trying to close a Lucene IndexReader not present: " + subReader.Directory());
                        // TODO: Should we try to close?
                        return;
                    }

                    //final close, the semaphore should be at 0 already
                    if (!finalClose)
                    {
                        readerData.Semaphore--;
                        if (trace)
                            log.Info("Semaphore decreased to: " + readerData.Semaphore + " for " + subReader);
                    }

                    if (readerData.Semaphore < 0)
                        log.Error("Semaphore negative: " + subReader.Directory());

                    if (!isActive && readerData.Semaphore == 0)
                    {
                        searchIndexReaderSemaphores.Remove(subReader);
                        closeReader = true;
                    }
                    else
                        closeReader = false;
                }
            }

            if (closeReader)
            {
                if (trace) log.Info("Closing IndexReader: " + subReader);
                try
                {
                    subReader.Close();
                }
                catch (IOException e)
                {
                    log.Warn("Unable to close Lucene IndexReader", e);
                }
            }
        }
Esempio n. 30
0
        public virtual void  TestDeleteLeftoverFiles()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            writer.SetMaxBufferedDocs(10);
            int i;

            for (i = 0; i < 35; i++)
            {
                AddDoc(writer, i);
            }
            writer.SetUseCompoundFile(false);
            for (; i < 45; i++)
            {
                AddDoc(writer, i);
            }
            writer.Close();

            // Delete one doc so we get a .del file:
            IndexReader reader     = IndexReader.Open(dir);
            Term        searchTerm = new Term("id", "7");
            int         delCount   = reader.DeleteDocuments(searchTerm);

            Assert.AreEqual(1, delCount, "didn't delete the right number of documents");

            // Set one norm so we get a .s0 file:
            reader.SetNorm(21, "content", (float)1.5);
            reader.Close();

            // Now, artificially create an extra .del file & extra
            // .s0 file:
            System.String[] files = dir.ListAll();

            /*
             * for(int j=0;j<files.length;j++) {
             * System.out.println(j + ": " + files[j]);
             * }
             */

            // The numbering of fields can vary depending on which
            // JRE is in use.  On some JREs we see content bound to
            // field 0; on others, field 1.  So, here we have to
            // figure out which field number corresponds to
            // "content", and then set our expected file names below
            // accordingly:
            CompoundFileReader cfsReader  = new CompoundFileReader(dir, "_2.cfs");
            FieldInfos         fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
            int contentFieldIndex         = -1;

            for (i = 0; i < fieldInfos.Size(); i++)
            {
                FieldInfo fi = fieldInfos.FieldInfo(i);
                if (fi.name_ForNUnit.Equals("content"))
                {
                    contentFieldIndex = i;
                    break;
                }
            }
            cfsReader.Close();
            Assert.IsTrue(contentFieldIndex != -1, "could not locate the 'content' field number in the _2.cfs segment");

            System.String normSuffix = "s" + contentFieldIndex;

            // Create a bogus separate norms file for a
            // segment/field that actually has a separate norms file
            // already:
            CopyFile(dir, "_2_1." + normSuffix, "_2_2." + normSuffix);

            // Create a bogus separate norms file for a
            // segment/field that actually has a separate norms file
            // already, using the "not compound file" extension:
            CopyFile(dir, "_2_1." + normSuffix, "_2_2.f" + contentFieldIndex);

            // Create a bogus separate norms file for a
            // segment/field that does not have a separate norms
            // file already:
            CopyFile(dir, "_2_1." + normSuffix, "_1_1." + normSuffix);

            // Create a bogus separate norms file for a
            // segment/field that does not have a separate norms
            // file already using the "not compound file" extension:
            CopyFile(dir, "_2_1." + normSuffix, "_1_1.f" + contentFieldIndex);

            // Create a bogus separate del file for a
            // segment that already has a separate del file:
            CopyFile(dir, "_0_1.del", "_0_2.del");

            // Create a bogus separate del file for a
            // segment that does not yet have a separate del file:
            CopyFile(dir, "_0_1.del", "_1_1.del");

            // Create a bogus separate del file for a
            // non-existent segment:
            CopyFile(dir, "_0_1.del", "_188_1.del");

            // Create a bogus segment file:
            CopyFile(dir, "_0.cfs", "_188.cfs");

            // Create a bogus fnm file when the CFS already exists:
            CopyFile(dir, "_0.cfs", "_0.fnm");

            // Create a deletable file:
            CopyFile(dir, "_0.cfs", "deletable");

            // Create some old segments file:
            CopyFile(dir, "segments_3", "segments");
            CopyFile(dir, "segments_3", "segments_2");

            // Create a bogus cfs file shadowing a non-cfs segment:
            CopyFile(dir, "_2.cfs", "_3.cfs");

            System.String[] filesPre = dir.ListAll();

            // Open & close a writer: it should delete the above 4
            // files and nothing more:
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
            writer.Close();

            System.String[] files2 = dir.ListAll();
            dir.Close();

            System.Array.Sort(files);
            System.Array.Sort(files2);

            System.Collections.Hashtable dif = DifFiles(files, files2);

            if (!SupportClass.CollectionsHelper.Equals(files, files2))
            {
                Assert.Fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.Length - files.Length) + " files but only deleted " + (filesPre.Length - files2.Length) + "; expected files:\n    " + AsString(files) + "\n  actual files:\n    " + AsString(files2) + "\ndif: " + SupportClass.CollectionsHelper.CollectionToString(dif));
            }
        }
Esempio n. 31
0
                static public void ReleaseReader (IndexReader reader)
                {
                        lock (reader_rav_map) {
                                ReaderAndVersion rav = (ReaderAndVersion) reader_rav_map [reader];

                                if (rav != null)
                                        UnrefReaderAndVersion_Unlocked (rav);
                                else
                                        reader.Close ();
                        }
                }
Esempio n. 32
0
                // Special function to permanently remove this reader
                // Called when StaticQueryables need to unload
                static internal void CloseReader (IndexReader reader)
                {
                        lock (reader_rav_map) {
                                ReaderAndVersion rav = (ReaderAndVersion) reader_rav_map [reader];

                                if (rav != null) {
                                        rav.Refcount --; // Remove one extra refcount that was added to keep the reader opened
                                        UnrefReaderAndVersion_Unlocked (rav);
                                } else
                                        reader.Close ();
                        }
                }
Esempio n. 33
0
        public virtual void TestExactFileNames()
        {
            System.String outputDir = "lucene.backwardscompat0.index";
            RmDir(outputDir);

            try
            {
                Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo(FullDir(outputDir)));

                IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true,
                                                     IndexWriter.MaxFieldLength.UNLIMITED);
                writer.SetRAMBufferSizeMB(16.0);
                for (int i = 0; i < 35; i++)
                {
                    AddDoc(writer, i);
                }
                Assert.AreEqual(35, writer.MaxDoc(), "wrong doc count");
                writer.Close();

                // Delete one doc so we get a .del file:
                IndexReader reader     = IndexReader.Open(dir, false);
                Term        searchTerm = new Term("id", "7");
                int         delCount   = reader.DeleteDocuments(searchTerm);
                Assert.AreEqual(1, delCount, "didn't delete the right number of documents");

                // Set one norm so we get a .s0 file:
                reader.SetNorm(21, "content", (float)1.5);
                reader.Close();

                // The numbering of fields can vary depending on which
                // JRE is in use.  On some JREs we see content bound to
                // field 0; on others, field 1.  So, here we have to
                // figure out which field number corresponds to
                // "content", and then set our expected file names below
                // accordingly:
                CompoundFileReader cfsReader  = new CompoundFileReader(dir, "_0.cfs");
                FieldInfos         fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
                int contentFieldIndex         = -1;
                for (int i = 0; i < fieldInfos.Size(); i++)
                {
                    FieldInfo fi = fieldInfos.FieldInfo(i);
                    if (fi.name_ForNUnit.Equals("content"))
                    {
                        contentFieldIndex = i;
                        break;
                    }
                }
                cfsReader.Close();
                Assert.IsTrue(contentFieldIndex != -1,
                              "could not locate the 'content' field number in the _2.cfs segment");

                // Now verify file names:
                System.String[] expected;
                expected = new System.String[]
                { "_0.cfs", "_0_1.del", "_0_1.s" + contentFieldIndex, "segments_3", "segments.gen" };

                System.String[] actual = dir.ListAll();
                System.Array.Sort(expected);
                System.Array.Sort(actual);
                if (!CollectionsHelper.Equals(expected, actual))
                {
                    Assert.Fail("incorrect filenames in index: expected:\n    " + AsString(expected) +
                                "\n  actual:\n    " + AsString(actual));
                }
                dir.Close();
            }
            finally
            {
                RmDir(outputDir);
            }
        }
        public void TestFieldContents_2()
        {
            try
            {
                indexReader = IndexReader.Open(store);

                ld = new LuceneDictionary(indexReader, "contents");
                it = ld.GetWordsIterator();
                
                // hasNext() should have no side effects //{{DIGY}} But has. Need a fix?
                //AssertTrue("First element isn't were it should be.", it.HasNext());
                //AssertTrue("First element isn't were it should be.", it.HasNext());
                //AssertTrue("First element isn't were it should be.", it.HasNext());

                // just iterate through words
                AssertTrue("First element isn't correct", it.Next().Equals("Jerry"));
                AssertTrue("Second element isn't correct", it.Next().Equals("Tom"));
                AssertTrue("Nonexistent element is really null", it.Next() == null);

                // hasNext() should still have no side effects ...
                AssertFalse("There should be any more elements", it.HasNext());
                AssertFalse("There should be any more elements", it.HasNext());
                AssertFalse("There should be any more elements", it.HasNext());

                // .. and there are really no more words
                AssertTrue("Nonexistent element is really null", it.Next() == null);
                AssertTrue("Nonexistent element is really null", it.Next() == null);
                AssertTrue("Nonexistent element is really null", it.Next() == null);
            }
            finally
            {
                if (indexReader != null) { indexReader.Close(); }
            }
        }
Esempio n. 35
0
        public virtual void  TestExpirationTimeDeletionPolicy()
        {
            double SECONDS = 2.0;

            bool autoCommit      = false;
            bool useCompoundFile = true;

            Directory dir = new RAMDirectory();
            ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(this, dir, SECONDS);
            IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);

            writer.SetUseCompoundFile(useCompoundFile);
            writer.Close();

            long lastDeleteTime = 0;

            for (int i = 0; i < 7; i++)
            {
                // Record last time when writer performed deletes of
                // past commits
                lastDeleteTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
                writer         = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
                writer.SetUseCompoundFile(useCompoundFile);
                for (int j = 0; j < 17; j++)
                {
                    AddDoc(writer);
                }
                writer.Close();

                // Make sure to sleep long enough so that some commit
                // points will be deleted:
                System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * (int)(1000.0 * (SECONDS / 5.0))));
            }

            // First, make sure the policy in fact deleted something:
            Assert.IsTrue(policy.numDelete > 0, "no commits were deleted");

            // Then simplistic check: just verify that the
            // segments_N's that still exist are in fact within SECONDS
            // seconds of the last one's mod time, and, that I can
            // open a reader on each:
            long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);

            System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
            dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
            while (gen > 0)
            {
                try
                {
                    IndexReader reader = IndexReader.Open(dir);
                    reader.Close();
                    fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
                    long modTime = dir.FileModified(fileName);
                    Assert.IsTrue(lastDeleteTime - modTime <= (SECONDS * 1000), "commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted");
                }
                catch (System.IO.IOException e)
                {
                    // OK
                    break;
                }

                dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
                gen--;
            }

            dir.Close();
        }
        public void TestFieldAaa()
        {
            try
            {
                indexReader = IndexReader.Open(store);

                ld = new LuceneDictionary(indexReader, "aaa");
                it = ld.GetWordsIterator();

                AssertTrue("First element doesn't exist.", it.HasNext());
                AssertTrue("First element isn't correct", it.Next().Equals("foo"));
                AssertFalse("More elements than expected", it.HasNext());
                AssertTrue("Nonexistent element is really null", it.Next() == null);
            }
            finally
            {
                if (indexReader != null) { indexReader.Close(); }
            }
        }
Esempio n. 37
0
        public virtual void  TestKeepAllDeletionPolicy()
        {
            for (int pass = 0; pass < 4; pass++)
            {
                bool autoCommit      = pass < 2;
                bool useCompoundFile = (pass % 2) > 0;

                // Never deletes a commit
                KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);

                Directory dir = new RAMDirectory();
                policy.dir = dir;

                IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
                writer.SetMaxBufferedDocs(10);
                writer.SetUseCompoundFile(useCompoundFile);
                writer.SetMergeScheduler(new SerialMergeScheduler());
                for (int i = 0; i < 107; i++)
                {
                    AddDoc(writer);
                    if (autoCommit && i % 10 == 0)
                    {
                        writer.Commit();
                    }
                }
                writer.Close();

                writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
                writer.SetUseCompoundFile(useCompoundFile);
                writer.Optimize();
                writer.Close();

                Assert.AreEqual(2, policy.numOnInit);
                if (!autoCommit)
                {
                    // If we are not auto committing then there should
                    // be exactly 2 commits (one per close above):
                    Assert.AreEqual(2, policy.numOnCommit);
                }

                // Test listCommits
                System.Collections.ICollection commits = IndexReader.ListCommits(dir);
                if (!autoCommit)
                {
                    // 1 from opening writer + 2 from closing writer
                    Assert.AreEqual(3, commits.Count);
                }
                // 1 from opening writer + 2 from closing writer +
                // 11 from calling writer.commit() explicitly above
                else
                {
                    Assert.AreEqual(14, commits.Count);
                }

                System.Collections.IEnumerator it = commits.GetEnumerator();
                // Make sure we can open a reader on each commit:
                while (it.MoveNext())
                {
                    IndexCommit commit = (IndexCommit)it.Current;
                    IndexReader r      = IndexReader.Open(commit, null);
                    r.Close();
                }

                // Simplistic check: just verify all segments_N's still
                // exist, and, I can open a reader on each:
                dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
                long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
                while (gen > 0)
                {
                    IndexReader reader = IndexReader.Open(dir);
                    reader.Close();
                    dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
                    gen--;

                    if (gen > 0)
                    {
                        // Now that we've removed a commit point, which
                        // should have orphan'd at least one index file.
                        // Open & close a writer and assert that it
                        // actually removed something:
                        int preCount = dir.ListAll().Length;
                        writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.LIMITED);
                        writer.Close();
                        int postCount = dir.ListAll().Length;
                        Assert.IsTrue(postCount < preCount);
                    }
                }

                dir.Close();
            }
        }
Esempio n. 38
0
        public virtual void  TestMapper()
        {
            TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);

            Assert.IsTrue(reader != null);
            SortedTermVectorMapper mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());

            reader.Get(0, mapper);
            var set_Renamed = mapper.TermVectorEntrySet;

            Assert.IsTrue(set_Renamed != null, "set is null and it shouldn't be");
            //three fields, 4 terms, all terms are the same
            Assert.IsTrue(set_Renamed.Count == 4, "set Size: " + set_Renamed.Count + " is not: " + 4);
            //Check offsets and positions
            for (System.Collections.IEnumerator iterator = set_Renamed.GetEnumerator(); iterator.MoveNext();)
            {
                TermVectorEntry tve = (TermVectorEntry)iterator.Current;
                Assert.IsTrue(tve != null, "tve is null and it shouldn't be");
                Assert.IsTrue(tve.GetOffsets() != null, "tve.getOffsets() is null and it shouldn't be");
                Assert.IsTrue(tve.GetPositions() != null, "tve.getPositions() is null and it shouldn't be");
            }

            mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
            reader.Get(1, mapper);
            set_Renamed = mapper.TermVectorEntrySet;
            Assert.IsTrue(set_Renamed != null, "set is null and it shouldn't be");
            //three fields, 4 terms, all terms are the same
            Assert.IsTrue(set_Renamed.Count == 4, "set Size: " + set_Renamed.Count + " is not: " + 4);
            //Should have offsets and positions b/c we are munging all the fields together
            for (System.Collections.IEnumerator iterator = set_Renamed.GetEnumerator(); iterator.MoveNext();)
            {
                TermVectorEntry tve = (TermVectorEntry)iterator.Current;
                Assert.IsTrue(tve != null, "tve is null and it shouldn't be");
                Assert.IsTrue(tve.GetOffsets() != null, "tve.getOffsets() is null and it shouldn't be");
                Assert.IsTrue(tve.GetPositions() != null, "tve.getPositions() is null and it shouldn't be");
            }


            FieldSortedTermVectorMapper fsMapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());

            reader.Get(0, fsMapper);
            var map = fsMapper.FieldToTerms;

            Assert.IsTrue(map.Count == testFields.Length, "map Size: " + map.Count + " is not: " + testFields.Length);
            for (var iterator = map.GetEnumerator(); iterator.MoveNext();)
            {
                var entry     = iterator.Current;
                var sortedSet = entry.Value;
                Assert.IsTrue(sortedSet.Count == 4, "sortedSet Size: " + sortedSet.Count + " is not: " + 4);
                for (var inner = sortedSet.GetEnumerator(); inner.MoveNext();)
                {
                    TermVectorEntry tve = inner.Current;
                    Assert.IsTrue(tve != null, "tve is null and it shouldn't be");
                    //Check offsets and positions.
                    Assert.IsTrue(tve != null, "tve is null and it shouldn't be");
                    System.String field = tve.Field;
                    if (field.Equals(testFields[0]))
                    {
                        //should have offsets

                        Assert.IsTrue(tve.GetOffsets() != null, "tve.getOffsets() is null and it shouldn't be");
                        Assert.IsTrue(tve.GetPositions() != null, "tve.getPositions() is null and it shouldn't be");
                    }
                    else if (field.Equals(testFields[1]))
                    {
                        //should not have offsets

                        Assert.IsTrue(tve.GetOffsets() == null, "tve.getOffsets() is not null and it shouldn't be");
                        Assert.IsTrue(tve.GetPositions() == null, "tve.getPositions() is not null and it shouldn't be");
                    }
                }
            }
            //Try mapper that ignores offs and positions
            fsMapper = new FieldSortedTermVectorMapper(true, true, new TermVectorEntryFreqSortedComparator());
            reader.Get(0, fsMapper);
            map = fsMapper.FieldToTerms;
            Assert.IsTrue(map.Count == testFields.Length, "map Size: " + map.Count + " is not: " + testFields.Length);
            for (var iterator = map.GetEnumerator(); iterator.MoveNext();)
            {
                var entry     = iterator.Current;
                var sortedSet = entry.Value;
                Assert.IsTrue(sortedSet.Count == 4, "sortedSet Size: " + sortedSet.Count + " is not: " + 4);
                for (var inner = sortedSet.GetEnumerator(); inner.MoveNext();)
                {
                    TermVectorEntry tve = inner.Current;
                    Assert.IsTrue(tve != null, "tve is null and it shouldn't be");
                    //Check offsets and positions.
                    Assert.IsTrue(tve != null, "tve is null and it shouldn't be");
                    System.String field = tve.Field;
                    if (field.Equals(testFields[0]))
                    {
                        //should have offsets

                        Assert.IsTrue(tve.GetOffsets() == null, "tve.getOffsets() is null and it shouldn't be");
                        Assert.IsTrue(tve.GetPositions() == null, "tve.getPositions() is null and it shouldn't be");
                    }
                    else if (field.Equals(testFields[1]))
                    {
                        //should not have offsets

                        Assert.IsTrue(tve.GetOffsets() == null, "tve.getOffsets() is not null and it shouldn't be");
                        Assert.IsTrue(tve.GetPositions() == null, "tve.getPositions() is not null and it shouldn't be");
                    }
                }
            }

            // test setDocumentNumber()
            IndexReader       ir = IndexReader.Open(dir, true);
            DocNumAwareMapper docNumAwareMapper = new DocNumAwareMapper();

            Assert.AreEqual(-1, docNumAwareMapper.GetDocumentNumber());

            ir.GetTermFreqVector(0, docNumAwareMapper);
            Assert.AreEqual(0, docNumAwareMapper.GetDocumentNumber());
            docNumAwareMapper.SetDocumentNumber(-1);

            ir.GetTermFreqVector(1, docNumAwareMapper);
            Assert.AreEqual(1, docNumAwareMapper.GetDocumentNumber());
            docNumAwareMapper.SetDocumentNumber(-1);

            ir.GetTermFreqVector(0, "f1", docNumAwareMapper);
            Assert.AreEqual(0, docNumAwareMapper.GetDocumentNumber());
            docNumAwareMapper.SetDocumentNumber(-1);

            ir.GetTermFreqVector(1, "f2", docNumAwareMapper);
            Assert.AreEqual(1, docNumAwareMapper.GetDocumentNumber());
            docNumAwareMapper.SetDocumentNumber(-1);

            ir.GetTermFreqVector(0, "f1", docNumAwareMapper);
            Assert.AreEqual(0, docNumAwareMapper.GetDocumentNumber());

            ir.Close();
        }
Esempio n. 39
0
        public virtual void  TestUpdateDocument()
        {
            bool optimize = true;

            Directory   dir1   = new MockRAMDirectory();
            IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);

            // create the index
            CreateIndexNoClose(!optimize, "index1", writer);

            // writer.flush(false, true, true);

            // get a reader
            IndexReader r1 = writer.GetReader();

            Assert.IsTrue(r1.IsCurrent());

            System.String id10 = r1.Document(10).GetField("id").StringValue();

            Document newDoc = r1.Document(10);

            newDoc.RemoveField("id");
            newDoc.Add(new Field("id", System.Convert.ToString(8000), Field.Store.YES, Field.Index.NOT_ANALYZED));
            writer.UpdateDocument(new Term("id", id10), newDoc);
            Assert.IsFalse(r1.IsCurrent());

            IndexReader r2 = writer.GetReader();

            Assert.IsTrue(r2.IsCurrent());
            Assert.AreEqual(0, Count(new Term("id", id10), r2));
            Assert.AreEqual(1, Count(new Term("id", System.Convert.ToString(8000)), r2));

            r1.Close();
            writer.Close();
            Assert.IsTrue(r2.IsCurrent());

            IndexReader r3 = IndexReader.Open(dir1);

            Assert.IsTrue(r3.IsCurrent());
            Assert.IsTrue(r2.IsCurrent());
            Assert.AreEqual(0, Count(new Term("id", id10), r3));
            Assert.AreEqual(1, Count(new Term("id", System.Convert.ToString(8000)), r3));

            writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
            Document doc = new Document();

            doc.Add(new Field("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));
            writer.AddDocument(doc);
            Assert.IsTrue(r2.IsCurrent());
            Assert.IsTrue(r3.IsCurrent());

            writer.Close();

            Assert.IsFalse(r2.IsCurrent());
            Assert.IsTrue(!r3.IsCurrent());

            r2.Close();
            r3.Close();

            dir1.Close();
        }
        private IndexReader ReplaceActiveReader(IndexReader outOfDateReader, object directoryProviderLock,
            IDirectoryProvider directoryProvider, IndexReader[] readers)
        {
            bool trace = log.IsInfoEnabled;
            IndexReader oldReader;
            bool closeOldReader = false;
            bool closeOutOfDateReader = false;
            IndexReader reader;
            /**
             * Since out of lock protection, can have multiple readers created in //
             * not worse than NotShared and limit the locking time, hence scalability
             */
            try
            {
                reader = IndexReader.Open(directoryProvider.Directory);
            }
            catch (IOException e)
            {
                throw new SearchException("Unable to open Lucene IndexReader", e);
            }
            lock (directoryProviderLock)
            {
                // Since not protected by a lock, other ones can have been added
                oldReader = activeSearchIndexReaders[directoryProvider] = reader;
                lock (semaphoreIndexReaderLock)
                {
                    searchIndexReaderSemaphores[reader] = new ReaderData(1, directoryProvider);
                    if (trace) log.Info("Semaphore: 1 for " + reader);
                    if (outOfDateReader != null)
                    {
                        ReaderData readerData;
                        searchIndexReaderSemaphores.TryGetValue(outOfDateReader, out readerData);
                        if (readerData == null)
                        {
                            closeOutOfDateReader = false; //already removed by another prevous thread
                        }
                        else if (readerData.Semaphore == 0)
                        {
                            searchIndexReaderSemaphores.Remove(outOfDateReader);
                            closeOutOfDateReader = true;
                        }
                        else
                        {
                            closeOutOfDateReader = false;
                        }
                    }

                    if (oldReader != null && oldReader != outOfDateReader)
                    {
                        ReaderData readerData = searchIndexReaderSemaphores[oldReader];
                        if (readerData == null)
                        {
                            log.Warn("Semaphore should not be null");
                            closeOldReader = true; //TODO should be true or false?
                        }
                        else if (readerData.Semaphore == 0)
                        {
                            searchIndexReaderSemaphores.Remove(oldReader);
                            closeOldReader = true;
                        }
                        else
                        {
                            closeOldReader = false;
                        }
                    }
                }
            }

            if (closeOutOfDateReader)
            {
                if (trace) log.Info("Closing out of date IndexReader " + outOfDateReader);
                try
                {
                    outOfDateReader.Close();
                }
                catch (IOException e)
                {
                    ReaderProviderHelper.Clean(readers);
                    throw new SearchException("Unable to close Lucene IndexReader", e);
                }
            }
            if (closeOldReader)
            {
                if (trace) log.Info("Closing old IndexReader " + oldReader);
                try
                {
                    oldReader.Close();
                }
                catch (IOException e)
                {
                    ReaderProviderHelper.Clean(readers);
                    throw new SearchException("Unable to close Lucene IndexReader", e);
                }
            }
            return reader;
        }
Esempio n. 41
0
        public virtual void  TestDuringAddIndexes()
        {
            Directory   dir1   = new MockRAMDirectory();
            IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);

            writer.SetInfoStream(infoStream);
            writer.SetMergeFactor(2);

            // create the index
            CreateIndexNoClose(false, "test", writer);
            writer.Commit();

            Directory[] dirs = new Directory[10];
            for (int i = 0; i < 10; i++)
            {
                dirs[i] = new MockRAMDirectory(dir1);
            }

            IndexReader r = writer.GetReader();

            int   NUM_THREAD = 5;
            float SECONDS    = 3;

            long endTime = (long)((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) + 1000.0 * SECONDS);

            System.Collections.IList excs = (System.Collections.IList)System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(new System.Collections.ArrayList()));

            SupportClass.ThreadClass[] threads = new SupportClass.ThreadClass[NUM_THREAD];
            for (int i = 0; i < NUM_THREAD; i++)
            {
                threads[i] = new AnonymousClassThread(endTime, writer, dirs, excs, this);
                threads[i].IsBackground = true;
                threads[i].Start();
            }

            int lastCount = 0;

            while ((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) < endTime)
            {
                IndexReader r2 = r.Reopen();
                if (r2 != r)
                {
                    r.Close();
                    r = r2;
                }
                Query q     = new TermQuery(new Term("indexname", "test"));
                int   count = new IndexSearcher(r).Search(q, 10).totalHits;
                Assert.IsTrue(count >= lastCount);
                lastCount = count;
            }

            for (int i = 0; i < NUM_THREAD; i++)
            {
                threads[i].Join();
            }

            Assert.AreEqual(0, excs.Count);
            writer.Close();

            _TestUtil.CheckIndex(dir1);
            r.Close();
            dir1.Close();
        }