Esempio n. 1
0
        public virtual void TestSimpleSkip()
        {
            Directory   dir    = new CountingRAMDirectory(this, new RAMDirectory());
            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer()).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat())).SetMergePolicy(NewLogMergePolicy()));
            Term        term   = new Term("test", "a");

            for (int i = 0; i < 5000; i++)
            {
                Document d1 = new Document();
                d1.Add(NewTextField(term.Field, term.Text(), Field.Store.NO));
                writer.AddDocument(d1);
            }
            writer.Commit();
            writer.ForceMerge(1);
            writer.Dispose();

            AtomicReader reader = GetOnlySegmentReader(DirectoryReader.Open(dir));

            for (int i = 0; i < 2; i++)
            {
                Counter = 0;
                DocsAndPositionsEnum tp = reader.TermPositionsEnum(term);
                CheckSkipTo(tp, 14, 185);  // no skips
                CheckSkipTo(tp, 17, 190);  // one skip on level 0
                CheckSkipTo(tp, 287, 200); // one skip on level 1, two on level 0

                // this test would fail if we had only one skip level,
                // because than more bytes would be read from the freqStream
                CheckSkipTo(tp, 4800, 250); // one skip on level 2
            }
        }
		public virtual void  TestSimpleSkip()
		{
			RAMDirectory dir = new RAMDirectory();
			IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
			Term term = new Term("test", "a");
			for (int i = 0; i < 5000; i++)
			{
				Document d1 = new Document();
				d1.Add(new Field(term.Field(), term.Text(), Field.Store.NO, Field.Index.ANALYZED));
				writer.AddDocument(d1);
			}
			writer.Flush();
			writer.Optimize();
			writer.Close();
			
			IndexReader reader = SegmentReader.GetOnlySegmentReader(dir);
			SegmentTermPositions tp = (SegmentTermPositions) reader.TermPositions();
			tp.freqStream_ForNUnit = new CountingStream(this, tp.freqStream_ForNUnit);
			
			for (int i = 0; i < 2; i++)
			{
				counter = 0;
				tp.Seek(term);
				
				CheckSkipTo(tp, 14, 185); // no skips
				CheckSkipTo(tp, 17, 190); // one skip on level 0
				CheckSkipTo(tp, 287, 200); // one skip on level 1, two on level 0
				
				// this test would fail if we had only one skip level,
				// because than more bytes would be read from the freqStream
				CheckSkipTo(tp, 4800, 250); // one skip on level 2
			}
		}
Esempio n. 3
0
 public override bool TermCompare(Term term)
 {
     prefix = base.GetPrefixTerm();
     if ((System.Object)term.Field() == (System.Object)prefix.Field() && term.Text().Equals(prefix.Text()))
     {
         return true;
     }
     endEnum = true;
     return false;
 }
		/// <summary>Equality compare on the term </summary>
		public override bool TermCompare(Term term)
		{
			if (_sField == term.Field())
			{
				string sSearchText = term.Text();
				if (sSearchText.StartsWith(_sPre)) return _regexImpl.Match(sSearchText);
			} //eif

			_bEndEnum = true;
			return false;
		}
Esempio n. 5
0
        public void  Set(Term term)
        {
            if (term == null)
            {
                Reset();
                return;
            }

            // copy text into the buffer
            SetTextLength(term.Text().Length);

            System.String sourceString = term.Text();
            int           sourceEnd    = term.Text().Length;

            for (int i = 0; i < sourceEnd; i++)
            {
                text[i] = (char)sourceString[i];
            }

            this.field = term.Field();
            this.term  = term;
        }
Esempio n. 6
0
        public virtual int[] ToDocsArray(Term term, IBits bits, IndexReader reader)
        {
            Fields    fields     = MultiFields.GetFields(reader);
            Terms     cterms     = fields.GetTerms(term.Field);
            TermsEnum ctermsEnum = cterms.GetIterator(null);

            if (ctermsEnum.SeekExact(new BytesRef(term.Text())))
            {
                DocsEnum docsEnum = TestUtil.Docs(Random, ctermsEnum, bits, null, DocsFlags.NONE);
                return(ToArray(docsEnum));
            }
            return(null);
        }
		public RegexTermEnum(IndexReader reader, Term term, IRegexCapabilities regexImpl)
		{
			_sField = term.Field();
			string sText = term.Text();
			
			_regexImpl = regexImpl;

			_regexImpl.Compile(sText);

			_sPre = _regexImpl.Prefix() ?? "";

			SetEnum(reader.Terms(new Term(term.Field(), _sPre)));
		}
Esempio n. 8
0
        public void TestSplitSeq()
        {
            MultiPassIndexSplitter splitter = new MultiPassIndexSplitter();

            Directory[] dirs = new Directory[] {
                NewDirectory(),
                NewDirectory(),
                NewDirectory()
            };
            try
            {
                splitter.Split(TEST_VERSION_CURRENT, input, dirs, true);
                Document    doc;
                int         start;
                IndexReader ir;
                using (ir = DirectoryReader.Open(dirs[0]))
                {
                    assertTrue(ir.NumDocs - NUM_DOCS / 3 <= 1);
                    doc = ir.Document(0);
                    assertEquals("0", doc.Get("id"));
                    start = ir.NumDocs;
                }
                using (ir = DirectoryReader.Open(dirs[1]))
                {
                    assertTrue(ir.NumDocs - NUM_DOCS / 3 <= 1);
                    doc = ir.Document(0);
                    assertEquals(start + "", doc.Get("id"));
                    start += ir.NumDocs;
                }
                using (ir = DirectoryReader.Open(dirs[2]))
                {
                    assertTrue(ir.NumDocs - NUM_DOCS / 3 <= 1);
                    doc = ir.Document(0);
                    assertEquals(start + "", doc.Get("id"));
                    // make sure the deleted doc is not here
                    TermsEnum te = MultiFields.GetTerms(ir, "id").GetIterator(null);
                    Term      t  = new Term("id", (NUM_DOCS - 1) + "");
                    assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.SeekCeil(new BytesRef(t.Text())));
                    assertNotSame(t.Text(), te.Term.Utf8ToString());
                }
            }
            finally
            {
                foreach (Directory d in dirs)
                {
                    d.Dispose();
                }
            }
        }
        public static int Count(Term t, IndexReader r)
        {
            int count = 0;
            DocsEnum td = TestUtil.Docs(Random(), r, t.Field(), new BytesRef(t.Text()), MultiFields.GetLiveDocs(r), null, 0);

            if (td != null)
            {
                while (td.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
                {
                    td.DocID();
                    count++;
                }
            }
            return count;
        }
Esempio n. 10
0
        public void  Set(Term term)
        {
            if (term == null)
            {
                Reset();
                return;
            }
            System.String termText = term.Text();
            int           termLen  = termText.Length;

            text.SetLength(termLen);
            SupportClass.TextSupport.GetCharsFromString(termText, 0, termLen, text.result, 0);
            dirty     = true;
            field     = term.Field();
            this.term = term;
        }
Esempio n. 11
0
        public void  Set(Term term)
        {
            if (term == null)
            {
                Reset();
                return;
            }
            string termText = term.Text();
            int    termLen  = termText.Length;

            text.setLength(termLen);
            for (int i = 0; i < termLen; i++)
            {
                text.result[i] = (char)termText[i];
            }
            dirty     = true;
            field     = term.Field();
            this.term = term;
        }
Esempio n. 12
0
        public virtual void  TestBasicDelete()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer     = null;
            IndexReader reader     = null;
            Term        searchTerm = new Term("content", "aaa");

            //  add 100 documents with term : aaa
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer, searchTerm.Text());
            }
            writer.Close();

            // OPEN READER AT THIS POINT - this should fix the view of the
            // index at the point of having 100 "aaa" documents and 0 "bbb"
            reader = IndexReader.Open(dir);
            Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
            AssertTermDocsCount("first reader", reader, searchTerm, 100);

            // DELETE DOCUMENTS CONTAINING TERM: aaa
            int deleted = 0;

            reader  = IndexReader.Open(dir);
            deleted = reader.Delete(searchTerm);
            Assert.AreEqual(100, deleted, "deleted count");
            Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
            AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
            reader.Close();

            // CREATE A NEW READER and re-test
            reader = IndexReader.Open(dir);
            Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
            AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
            reader.Close();
        }
Esempio n. 13
0
 public override Term VisitTerm(Term term)
 {
     if (term.Text().Equals(ContentQuery.EmptyText))
         return null;
     return base.VisitTerm(term);
 }
Esempio n. 14
0
        /// <summary>
        /// Create a new <see cref="SlowFuzzyQuery"/> that will match terms with a similarity 
        /// of at least <paramref name="minimumSimilarity"/> to <paramref name="term"/>.
        /// If a <paramref name="prefixLength"/> &gt; 0 is specified, a common prefix
        /// of that length is also required.
        /// </summary>
        /// <param name="term">the term to search for</param>
        /// <param name="minimumSimilarity">
        /// a value between 0 and 1 to set the required similarity
        /// between the query term and the matching terms. For example, for a
        /// <paramref name="minimumSimilarity"/> of <c>0.5</c> a term of the same length
        /// as the query term is considered similar to the query term if the edit distance
        /// between both terms is less than <c>length(term)*0.5</c>
        /// <para/>
        /// Alternatively, if <paramref name="minimumSimilarity"/> is >= 1f, it is interpreted
        /// as a pure Levenshtein edit distance. For example, a value of <c>2f</c>
        /// will match all terms within an edit distance of <c>2</c> from the
        /// query term. Edit distances specified in this way may not be fractional.
        /// </param>
        /// <param name="prefixLength">length of common (non-fuzzy) prefix</param>
        /// <param name="maxExpansions">
        /// the maximum number of terms to match. If this number is
        /// greater than <see cref="BooleanQuery.MaxClauseCount"/> when the query is rewritten,
        /// then the maxClauseCount will be used instead.
        /// </param>
        /// <exception cref="ArgumentException">
        /// if <paramref name="minimumSimilarity"/> is &gt;= 1 or &lt; 0
        /// or if <paramref name="prefixLength"/> &lt; 0
        /// </exception>
        public SlowFuzzyQuery(Term term, float minimumSimilarity, int prefixLength,
            int maxExpansions)
            : base(term.Field)
        {
            this.term = term;

            if (minimumSimilarity >= 1.0f && minimumSimilarity != (int)minimumSimilarity)
                throw new ArgumentException("fractional edit distances are not allowed");
            if (minimumSimilarity < 0.0f)
                throw new ArgumentException("minimumSimilarity < 0");
            if (prefixLength < 0)
                throw new ArgumentException("prefixLength < 0");
            if (maxExpansions < 0)
                throw new ArgumentException("maxExpansions < 0");

            SetRewriteMethod(new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(maxExpansions));

            string text = term.Text();
            int len = text.CodePointCount(0, text.Length);
            if (len > 0 && (minimumSimilarity >= 1f || len > 1.0f / (1.0f - minimumSimilarity)))
            {
                this.termLongEnough = true;
            }

            this.minimumSimilarity = minimumSimilarity;
            this.prefixLength = prefixLength;
        }
Esempio n. 15
0
		public void  Set(Term term)
		{
			if (term == null)
			{
				Reset();
				return ;
			}
			
			// copy text into the buffer
			SetTextLength(term.Text().Length);

			System.String sourceString = term.Text();
			int sourceEnd = term.Text().Length;
			for (int i = 0; i < sourceEnd; i++)
			{
				text[i] = (char) sourceString[i];
			}
			
			this.field = term.Field();
			this.term = term;
		}
        public virtual void TestSimpleSkip()
        {
            Directory dir = new CountingRAMDirectory(this, new RAMDirectory());
            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer()).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat())).SetMergePolicy(NewLogMergePolicy()));
            Term term = new Term("test", "a");
            for (int i = 0; i < 5000; i++)
            {
                Document d1 = new Document();
                d1.Add(NewTextField(term.Field(), term.Text(), Field.Store.NO));
                writer.AddDocument(d1);
            }
            writer.Commit();
            writer.ForceMerge(1);
            writer.Dispose();

            AtomicReader reader = GetOnlySegmentReader(DirectoryReader.Open(dir));

            for (int i = 0; i < 2; i++)
            {
                Counter = 0;
                DocsAndPositionsEnum tp = reader.TermPositionsEnum(term);
                CheckSkipTo(tp, 14, 185); // no skips
                CheckSkipTo(tp, 17, 190); // one skip on level 0
                CheckSkipTo(tp, 287, 200); // one skip on level 1, two on level 0

                // this test would fail if we had only one skip level,
                // because than more bytes would be read from the freqStream
                CheckSkipTo(tp, 4800, 250); // one skip on level 2
            }
        }
Esempio n. 17
0
 internal virtual void AssertTermDocsCount(string msg, IndexReader reader, Term term, int expected)
 {
     DocsEnum tdocs = TestUtil.Docs(Random(), reader, term.Field(), new BytesRef(term.Text()), MultiFields.GetLiveDocs(reader), null, 0);
     int count = 0;
     if (tdocs != null)
     {
         while (tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
         {
             count++;
         }
     }
     Assert.AreEqual(expected, count, msg + ", count mismatch");
 }
Esempio n. 18
0
 //--
 public virtual Term VisitTerm(Term term)
 {
     var field = term.Field();
     var text = term.Text();
     var visitedField = VisitField(field);
     var visitedText = VisitFieldText(text);
     if (field == visitedField && text == visitedText)
         return term;
     return new Term(visitedField, visitedText);
 }
Esempio n. 19
0
		private void  DeleteReaderWriterConflict(bool optimize)
		{
			//Directory dir = new RAMDirectory();
			Directory dir = GetDirectory(true);
			
			Term searchTerm = new Term("content", "aaa");
			Term searchTerm2 = new Term("content", "bbb");
			
			//  add 100 documents with term : aaa
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			for (int i = 0; i < 100; i++)
			{
				AddDoc(writer, searchTerm.Text());
			}
			writer.Close();
			
			// OPEN READER AT THIS POINT - this should fix the view of the
			// index at the point of having 100 "aaa" documents and 0 "bbb"
			IndexReader reader = IndexReader.Open(dir);
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
			Assert.AreEqual(0, reader.DocFreq(searchTerm2), "first docFreq");
			AssertTermDocsCount("first reader", reader, searchTerm, 100);
			AssertTermDocsCount("first reader", reader, searchTerm2, 0);
			
			// add 100 documents with term : bbb
			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
			for (int i = 0; i < 100; i++)
			{
				AddDoc(writer, searchTerm2.Text());
			}
			
			// REQUEST OPTIMIZATION
			// This causes a new segment to become current for all subsequent
			// searchers. Because of this, deletions made via a previously open
			// reader, which would be applied to that reader's segment, are lost
			// for subsequent searchers/readers
			if (optimize)
				writer.Optimize();
			writer.Close();
			
			// The reader should not see the new data
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
			Assert.AreEqual(0, reader.DocFreq(searchTerm2), "first docFreq");
			AssertTermDocsCount("first reader", reader, searchTerm, 100);
			AssertTermDocsCount("first reader", reader, searchTerm2, 0);
			
			
			// DELETE DOCUMENTS CONTAINING TERM: aaa
			// NOTE: the reader was created when only "aaa" documents were in
			int deleted = 0;
			try
			{
				deleted = reader.Delete(searchTerm);
				Assert.Fail("Delete allowed on an index reader with stale segment information");
			}
			catch (System.IO.IOException e)
			{
				/* success */
			}
			
			// Re-open index reader and try again. This time it should see
			// the new data.
			reader.Close();
			reader = IndexReader.Open(dir);
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
			Assert.AreEqual(100, reader.DocFreq(searchTerm2), "first docFreq");
			AssertTermDocsCount("first reader", reader, searchTerm, 100);
			AssertTermDocsCount("first reader", reader, searchTerm2, 100);
			
			deleted = reader.Delete(searchTerm);
			Assert.AreEqual(100, deleted, "deleted count");
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
			Assert.AreEqual(100, reader.DocFreq(searchTerm2), "deleted docFreq");
			AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
			AssertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
			reader.Close();
			
			// CREATE A NEW READER and re-test
			reader = IndexReader.Open(dir);
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
			Assert.AreEqual(100, reader.DocFreq(searchTerm2), "deleted docFreq");
			AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
			AssertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
			reader.Close();
		}
Esempio n. 20
0
 public virtual int[] ToDocsArray(Term term, Bits bits, IndexReader reader)
 {
     Fields fields = MultiFields.GetFields(reader);
     Terms cterms = fields.Terms(term.Field());
     TermsEnum ctermsEnum = cterms.Iterator(null);
     if (ctermsEnum.SeekExact(new BytesRef(term.Text())))
     {
         DocsEnum docsEnum = TestUtil.Docs(Random(), ctermsEnum, bits, null, DocsEnum.FLAG_NONE);
         return ToArray(docsEnum);
     }
     return null;
 }
Esempio n. 21
0
        private void  DeleteReaderReaderConflict(bool optimize)
        {
            Directory dir = GetDirectory(true);

            Term searchTerm1 = new Term("content", "aaa");
            Term searchTerm2 = new Term("content", "bbb");
            Term searchTerm3 = new Term("content", "ccc");

            //  add 100 documents with term : aaa
            //  add 100 documents with term : bbb
            //  add 100 documents with term : ccc
            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);

            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer, searchTerm1.Text());
                AddDoc(writer, searchTerm2.Text());
                AddDoc(writer, searchTerm3.Text());
            }
            if (optimize)
            {
                writer.Optimize();
            }
            writer.Close();

            // OPEN TWO READERS
            // Both readers get segment info as exists at this time
            IndexReader reader1 = IndexReader.Open(dir);

            Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "first opened");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "first opened");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "first opened");
            AssertTermDocsCount("first opened", reader1, searchTerm1, 100);
            AssertTermDocsCount("first opened", reader1, searchTerm2, 100);
            AssertTermDocsCount("first opened", reader1, searchTerm3, 100);

            IndexReader reader2 = IndexReader.Open(dir);

            Assert.AreEqual(100, reader2.DocFreq(searchTerm1), "first opened");
            Assert.AreEqual(100, reader2.DocFreq(searchTerm2), "first opened");
            Assert.AreEqual(100, reader2.DocFreq(searchTerm3), "first opened");
            AssertTermDocsCount("first opened", reader2, searchTerm1, 100);
            AssertTermDocsCount("first opened", reader2, searchTerm2, 100);
            AssertTermDocsCount("first opened", reader2, searchTerm3, 100);

            // DELETE DOCS FROM READER 2 and CLOSE IT
            // delete documents containing term: aaa
            // when the reader is closed, the segment info is updated and
            // the first reader is now stale
            reader2.Delete(searchTerm1);
            Assert.AreEqual(100, reader2.DocFreq(searchTerm1), "after delete 1");
            Assert.AreEqual(100, reader2.DocFreq(searchTerm2), "after delete 1");
            Assert.AreEqual(100, reader2.DocFreq(searchTerm3), "after delete 1");
            AssertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
            AssertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
            AssertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
            reader2.Close();

            // Make sure reader 1 is unchanged since it was open earlier
            Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "after delete 1");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "after delete 1");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "after delete 1");
            AssertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
            AssertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
            AssertTermDocsCount("after delete 1", reader1, searchTerm3, 100);


            // ATTEMPT TO DELETE FROM STALE READER
            // delete documents containing term: bbb
            try
            {
                reader1.Delete(searchTerm2);
                Assert.Fail("Delete allowed from a stale index reader");
            }
            catch (System.IO.IOException e)
            {
                /* success */
            }

            // RECREATE READER AND TRY AGAIN
            reader1.Close();
            reader1 = IndexReader.Open(dir);
            Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "reopened");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "reopened");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "reopened");
            AssertTermDocsCount("reopened", reader1, searchTerm1, 0);
            AssertTermDocsCount("reopened", reader1, searchTerm2, 100);
            AssertTermDocsCount("reopened", reader1, searchTerm3, 100);

            reader1.Delete(searchTerm2);
            Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "deleted 2");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "deleted 2");
            Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "deleted 2");
            AssertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
            AssertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
            AssertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
            reader1.Close();

            // Open another reader to confirm that everything is deleted
            reader2 = IndexReader.Open(dir);
            Assert.AreEqual(100, reader2.DocFreq(searchTerm1), "reopened 2");
            Assert.AreEqual(100, reader2.DocFreq(searchTerm2), "reopened 2");
            Assert.AreEqual(100, reader2.DocFreq(searchTerm3), "reopened 2");
            AssertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
            AssertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
            AssertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
            reader2.Close();

            dir.Close();
        }
Esempio n. 22
0
        private void  DeleteReaderWriterConflict(bool optimize)
        {
            //Directory dir = new RAMDirectory();
            Directory dir = GetDirectory(true);

            Term searchTerm  = new Term("content", "aaa");
            Term searchTerm2 = new Term("content", "bbb");

            //  add 100 documents with term : aaa
            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);

            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer, searchTerm.Text());
            }
            writer.Close();

            // OPEN READER AT THIS POINT - this should fix the view of the
            // index at the point of having 100 "aaa" documents and 0 "bbb"
            IndexReader reader = IndexReader.Open(dir);

            Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
            Assert.AreEqual(0, reader.DocFreq(searchTerm2), "first docFreq");
            AssertTermDocsCount("first reader", reader, searchTerm, 100);
            AssertTermDocsCount("first reader", reader, searchTerm2, 0);

            // add 100 documents with term : bbb
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer, searchTerm2.Text());
            }

            // REQUEST OPTIMIZATION
            // This causes a new segment to become current for all subsequent
            // searchers. Because of this, deletions made via a previously open
            // reader, which would be applied to that reader's segment, are lost
            // for subsequent searchers/readers
            if (optimize)
            {
                writer.Optimize();
            }
            writer.Close();

            // The reader should not see the new data
            Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
            Assert.AreEqual(0, reader.DocFreq(searchTerm2), "first docFreq");
            AssertTermDocsCount("first reader", reader, searchTerm, 100);
            AssertTermDocsCount("first reader", reader, searchTerm2, 0);


            // DELETE DOCUMENTS CONTAINING TERM: aaa
            // NOTE: the reader was created when only "aaa" documents were in
            int deleted = 0;

            try
            {
                deleted = reader.Delete(searchTerm);
                Assert.Fail("Delete allowed on an index reader with stale segment information");
            }
            catch (System.IO.IOException e)
            {
                /* success */
            }

            // Re-open index reader and try again. This time it should see
            // the new data.
            reader.Close();
            reader = IndexReader.Open(dir);
            Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
            Assert.AreEqual(100, reader.DocFreq(searchTerm2), "first docFreq");
            AssertTermDocsCount("first reader", reader, searchTerm, 100);
            AssertTermDocsCount("first reader", reader, searchTerm2, 100);

            deleted = reader.Delete(searchTerm);
            Assert.AreEqual(100, deleted, "deleted count");
            Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
            Assert.AreEqual(100, reader.DocFreq(searchTerm2), "deleted docFreq");
            AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
            AssertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
            reader.Close();

            // CREATE A NEW READER and re-test
            reader = IndexReader.Open(dir);
            Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
            Assert.AreEqual(100, reader.DocFreq(searchTerm2), "deleted docFreq");
            AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
            AssertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
            reader.Close();
        }
Esempio n. 23
0
        private void contextMenuItemShowAll_Click(object sender, System.EventArgs e)
        {
            if (listTerms.SelectedItems == null) return;

            ListViewItem selItem = listTerms.SelectedItems[0];
            if (selItem == null) return;

            string field = selItem.SubItems[2].Text.Trim().Substring(1, selItem.SubItems[2].Text.Trim().Length - 2);
            string text = selItem.SubItems[3].Text;

            if (field == null || text == null)
                return;

            Term t = new Term(field, text);

            _luke.Search(t.Field() + ":" + t.Text());
        }
 public void TestSplitSeq()
 {
     MultiPassIndexSplitter splitter = new MultiPassIndexSplitter();
     Directory[] dirs = new Directory[]{
         NewDirectory(),
         NewDirectory(),
         NewDirectory()
     };
     try
     {
         splitter.Split(TEST_VERSION_CURRENT, input, dirs, true);
         Document doc;
         int start;
         IndexReader ir;
         using (ir = DirectoryReader.Open(dirs[0]))
         {
             assertTrue(ir.NumDocs - NUM_DOCS / 3 <= 1);
             doc = ir.Document(0);
             assertEquals("0", doc.Get("id"));
             start = ir.NumDocs;
         }
         using (ir = DirectoryReader.Open(dirs[1]))
         {
             assertTrue(ir.NumDocs - NUM_DOCS / 3 <= 1);
             doc = ir.Document(0);
             assertEquals(start + "", doc.Get("id"));
             start += ir.NumDocs;
         }
         using (ir = DirectoryReader.Open(dirs[2]))
         {
             assertTrue(ir.NumDocs - NUM_DOCS / 3 <= 1);
             doc = ir.Document(0);
             assertEquals(start + "", doc.Get("id"));
             // make sure the deleted doc is not here
             TermsEnum te = MultiFields.GetTerms(ir, "id").Iterator(null);
             Term t = new Term("id", (NUM_DOCS - 1) + "");
             assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.SeekCeil(new BytesRef(t.Text())));
             assertNotSame(t.Text(), te.Term().Utf8ToString());
         }
     }
     finally
     {
         foreach (Directory d in dirs)
         {
             d.Dispose();
         }
     }
 }
Esempio n. 25
0
		public virtual void  TestWritingNormsNoReader()
		{
			Directory dir = new MockRAMDirectory();
			IndexWriter writer = null;
			IndexReader reader = null;
			Term searchTerm = new Term("content", "aaa");
			
			//  add 1 documents with term : aaa
			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			writer.SetUseCompoundFile(false);
			AddDoc(writer, searchTerm.Text());
			writer.Close();
			
			//  now open reader & set norm for doc 0 (writes to
			//  _0_1.s0)
			reader = IndexReader.Open(dir);
			reader.SetNorm(0, "content", (float) 2.0);
			reader.Close();
			
			//  now open reader again & set norm for doc 0 (writes to _0_2.s0)
			reader = IndexReader.Open(dir);
			reader.SetNorm(0, "content", (float) 2.0);
			reader.Close();
			Assert.IsFalse(dir.FileExists("_0_1.s0"), "failed to remove first generation norms file on writing second generation");
			
			dir.Close();
		}
Esempio n. 26
0
        private string GetTermText(Term term)
        {
            var fieldName = term.Field();
            var fieldText = term.Text();
            if (fieldText == null)
                return null;

            var info = SenseNet.ContentRepository.Schema.ContentTypeManager.GetPerFieldIndexingInfo(fieldName);
            if (info == null)
            {
                var c = fieldText.ToCharArray();
                for (int i = 0; i < c.Length; i++)
                    if (c[i] < ' ')
                        c[i] = '.';
                return new String(c);
            }
            var fieldHandler = info.IndexFieldHandler;
            switch (fieldHandler.IndexFieldType)
            {
                case SenseNet.Search.Indexing.IndexFieldType.String:
                    return GetTermText(fieldText);
                case SenseNet.Search.Indexing.IndexFieldType.Int:
                    return Convert.ToString(NumericUtils.PrefixCodedToInt(fieldText), CultureInfo.InvariantCulture);
                case SenseNet.Search.Indexing.IndexFieldType.Long:
                    return Convert.ToString(NumericUtils.PrefixCodedToLong(fieldText), CultureInfo.InvariantCulture);
                case SenseNet.Search.Indexing.IndexFieldType.Float:
                    return Convert.ToString(NumericUtils.PrefixCodedToFloat(fieldText), CultureInfo.InvariantCulture);
                case SenseNet.Search.Indexing.IndexFieldType.Double:
                    return Convert.ToString(NumericUtils.PrefixCodedToDouble(fieldText), CultureInfo.InvariantCulture);
                case SenseNet.Search.Indexing.IndexFieldType.DateTime:
                    var d = new DateTime(NumericUtils.PrefixCodedToLong(fieldText));
                    if (d.Hour == 0 && d.Minute == 0 && d.Second == 0)
                        return GetTermText(d.ToString("yyyy-MM-dd"));
                    if (d.Second == 0)
                        return GetTermText(d.ToString("yyyy-MM-dd HH:mm"));
                    return GetTermText(d.ToString("yyyy-MM-dd HH:mm:ss"));
                default:
                    throw new NotImplementedException("Unknown IndexFieldType: " + fieldHandler.IndexFieldType);
            }
        }
Esempio n. 27
0
		public virtual void  TestBasicDelete()
		{
			Directory dir = new RAMDirectory();
			
			IndexWriter writer = null;
			IndexReader reader = null;
			Term searchTerm = new Term("content", "aaa");
			
			//  add 100 documents with term : aaa
			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			for (int i = 0; i < 100; i++)
			{
				AddDoc(writer, searchTerm.Text());
			}
			writer.Close();
			
			// OPEN READER AT THIS POINT - this should fix the view of the
			// index at the point of having 100 "aaa" documents and 0 "bbb"
			reader = IndexReader.Open(dir);
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "first docFreq");
			AssertTermDocsCount("first reader", reader, searchTerm, 100);
			
			// DELETE DOCUMENTS CONTAINING TERM: aaa
			int deleted = 0;
			reader = IndexReader.Open(dir);
			deleted = reader.Delete(searchTerm);
			Assert.AreEqual(100, deleted, "deleted count");
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
			AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
			reader.Close();
			
			// CREATE A NEW READER and re-test
			reader = IndexReader.Open(dir);
			Assert.AreEqual(100, reader.DocFreq(searchTerm), "deleted docFreq");
			AssertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
			reader.Close();
		}
Esempio n. 28
0
		public void  Set(Term term)
		{
			if (term == null)
			{
				Reset();
				return ;
			}
			
			// copy text into the buffer
			SetTextLength(term.Text().Length);
			text = term.Text().ToCharArray();
			
			this.field = term.Field();
			this.term = term;
		}
Esempio n. 29
0
		private void  DeleteReaderReaderConflict(bool optimize)
		{
			Directory dir = GetDirectory(true);
			
			Term searchTerm1 = new Term("content", "aaa");
			Term searchTerm2 = new Term("content", "bbb");
			Term searchTerm3 = new Term("content", "ccc");
			
			//  add 100 documents with term : aaa
			//  add 100 documents with term : bbb
			//  add 100 documents with term : ccc
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			for (int i = 0; i < 100; i++)
			{
				AddDoc(writer, searchTerm1.Text());
				AddDoc(writer, searchTerm2.Text());
				AddDoc(writer, searchTerm3.Text());
			}
			if (optimize)
				writer.Optimize();
			writer.Close();
			
			// OPEN TWO READERS
			// Both readers get segment info as exists at this time
			IndexReader reader1 = IndexReader.Open(dir);
			Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "first opened");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "first opened");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "first opened");
			AssertTermDocsCount("first opened", reader1, searchTerm1, 100);
			AssertTermDocsCount("first opened", reader1, searchTerm2, 100);
			AssertTermDocsCount("first opened", reader1, searchTerm3, 100);
			
			IndexReader reader2 = IndexReader.Open(dir);
			Assert.AreEqual(100, reader2.DocFreq(searchTerm1), "first opened");
			Assert.AreEqual(100, reader2.DocFreq(searchTerm2), "first opened");
			Assert.AreEqual(100, reader2.DocFreq(searchTerm3), "first opened");
			AssertTermDocsCount("first opened", reader2, searchTerm1, 100);
			AssertTermDocsCount("first opened", reader2, searchTerm2, 100);
			AssertTermDocsCount("first opened", reader2, searchTerm3, 100);
			
			// DELETE DOCS FROM READER 2 and CLOSE IT
			// delete documents containing term: aaa
			// when the reader is closed, the segment info is updated and
			// the first reader is now stale
			reader2.Delete(searchTerm1);
			Assert.AreEqual(100, reader2.DocFreq(searchTerm1), "after delete 1");
			Assert.AreEqual(100, reader2.DocFreq(searchTerm2), "after delete 1");
			Assert.AreEqual(100, reader2.DocFreq(searchTerm3), "after delete 1");
			AssertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
			AssertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
			AssertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
			reader2.Close();
			
			// Make sure reader 1 is unchanged since it was open earlier
			Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "after delete 1");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "after delete 1");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "after delete 1");
			AssertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
			AssertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
			AssertTermDocsCount("after delete 1", reader1, searchTerm3, 100);
			
			
			// ATTEMPT TO DELETE FROM STALE READER
			// delete documents containing term: bbb
			try
			{
				reader1.Delete(searchTerm2);
				Assert.Fail("Delete allowed from a stale index reader");
			}
			catch (System.IO.IOException e)
			{
				/* success */
			}
			
			// RECREATE READER AND TRY AGAIN
			reader1.Close();
			reader1 = IndexReader.Open(dir);
			Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "reopened");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "reopened");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "reopened");
			AssertTermDocsCount("reopened", reader1, searchTerm1, 0);
			AssertTermDocsCount("reopened", reader1, searchTerm2, 100);
			AssertTermDocsCount("reopened", reader1, searchTerm3, 100);
			
			reader1.Delete(searchTerm2);
			Assert.AreEqual(100, reader1.DocFreq(searchTerm1), "deleted 2");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm2), "deleted 2");
			Assert.AreEqual(100, reader1.DocFreq(searchTerm3), "deleted 2");
			AssertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
			AssertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
			AssertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
			reader1.Close();
			
			// Open another reader to confirm that everything is deleted
			reader2 = IndexReader.Open(dir);
			Assert.AreEqual(100, reader2.DocFreq(searchTerm1), "reopened 2");
			Assert.AreEqual(100, reader2.DocFreq(searchTerm2), "reopened 2");
			Assert.AreEqual(100, reader2.DocFreq(searchTerm3), "reopened 2");
			AssertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
			AssertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
			AssertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
			reader2.Close();
			
			dir.Close();
		}
Esempio n. 30
0
        public virtual void TestSkipTo(int indexDivisor)
        {
            Directory   dir    = NewDirectory();
            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));

            Term ta = new Term("content", "aaa");

            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer, "aaa aaa aaa aaa");
            }

            Term tb = new Term("content", "bbb");

            for (int i = 0; i < 16; i++)
            {
                AddDoc(writer, "bbb bbb bbb bbb");
            }

            Term tc = new Term("content", "ccc");

            for (int i = 0; i < 50; i++)
            {
                AddDoc(writer, "ccc ccc ccc ccc");
            }

            // assure that we deal with a single segment
            writer.ForceMerge(1);
            writer.Dispose();

            IndexReader reader = DirectoryReader.Open(dir, indexDivisor);

            DocsEnum tdocs = TestUtil.Docs(Random(), reader, ta.Field(), new BytesRef(ta.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            // without optimization (assumption skipInterval == 16)

            // with next
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(0, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(1, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.Advance(2) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(2, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(4) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(4, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(9) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(9, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(10) != DocIdSetIterator.NO_MORE_DOCS);

            // without next
            tdocs = TestUtil.Docs(Random(), reader, ta.Field(), new BytesRef(ta.Text()), MultiFields.GetLiveDocs(reader), null, 0);

            Assert.IsTrue(tdocs.Advance(0) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(0, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(4) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(4, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(9) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(9, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(10) != DocIdSetIterator.NO_MORE_DOCS);

            // exactly skipInterval documents and therefore with optimization

            // with next
            tdocs = TestUtil.Docs(Random(), reader, tb.Field(), new BytesRef(tb.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(10, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(11, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.Advance(12) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(12, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(15) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(15, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(24) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(24, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(25) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(25, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(26) != DocIdSetIterator.NO_MORE_DOCS);

            // without next
            tdocs = TestUtil.Docs(Random(), reader, tb.Field(), new BytesRef(tb.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            Assert.IsTrue(tdocs.Advance(5) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(10, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(15) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(15, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(24) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(24, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(25) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(25, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(26) != DocIdSetIterator.NO_MORE_DOCS);

            // much more than skipInterval documents and therefore with optimization

            // with next
            tdocs = TestUtil.Docs(Random(), reader, tc.Field(), new BytesRef(tc.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(26, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(27, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.Advance(28) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(28, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(40) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(40, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(57) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(57, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(74) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(74, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(75) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(75, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(76) != DocIdSetIterator.NO_MORE_DOCS);

            //without next
            tdocs = TestUtil.Docs(Random(), reader, tc.Field(), new BytesRef(tc.Text()), MultiFields.GetLiveDocs(reader), null, 0);
            Assert.IsTrue(tdocs.Advance(5) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(26, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(40) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(40, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(57) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(57, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(74) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(74, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(75) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(75, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(76) != DocIdSetIterator.NO_MORE_DOCS);

            reader.Dispose();
            dir.Dispose();
        }
Esempio n. 31
0
 public void Set(Term term)
 {
     if (term == null)
     {
         Reset();
         return ;
     }
     string termText = term.Text();
     int termLen = termText.Length;
     text.setLength(termLen);
     for (int i = 0; i < termLen; i++)
     {
         text.result[i] = (char) termText[i];
     }
     dirty = true;
     field = term.Field();
     this.term = term;
 }
Esempio n. 32
0
        internal void _ShowTerm(Term t)
        {
            if (t == null)
            {
                _luke.ShowStatus(_luke.resources.GetString("NoTerms"));
                return;
            }
            if (_luke.IndexReader == null)
            {
                _luke.ShowStatus(_luke.resources.GetString("NoIndex"));
                return;
            }

            termDocs = null;
            this.term = t;
            comboTerms.SelectedItem = t.Field();
            textTerm.Text = t.Text();

            labelDocNum.Text = "?";
            labelTermFreq.Text = "?";

            try
            {
                int freq = _luke.IndexReader.DocFreq(t);
                labelDocFreq.Text = freq.ToString();
                labelDocMax.Text = freq.ToString();
            }
            catch (Exception e)
            {
                _luke.ShowStatus(e.Message);
                labelDocFreq.Text = "?";
            }
        }
Esempio n. 33
0
 public void Set(Term term)
 {
     if (term == null)
     {
         Reset();
         return ;
     }
     System.String termText = term.Text();
     int termLen = termText.Length;
     text.SetLength(termLen);
     SupportClass.TextSupport.GetCharsFromString(termText, 0, termLen, text.result, 0);
     dirty = true;
     field = term.Field();
     this.term = term;
 }
Esempio n. 34
0
		public virtual void  TestChangesAfterClose()
		{
			Directory dir = new RAMDirectory();
			
			IndexWriter writer = null;
			IndexReader reader = null;
			Term searchTerm = new Term("content", "aaa");
			
			//  add 11 documents with term : aaa
			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			for (int i = 0; i < 11; i++)
			{
				AddDoc(writer, searchTerm.Text());
			}
			writer.Close();
			
			reader = IndexReader.Open(dir);
			
			// Close reader:
			reader.Close();
			
			// Then, try to make changes:
			try
			{
				reader.DeleteDocument(4);
				Assert.Fail("deleteDocument after close failed to throw IOException");
			}
			catch (AlreadyClosedException)
			{
				// expected
			}
			
			try
			{
				reader.SetNorm(5, "aaa", 2.0f);
				Assert.Fail("setNorm after close failed to throw IOException");
			}
			catch (AlreadyClosedException)
			{
				// expected
			}
			
			try
			{
				reader.UndeleteAll();
				Assert.Fail("undeleteAll after close failed to throw IOException");
			}
			catch (AlreadyClosedException)
			{
				// expected
			}
		}
Esempio n. 35
0
        public virtual void TestSkipTo(int indexDivisor)
        {
            Directory dir = NewDirectory();
            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));

            Term ta = new Term("content", "aaa");
            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer, "aaa aaa aaa aaa");
            }

            Term tb = new Term("content", "bbb");
            for (int i = 0; i < 16; i++)
            {
                AddDoc(writer, "bbb bbb bbb bbb");
            }

            Term tc = new Term("content", "ccc");
            for (int i = 0; i < 50; i++)
            {
                AddDoc(writer, "ccc ccc ccc ccc");
            }

            // assure that we deal with a single segment
            writer.ForceMerge(1);
            writer.Dispose();

            IndexReader reader = DirectoryReader.Open(dir, indexDivisor);

            DocsEnum tdocs = TestUtil.Docs(Random(), reader, ta.Field(), new BytesRef(ta.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            // without optimization (assumption skipInterval == 16)

            // with next
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(0, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(1, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.Advance(2) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(2, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(4) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(4, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(9) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(9, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(10) != DocIdSetIterator.NO_MORE_DOCS);

            // without next
            tdocs = TestUtil.Docs(Random(), reader, ta.Field(), new BytesRef(ta.Text()), MultiFields.GetLiveDocs(reader), null, 0);

            Assert.IsTrue(tdocs.Advance(0) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(0, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(4) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(4, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(9) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(9, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(10) != DocIdSetIterator.NO_MORE_DOCS);

            // exactly skipInterval documents and therefore with optimization

            // with next
            tdocs = TestUtil.Docs(Random(), reader, tb.Field(), new BytesRef(tb.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(10, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(11, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.Advance(12) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(12, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(15) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(15, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(24) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(24, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(25) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(25, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(26) != DocIdSetIterator.NO_MORE_DOCS);

            // without next
            tdocs = TestUtil.Docs(Random(), reader, tb.Field(), new BytesRef(tb.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            Assert.IsTrue(tdocs.Advance(5) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(10, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(15) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(15, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(24) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(24, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(25) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(25, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(26) != DocIdSetIterator.NO_MORE_DOCS);

            // much more than skipInterval documents and therefore with optimization

            // with next
            tdocs = TestUtil.Docs(Random(), reader, tc.Field(), new BytesRef(tc.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);

            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(26, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(27, tdocs.DocID());
            Assert.AreEqual(4, tdocs.Freq());
            Assert.IsTrue(tdocs.Advance(28) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(28, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(40) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(40, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(57) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(57, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(74) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(74, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(75) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(75, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(76) != DocIdSetIterator.NO_MORE_DOCS);

            //without next
            tdocs = TestUtil.Docs(Random(), reader, tc.Field(), new BytesRef(tc.Text()), MultiFields.GetLiveDocs(reader), null, 0);
            Assert.IsTrue(tdocs.Advance(5) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(26, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(40) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(40, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(57) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(57, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(74) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(74, tdocs.DocID());
            Assert.IsTrue(tdocs.Advance(75) != DocIdSetIterator.NO_MORE_DOCS);
            Assert.AreEqual(75, tdocs.DocID());
            Assert.IsFalse(tdocs.Advance(76) != DocIdSetIterator.NO_MORE_DOCS);

            reader.Dispose();
            dir.Dispose();
        }
Esempio n. 36
0
		public virtual void  TestLockObtainFailed()
		{
			Directory dir = new RAMDirectory();
			
			IndexWriter writer = null;
			IndexReader reader = null;
			Term searchTerm = new Term("content", "aaa");
			
			//  add 11 documents with term : aaa
			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			for (int i = 0; i < 11; i++)
			{
				AddDoc(writer, searchTerm.Text());
			}
			
			// Create reader:
			reader = IndexReader.Open(dir);
			
			// Try to make changes
			try
			{
				reader.DeleteDocument(4);
				Assert.Fail("deleteDocument should have hit LockObtainFailedException");
			}
			catch (LockObtainFailedException)
			{
				// expected
			}
			
			try
			{
				reader.SetNorm(5, "aaa", 2.0f);
				Assert.Fail("setNorm should have hit LockObtainFailedException");
			}
			catch (LockObtainFailedException)
			{
				// expected
			}
			
			try
			{
				reader.UndeleteAll();
				Assert.Fail("undeleteAll should have hit LockObtainFailedException");
			}
			catch (LockObtainFailedException)
			{
				// expected
			}
			writer.Close();
			reader.Close();
		}
Esempio n. 37
0
		// Buffer a term in bufferedDeleteTerms, which records the
		// current number of documents buffered in ram so that the
		// delete term will be applied to those documents as well
		// as the disk segments.
		private void  AddDeleteTerm(Term term, int docCount)
		{
			lock (this)
			{
				Num num = (Num) bufferedDeleteTerms[term];
				if (num == null)
				{
					bufferedDeleteTerms[term] = new Num(docCount);
					// This is coarse approximation of actual bytes used:
					numBytesUsed += (term.Field().Length + term.Text().Length) * BYTES_PER_CHAR + 4 + 5 * OBJECT_HEADER_BYTES + 5 * OBJECT_POINTER_BYTES;
					if (ramBufferSize != IndexWriter.DISABLE_AUTO_FLUSH && numBytesUsed > ramBufferSize)
					{
						bufferIsFull = true;
					}
				}
				else
				{
					num.SetNum(docCount);
				}
				numBufferedDeleteTerms++;
			}
		}
Esempio n. 38
0
		public virtual void  TestWritingNorms()
		{
			System.String tempDir = SupportClass.AppSettings.Get("tempDir", "");
			if (tempDir == null)
				throw new System.IO.IOException("tempDir undefined, cannot run test");
			
			System.IO.FileInfo indexDir = new System.IO.FileInfo(tempDir + "\\" + "lucenetestnormwriter");
			Directory dir = FSDirectory.GetDirectory(indexDir);
			IndexWriter writer;
			IndexReader reader;
			Term searchTerm = new Term("content", "aaa");
			
			//  add 1 documents with term : aaa
			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			AddDoc(writer, searchTerm.Text());
			writer.Close();
			
			//  now open reader & set norm for doc 0
			reader = IndexReader.Open(dir);
			reader.SetNorm(0, "content", (float) 2.0);
			
			// we should be holding the write lock now:
			Assert.IsTrue(IndexReader.IsLocked(dir), "locked");
			
			reader.Commit();
			
			// we should not be holding the write lock now:
			Assert.IsTrue(!IndexReader.IsLocked(dir), "not locked");
			
			// open a 2nd reader:
			IndexReader reader2 = IndexReader.Open(dir);
			
			// set norm again for doc 0
			reader.SetNorm(0, "content", (float) 3.0);
			Assert.IsTrue(IndexReader.IsLocked(dir), "locked");
			
			reader.Close();
			
			// we should not be holding the write lock now:
			Assert.IsTrue(!IndexReader.IsLocked(dir), "not locked");
			
			reader2.Close();
			dir.Close();
			
			RmDir(indexDir);
		}
Esempio n. 39
0
 public void AddTerm(Term term, float boost)
 {
     QueryPhraseMap map = GetOrNewMap(subMap, term.Text());
     map.MarkTerminal(boost);
 }
Esempio n. 40
0
 private string TermToString(Term t)
 {
     var fieldName = t.Field();
     var value = t.Text();
     return String.Concat(fieldName, ":", value);
 }