IndexReader is an abstract class, providing an interface for accessing an index. Search of an index is done entirely through this abstract interface, so that any subclass which implements it is searchable.

There are two different types of IndexReaders:

  • AtomicReader: These indexes do not consist of several sub-readers, they are atomic. They support retrieval of stored fields, doc values, terms, and postings.
  • CompositeReader: Instances (like DirectoryReader) of this reader can only be used to get stored fields from the underlying AtomicReaders, but it is not possible to directly retrieve postings. To do that, get the sub-readers via CompositeReader#getSequentialSubReaders. Alternatively, you can mimic an AtomicReader (with a serious slowdown), by wrapping composite readers with SlowCompositeReaderWrapper.

IndexReader instances for indexes on disk are usually constructed with a call to one of the static DirectoryReader.open() methods, e.g. DirectoryReader#open(Lucene.Net.Store.Directory). DirectoryReader implements the CompositeReader interface, it is not possible to directly get postings.

For efficiency, in this API documents are often referred to via document numbers, non-negative integers which each name a unique document in the index. These document numbers are ephemeral -- they may change as documents are added to and deleted from an index. Clients should thus not rely on a given document having the same number between sessions.

NOTE: {@link IndexReader} instances are completely thread safe, meaning multiple threads can call any of its methods, concurrently. If your application requires external synchronization, you should not synchronize on the IndexReader instance; use your own (non-Lucene) objects instead.

Inheritance: IDisposable
Esempio n. 1
0
        /// <summary>
        /// Dispose(bool disposing) executes in two distinct scenarios.
        /// If disposing equals true, the method has been called directly
        /// or indirectly by a user's code. Managed and unmanaged resources
        /// can be disposed.
        /// If disposing equals false, the method has been called by the
        /// runtime from inside the finalizer and you should not reference
        /// other objects. Only unmanaged resources can be disposed.
        /// </summary>
        protected virtual void Dispose(bool disposing)
        {
            // Check to see if Dispose has already been called.
            if (!this._disposed)
            {
                // Note disposing has been done.
                _disposed = true;

                // If disposing equals true, dispose all managed
                // and unmanaged resources.
                if (disposing)
                {
                    if (_reader != null)
                    {
                        _reader.Dispose();
                    }

                    if (_facetReader != null)
                    {
                        _facetReader.Dispose();
                    }
                }

                // Call the appropriate methods to clean up
                // unmanaged resources here.
                _reader      = null;
                _facetReader = null;
            }
        }
Esempio n. 2
0
        public virtual void  TestNPESpanQuery()
        {
            Directory   dir    = new MockRAMDirectory();
            IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT, Support.Compatibility.SetFactory.CreateHashSet <string>()), IndexWriter.MaxFieldLength.LIMITED, null);

            // Add documents
            AddDoc(writer, "1", "the big dogs went running to the market");
            AddDoc(writer, "2", "the cat chased the mouse, then the cat ate the mouse quickly");

            // Commit
            writer.Close();

            // Get searcher
            IndexReader   reader   = IndexReader.Open(dir, true, null);
            IndexSearcher searcher = new IndexSearcher(reader);

            // Control (make sure docs indexed)
            Assert.AreEqual(2, HitCount(searcher, "the"));
            Assert.AreEqual(1, HitCount(searcher, "cat"));
            Assert.AreEqual(1, HitCount(searcher, "dogs"));
            Assert.AreEqual(0, HitCount(searcher, "rabbit"));

            // This throws exception (it shouldn't)
            Assert.AreEqual(1, searcher.Search(CreateSpan(0, true, new SpanQuery[] { CreateSpan(4, false, "chased", "cat"), CreateSpan("ate") }), 10, null).TotalHits);
            reader.Close();
            dir.Close();
        }
Esempio n. 3
0
        /// <summary>
        /// Full-Text search engine search provider, used to search indexed documents.
        /// </summary>
        /// <param name="directoryIndexInfos">The array directory infomation where the index files are located.</param>
        public SearchProvider(DirectoryInfo[] directoryIndexInfos)
        {
            try
            {
                List <Lucene.Net.Index.IndexReader> readers = new List <IndexReader>();

                // For each directory.
                foreach (DirectoryInfo item in directoryIndexInfos)
                {
                    // Create the index reader.
                    Lucene.Net.Store.Directory   directory = FSDirectory.Open(item);
                    Lucene.Net.Index.IndexReader reader    = Lucene.Net.Index.DirectoryReader.Open(directory);
                    readers.Add(reader);
                }

                // Create the multiple index readers.
                _reader = new Lucene.Net.Index.MultiReader(readers.ToArray(), true);
            }
            catch (Exception)
            {
                if (_reader != null)
                {
                    _reader.Dispose();
                }

                throw;
            }
        }
        public virtual void  Test()
        {
            IndexReader reader = null;

            try
            {
                reader = IndexReader.Open((Directory)directory, true, null);
                for (int i = 1; i <= numThreads; i++)
                {
                    TestTermPositionVectors(reader, i);
                }
            }
            catch (System.IO.IOException ioe)
            {
                Assert.Fail(ioe.Message);
            }
            finally
            {
                if (reader != null)
                {
                    try
                    {
                        /* close the opened reader */
                        reader.Close();
                    }
                    catch (System.IO.IOException ioe)
                    {
                        System.Console.Error.WriteLine(ioe.StackTrace);
                    }
                }
            }
        }
        public override void SetUp()
        {
            base.SetUp();
            dir = NewDirectory();
            var iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
            int numDocs = TestUtil.NextInt(Random(), 2049, 4000);
            for (int i = 0; i < numDocs; i++)
            {
                var document = new Document
				{
				    NewTextField("english", English.IntToEnglish(i), Field.Store.NO),
				    NewTextField("oddeven", (i%2 == 0) ? "even" : "odd", Field.Store.NO
				        ),
				    NewStringField("byte", string.Empty + (unchecked((byte) Random().Next
				        ())), Field.Store.NO),
				    NewStringField("short", string.Empty + ((short) Random().Next()), Field.Store
				        .NO),
				    new IntField("int", Random().Next(), Field.Store.NO),
				    new LongField("long", Random().NextLong(), Field.Store.NO),
				    new FloatField("float", Random().NextFloat(), Field.Store.NO),
				    new DoubleField("double", Random().NextDouble(), Field.Store.NO),
				    new NumericDocValuesField("intdocvalues", Random().Next()),
				    new FloatDocValuesField("floatdocvalues", Random().NextFloat())
				};
                iw.AddDocument(document);
            }
            reader = iw.Reader;
            iw.Dispose();
            searcher = NewSearcher(reader);
        }
        public void CloseReader(IndexReader reader)
        {
            bool trace = log.IsInfoEnabled;
            if (reader == null) return;
            IndexReader[] readers;

            // TODO: Java says don't force this to be CacheableMultiReader, but if we do we could avoid the reflection
            if (reader is MultiReader)
            {
                try
                {
                    // TODO: Need to account for Medium Trust - can't reflect on private members
                    readers = (IndexReader[]) subReadersField.GetValue(reader);
                }
                catch (Exception e)
                {
                    throw new SearchException("Incompatible version of Lucene: MultiReader.subReaders not accessible", e);
                }
                if (trace) log.Info("Closing MultiReader: " + reader);
            }
            else
            {
                throw new AssertionFailure("Everything should be wrapped in a MultiReader");
            }

            foreach (IndexReader subReader in readers)
                CloseInternalReader(trace, subReader, false);
        }
Esempio n. 7
0
            public virtual T Get(IndexReader reader, Entry key, IState state)
            {
                var innerCache = readerCache.GetOrCreateValue(reader.FieldCacheKey);
                var value      = innerCache.GetOrAdd(key, new Lazy <T>(() => CreateValue(reader, key, state)));

                if (value is Lazy <T> progress)
                {
                    lock (progress) // we need this lock because create value is expensive, we don't want to perform it several times.
                    {
                        if (progress.IsValueCreated == false)
                        {
                            innerCache[key] = progress.Value;

                            // Only check if key.custom (the parser) is
                            // non-null; else, we check twice for a single
                            // call to FieldCache.getXXX
                            if (key.custom != null && wrapper != null)
                            {
                                System.IO.StreamWriter infoStream = wrapper.InfoStream;
                                if (infoStream != null)
                                {
                                    PrintNewInsanity(infoStream, progress.Value);
                                }
                            }
                        }
                        return(progress.Value);
                    }
                }

                return((T)value);
            }
Esempio n. 8
0
            public override Explanation Explain(IndexReader ir, int i, IState state)
            {
                Explanation inner = weight.Explain(ir, i, state);

                if (Enclosing_Instance.Boost != 1)
                {
                    Explanation preBoost = inner;
                    inner = new Explanation(inner.Value * Enclosing_Instance.Boost, "product of:");
                    inner.AddDetail(new Explanation(Enclosing_Instance.Boost, "boost"));
                    inner.AddDetail(preBoost);
                }
                Filter           f                = Enclosing_Instance.filter;
                DocIdSet         docIdSet         = f.GetDocIdSet(ir, state);
                DocIdSetIterator docIdSetIterator = docIdSet == null?DocIdSet.EMPTY_DOCIDSET.Iterator(state) : docIdSet.Iterator(state);

                if (docIdSetIterator == null)
                {
                    docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.Iterator(state);
                }
                if (docIdSetIterator.Advance(i, state) == i)
                {
                    return(inner);
                }
                else
                {
                    Explanation result = new Explanation(0.0f, "failure to match filter: " + f.ToString());
                    result.AddDetail(inner);
                    return(result);
                }
            }
     public virtual Scorer CreateScorer(Scorer innerScorer, IndexReader reader, bool scoreDocsInOrder, bool topScorer)
     {
         if(!(reader is BoboIndexReader)) 
             throw new ArgumentException("IndexReader is not BoboIndexReader");
 
         return new FacetBasedBoostingScorer(this, (BoboIndexReader)reader, innerScorer.Similarity, innerScorer);
     }
Esempio n. 10
0
        /*(non-Javadoc) <see cref="Lucene.Net.Search.Query.rewrite(Lucene.Net.Index.IndexReader) */
        public override Query Rewrite(IndexReader reader)
        {
            CustomScoreQuery clone = null;

            Query sq = subQuery.Rewrite(reader);

            if (sq != subQuery)
            {
                clone          = (CustomScoreQuery)Clone();
                clone.subQuery = sq;
            }

            for (int i = 0; i < valSrcQueries.Length; i++)
            {
                ValueSourceQuery v = (ValueSourceQuery)valSrcQueries[i].Rewrite(reader);
                if (v != valSrcQueries[i])
                {
                    if (clone == null)
                    {
                        clone = (CustomScoreQuery)Clone();
                    }
                    clone.valSrcQueries[i] = v;
                }
            }

            return((clone == null) ? this : clone);
        }
        public override DocValues GetValues(IndexReader reader)
        {
            var @base = 0; //reader.DocBase;
            var vals = source.GetValues(reader);
            return new CachingDoubleDocValue(@base, vals, cache);

        }
Esempio n. 12
0
 public DocumentRepository(IndexWriter index, ILogWrapper log)
 {
     this.index = index;
     this.reader = this.index.GetReader();
     this.searcher = new IndexSearcher(this.reader);
     this.log = log;
 }
Esempio n. 13
0
 /// <summary>
 /// Expert: directly specify the reader, subReaders and their
 /// DocID starts
 /// <p/>
 /// <b>NOTE:</b> This API is experimental and
 /// might change in incompatible ways in the next
 /// release<p/>
 /// </summary>
 public IndexSearcher(IndexReader reader, IndexReader[] subReaders, int[] docStarts)
 {
     this.reader      = reader;
     this.subReaders  = subReaders;
     this.docStarts   = docStarts;
     this.closeReader = false;
 }
Esempio n. 14
0
            protected internal override string[] CreateValue(IndexReader reader, Entry entryKey, IState state)
            {
                System.String   field    = StringHelper.Intern(entryKey.field);
                System.String[] retArray = new System.String[reader.MaxDoc];
                TermDocs        termDocs = reader.TermDocs(state);
                TermEnum        termEnum = reader.Terms(new Term(field), state);

                try
                {
                    do
                    {
                        Term term = termEnum.Term;
                        if (term == null || (System.Object)term.Field != (System.Object)field)
                        {
                            break;
                        }
                        System.String termval = term.Text;
                        termDocs.Seek(termEnum, state);
                        while (termDocs.Next(state))
                        {
                            retArray[termDocs.Doc] = termval;
                        }
                    }while (termEnum.Next(state));
                }
                finally
                {
                    termDocs.Close();
                    termEnum.Close();
                }
                return(retArray);
            }
Esempio n. 15
0
        /// <summary>
        /// Inits this instance.
        /// </summary>
        public void Init()
        {
            server = new HttpServer();
            Url = server.Url;
            var indexDir = Content.Index;
            indexReader = IndexReader.Open(indexDir, true);
            indexSearcher = new IndexSearcher(indexReader);
            analyzer = new StandardAnalyzer(Version.LUCENE_29);

            server.ProcessRequest += ServerProcessRequest;
            server.Start();

            var labels = new StringBuilder();

            // Sort tags by alphabetical order
            Content.Tags.Sort((from, to) => from.Name.CompareTo(to.Name));

            foreach (var tagIndex in Content.Tags)
            {
                labels.AppendFormat(
                    @"<label style='float: left; margin-right: 8px'><input type='checkbox' name='{0}' checked='true'/>{1}</label>",
                    tagIndex.Id, tagIndex.Name).AppendLine();
            }
            searchOptions = labels.ToString();
        }
Esempio n. 16
0
        public override void SetUp()
        {
            base.SetUp();
            // we generate aweful regexps: good for testing.
            // but for preflex codec, the test can be very slow, so use less iterations.
            NumIterations = Codec.Default.Name.Equals("Lucene3x") ? 10 * RANDOM_MULTIPLIER : AtLeast(50);
            Dir = NewDirectory();
            RandomIndexWriter writer = new RandomIndexWriter(Random(), Dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.KEYWORD, false)).SetMaxBufferedDocs(TestUtil.NextInt(Random(), 50, 1000)));
            Document doc = new Document();
            Field field = NewStringField("field", "", Field.Store.YES);
            doc.Add(field);
            Terms = new SortedSet<BytesRef>();

            int num = AtLeast(200);
            for (int i = 0; i < num; i++)
            {
                string s = TestUtil.RandomUnicodeString(Random());
                field.StringValue = s;
                Terms.Add(new BytesRef(s));
                writer.AddDocument(doc);
            }

            TermsAutomaton = BasicAutomata.MakeStringUnion(Terms);

            Reader = writer.Reader;
            Searcher = NewSearcher(Reader);
            writer.Dispose();
        }
Esempio n. 17
0
        public virtual void  TestVerifyIndex()
        {
            IndexReader reader = IndexReader.Open(mDirectory, true, null);

            Assert.AreEqual(8, reader.NumDocs());
            reader.Close();
        }
        /// <summary>
        /// Set up a new index in RAM with three test phrases and the supplied Analyzer.
        /// </summary>
        /// <exception cref="Exception"> if an error occurs with index writer or searcher </exception>
        public override void SetUp()
        {
            base.SetUp();
            analyzer = new ShingleAnalyzerWrapper(new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false), 2);
            directory = NewDirectory();
            IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer));

            Document doc;
            doc = new Document();
            doc.Add(new TextField("content", "please divide this sentence into shingles", Field.Store.YES));
            writer.AddDocument(doc);

            doc = new Document();
            doc.Add(new TextField("content", "just another test sentence", Field.Store.YES));
            writer.AddDocument(doc);

            doc = new Document();
            doc.Add(new TextField("content", "a sentence which contains no test", Field.Store.YES));
            writer.AddDocument(doc);

            writer.Dispose();

            reader = DirectoryReader.Open(directory);
            searcher = NewSearcher(reader);
        }
Esempio n. 19
0
 internal MatchAllScorer(MatchAllDocsQuery enclosingInstance, IndexReader reader, Similarity similarity, Weight w, byte[] norms, IState state) : base(similarity)
 {
     InitBlock(enclosingInstance);
     this.termDocs = reader.TermDocs(null, state);
     score         = w.Value;
     this.norms    = norms;
 }
Esempio n. 20
0
        private static void doSpellCheckerIndexing(string LuceneIndexDir, string SpellCheckerIndexDir)
        {
            try
            {
                // http://lucene.apache.org/java/2_2_0/api/org/apache/lucene/search/spell/SpellChecker.html
                FSDirectory spellCheckerIndexDir = FSDirectory.GetDirectory(SpellCheckerIndexDir, false);
                FSDirectory indexDir             = FSDirectory.GetDirectory(LuceneIndexDir, false);

                SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(spellCheckerIndexDir);
                spellchecker.ClearIndex();
                // SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker (global::Lucene.Net.Store.Directory SpellChecker(spellIndexDirectory);

                IndexReader r = IndexReader.Open(indexDir);
                try
                {
                    // To index a field of a user index:
                    Dictionary dict = new SpellChecker.Net.Search.Spell.LuceneDictionary(r, "title");

                    spellchecker.IndexDictionary(dict);
                }
                finally
                {
                    r.Close();
                }
            }
            catch (Exception ex)
            {
                Console.Write("Could not create spell-checking index" + ex.Message);
            }
        }
Esempio n. 21
0
        } // SearchActiveDocument

        public string getSpellingSuggestion(string query)
        {
            FSDirectory indexDir = FSDirectory.GetDirectory(this.spellingIndexDir, false);

            SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(indexDir);
            IndexReader my_lucene_reader = IndexReader.Open(indexDir);

            string[]      words          = query.Split(new char[] { ' ', ',', ';' }, StringSplitOptions.RemoveEmptyEntries);
            List <string> allSuggestions = new List <string>();

            foreach (string word in words)
            {
                string[] suggestions = spellchecker.SuggestSimilar(word, 1);
                if (suggestions.Length > 0)
                {
                    allSuggestions.Add(suggestions[0]);
                }
                else
                {
                    allSuggestions.Add(word);
                }
            }

            string completeSuggestion = String.Join(" ", allSuggestions.ToArray());

            return(completeSuggestion);
        }
Esempio n. 22
0
        public override void  SetUp()
        {
            base.SetUp();
            RAMDirectory directory = new RAMDirectory();
            IndexWriter  writer    = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);
            long         theLong   = System.Int64.MaxValue;
            double       theDouble = System.Double.MaxValue;
            sbyte        theByte   = (sbyte)System.SByte.MaxValue;
            short        theShort  = System.Int16.MaxValue;
            int          theInt    = System.Int32.MaxValue;
            float        theFloat  = System.Single.MaxValue;

            for (int i = 0; i < NUM_DOCS; i++)
            {
                Document doc = new Document();
                doc.Add(new Field("theLong", System.Convert.ToString(theLong--), Field.Store.NO, Field.Index.NOT_ANALYZED));
                doc.Add(new Field("theDouble", (theDouble--).ToString("E16"), Field.Store.NO, Field.Index.NOT_ANALYZED));
                doc.Add(new Field("theByte", System.Convert.ToString((sbyte)theByte--), Field.Store.NO, Field.Index.NOT_ANALYZED));
                doc.Add(new Field("theShort", System.Convert.ToString(theShort--), Field.Store.NO, Field.Index.NOT_ANALYZED));
                doc.Add(new Field("theInt", System.Convert.ToString(theInt--), Field.Store.NO, Field.Index.NOT_ANALYZED));
                doc.Add(new Field("theFloat", (theFloat--).ToString("E8"), Field.Store.NO, Field.Index.NOT_ANALYZED));
                writer.AddDocument(doc, null);
            }
            writer.Close();
            reader = IndexReader.Open((Directory)directory, true, null);
        }
Esempio n. 23
0
            public override Explanation Explain(IndexReader reader, int doc)
            {
                var  cs     = new ConstantScorer(enclosingInstance, similarity, reader, this);
                bool exists = cs.docIdSetIterator.Advance(doc) == doc;

                var result = new ComplexExplanation();

                if (exists)
                {
                    result.Description = "ConstantScoreQuery(" + Enclosing_Instance.internalFilter + "), product of:";
                    result.Value       = queryWeight;
                    bool tempAux = true;
                    result.Match = tempAux;
                    result.AddDetail(new Explanation(Enclosing_Instance.Boost, "boost"));
                    result.AddDetail(new Explanation(queryNorm, "queryNorm"));
                }
                else
                {
                    result.Description = "ConstantScoreQuery(" + Enclosing_Instance.internalFilter + ") doesn't match id " + doc;
                    result.Value       = 0;
                    bool tempAux2 = false;
                    result.Match = tempAux2;
                }
                return(result);
            }
Esempio n. 24
0
        public override IComparable Generate(IndexReader reader, int doc)
        {
            var ravenDoc = reader.Document(doc);

            var payingTagField = ravenDoc.GetField("PayingTag_" + CustomTagId);
            var queriedPayingTag = payingTagField != null && Boolean.Parse(payingTagField.StringValue);
            var tagValue = Int32.Parse(ravenDoc.GetField("TagId").StringValue);
            var pointsValue = Int32.Parse(ravenDoc.GetField("Points").StringValue);

            CustomerDocumentOrderWithRandomEffect.OrderCategory cat;

            if (tagValue == CustomTagId && queriedPayingTag )
            {
                cat = CustomerDocumentOrderWithRandomEffect.OrderCategory.TagAndPaying;
            }
            else if (queriedPayingTag)
            {
                cat = CustomerDocumentOrderWithRandomEffect.OrderCategory.OnlyPaying;
            }
            else if (tagValue == CustomTagId )
            {
                cat = CustomerDocumentOrderWithRandomEffect.OrderCategory.OnlyTag;
            }
            else
            {
                cat = CustomerDocumentOrderWithRandomEffect.OrderCategory.NoneOfTheAbove;
            }

            return new CustomerDocumentOrderWithRandomEffect()
            {
                Category = cat,
                Points = pointsValue
            };
        }
Esempio n. 25
0
        public override Query Rewrite(IndexReader reader)
        {
            SpanOrQuery clone = null;

            for (int i = 0; i < clauses.Count; i++)
            {
                SpanQuery c     = clauses[i];
                SpanQuery query = (SpanQuery)c.Rewrite(reader);
                if (query != c)
                {
                    // clause rewrote: must clone
                    if (clone == null)
                    {
                        clone = (SpanOrQuery)this.Clone();
                    }
                    clone.clauses[i] = query;
                }
            }
            if (clone != null)
            {
                return(clone); // some clauses rewrote
            }
            else
            {
                return(this); // no clauses rewrote
            }
        }
Esempio n. 26
0
        private void AssertSumDocFreq(IndexReader ir)
        {
            // compute sumDocFreq across all fields
            Fields fields = MultiFields.GetFields(ir);

            foreach (string f in fields)
            {
                Terms terms = fields.Terms(f);
                long sumDocFreq = terms.SumDocFreq;
                if (sumDocFreq == -1)
                {
                    if (VERBOSE)
                    {
                        Console.WriteLine("skipping field: " + f + ", codec does not support sumDocFreq");
                    }
                    continue;
                }

                long computedSumDocFreq = 0;
                TermsEnum termsEnum = terms.Iterator(null);
                while (termsEnum.Next() != null)
                {
                    computedSumDocFreq += termsEnum.DocFreq();
                }
                Assert.AreEqual(computedSumDocFreq, sumDocFreq);
            }
        }
        public static Filter Create(IndexReader indexReader, bool includePrerelease, bool includeUnlisted)
        {
            IDictionary<string, OpenBitSet> openBitSetLookup = new Dictionary<string, OpenBitSet>();

            if (indexReader.GetSequentialSubReaders() != null)
            {
                foreach (SegmentReader segmentReader in indexReader.GetSequentialSubReaders())
                {
                    openBitSetLookup.Add(segmentReader.SegmentName, new OpenBitSet());
                }
            }
            else
            {
                openBitSetLookup.Add(string.Empty, new OpenBitSet());
            }

            IDictionary<string, Tuple<NuGetVersion, string, int>> lookup = MakeLatestVersionLookup(indexReader, includePrerelease, includeUnlisted);

            foreach (Tuple<NuGetVersion, string, int> entry in lookup.Values)
            {
                string readerName = entry.Item2;
                int readerDocumentId = entry.Item3;

                openBitSetLookup[readerName].Set(readerDocumentId);
            }

            return new OpenBitSetLookupFilter(openBitSetLookup);
        }
Esempio n. 28
0
        public override BitArray Bits(IndexReader reader)
        {
            if (chainedFilters.Count == 0)
            {
                throw new AssertionFailure("ChainedFilter has no filters to chain for");
            }

            // We need to copy the first BitArray because BitArray is assigned to by And
            Filter filter = chainedFilters[0];
            BitArray result = (BitArray)filter.Bits(reader).Clone();
            int size = result.Count;
            for (int index = 1; index < chainedFilters.Count; index++ )
            {
                BitArray b2 = chainedFilters[index].Bits(reader);
                int s2 = b2.Count;
                if (s2 != size)
                {
                    // Align the lengths, any extra elements are set to false, ok as as we are Anding
                    b2.Length = size;
                }

                // Stared at this for hours - C# compiler doesn't warn when you discard a function result!
                result = result.And(b2);
            }

            return result;
        }
 /// <summary>
 ///     Initializes a new instance of the <see cref="SearchResults" /> class.
 /// </summary>
 /// <param name="searcher">The searcher.</param>
 /// <param name="reader">The reader.</param>
 /// <param name="docs">The hits.</param>
 /// <param name="criteria">The criteria.</param>
 /// <param name="query">The query.</param>
 public LuceneSearchResults(Searcher searcher, IndexReader reader, TopDocs docs, ISearchCriteria criteria, Query query)
 {
     Results = new SearchResults(criteria, null);
     CreateDocuments(searcher, docs);
     CreateFacets(reader, query);
     CreateSuggestions(reader, criteria);
 }
		public override void  SetUp()
		{
			base.SetUp();
			Document doc;
			
			RAMDirectory rd1 = new RAMDirectory();
			IndexWriter iw1 = new IndexWriter(rd1, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
			
			doc = new Document();
			doc.Add(new Field("field1", "the quick brown fox jumps", Field.Store.YES, Field.Index.ANALYZED));
			doc.Add(new Field("field2", "the quick brown fox jumps", Field.Store.YES, Field.Index.ANALYZED));
			doc.Add(new Field("field4", "", Field.Store.NO, Field.Index.ANALYZED));
			iw1.AddDocument(doc);
			
			iw1.Close();
			RAMDirectory rd2 = new RAMDirectory();
			IndexWriter iw2 = new IndexWriter(rd2, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
			
			doc = new Document();
			doc.Add(new Field("field0", "", Field.Store.NO, Field.Index.ANALYZED));
			doc.Add(new Field("field1", "the fox jumps over the lazy dog", Field.Store.YES, Field.Index.ANALYZED));
			doc.Add(new Field("field3", "the fox jumps over the lazy dog", Field.Store.YES, Field.Index.ANALYZED));
			iw2.AddDocument(doc);
			
			iw2.Close();
			
			this.ir1 = IndexReader.Open(rd1, true);
		    this.ir2 = IndexReader.Open(rd2, true);
		}
Esempio n. 31
0
        public override Query Rewrite(IndexReader reader)
        {
            SpanNotQuery clone = null;

            SpanQuery rewrittenInclude = (SpanQuery)include.Rewrite(reader);

            if (rewrittenInclude != include)
            {
                clone         = (SpanNotQuery)this.Clone();
                clone.include = rewrittenInclude;
            }
            SpanQuery rewrittenExclude = (SpanQuery)exclude.Rewrite(reader);

            if (rewrittenExclude != exclude)
            {
                if (clone == null)
                {
                    clone = (SpanNotQuery)this.Clone();
                }
                clone.exclude = rewrittenExclude;
            }

            if (clone != null)
            {
                return(clone);                // some clauses rewrote
            }
            else
            {
                return(this);                // no clauses rewrote
            }
        }
Esempio n. 32
0
		private void  Initialize(IndexReader[] subReaders, bool closeSubReaders)
		{
			this.subReaders = new IndexReader[subReaders.Length];
			subReaders.CopyTo(this.subReaders, 0);
			starts = new int[subReaders.Length + 1]; // build starts array
			decrefOnClose = new bool[subReaders.Length];
			for (int i = 0; i < subReaders.Length; i++)
			{
				starts[i] = maxDoc;
				maxDoc += subReaders[i].MaxDoc(); // compute maxDocs
				
				if (!closeSubReaders)
				{
					subReaders[i].IncRef();
					decrefOnClose[i] = true;
				}
				else
				{
					decrefOnClose[i] = false;
				}
				
				if (subReaders[i].HasDeletions())
					hasDeletions = true;
			}
			starts[subReaders.Length] = maxDoc;
		}
            public override DocIdSet GetDocIdSet(IndexReader reader)
            {
                Lucene.Net.Search.StringIndex fcsi = Lucene.Net.Search.FieldCache_Fields.DEFAULT.GetStringIndex(reader, field);
                int lowerPoint = fcsi.BinarySearchLookup(lowerVal);
                int upperPoint = fcsi.BinarySearchLookup(upperVal);

                int inclusiveLowerPoint;
                int inclusiveUpperPoint;

                // Hints:
                // * binarySearchLookup returns 0, if value was null.
                // * the value is <0 if no exact hit was found, the returned value
                //   is (-(insertion point) - 1)
                if (lowerPoint == 0)
                {
                    System.Diagnostics.Debug.Assert(lowerVal == null);
                    inclusiveLowerPoint = 1;
                }
                else if (includeLower && lowerPoint > 0)
                {
                    inclusiveLowerPoint = lowerPoint;
                }
                else if (lowerPoint > 0)
                {
                    inclusiveLowerPoint = lowerPoint + 1;
                }
                else
                {
                    inclusiveLowerPoint = System.Math.Max(1, -lowerPoint - 1);
                }

                if (upperPoint == 0)
                {
                    System.Diagnostics.Debug.Assert(upperVal == null);
                    inclusiveUpperPoint = System.Int32.MaxValue;
                }
                else if (includeUpper && upperPoint > 0)
                {
                    inclusiveUpperPoint = upperPoint;
                }
                else if (upperPoint > 0)
                {
                    inclusiveUpperPoint = upperPoint - 1;
                }
                else
                {
                    inclusiveUpperPoint = -upperPoint - 2;
                }

                if (inclusiveUpperPoint <= 0 || inclusiveLowerPoint > inclusiveUpperPoint)
                {
                    return(DocIdSet.EMPTY_DOCIDSET);
                }

                System.Diagnostics.Debug.Assert(inclusiveLowerPoint > 0 && inclusiveUpperPoint > 0);

                // for this DocIdSet, we never need to use TermDocs,
                // because deleted docs have an order of 0 (null entry in StringIndex)
                return(new AnonymousClassFieldCacheDocIdSet(fcsi, inclusiveLowerPoint, inclusiveUpperPoint, this, reader, false));
            }
        public virtual Explanation Explain(IndexReader indexReader, int docid, Explanation innerExplaination)
        {
            if (!(indexReader is BoboIndexReader)) throw new ArgumentException("IndexReader is not BoboIndexReader");
            BoboIndexReader reader = (BoboIndexReader)indexReader;

            Explanation exp = new Explanation();
            exp.Description = "FacetBasedBoost";

            float boost = 1.0f;
            foreach (var boostEntry in _boostMaps)
            {
                string facetName = boostEntry.Key;
                IFacetHandler handler = reader.GetFacetHandler(facetName);
                if (!(handler is IFacetScoreable))
                    throw new ArgumentException(facetName + " does not implement IFacetScoreable");

                IFacetScoreable facetScoreable = (IFacetScoreable)handler;
                BoboDocScorer scorer = facetScoreable.GetDocScorer(reader, _scoringFunctionFactory, boostEntry.Value);
                float facetBoost = scorer.Score(docid);

                Explanation facetExp = new Explanation();
                facetExp.Description = facetName;
                facetExp.Value = facetBoost;
                facetExp.AddDetail(scorer.Explain(docid));
                boost *= facetBoost;
                exp.AddDetail(facetExp);
            }
            exp.Value = boost;
            exp.AddDetail(innerExplaination);
            return exp;
        }
            /* Explain the score we computed for doc */
            public override Explanation Explain(IndexReader reader, int doc)
            {
                if (Enclosing_Instance.disjuncts.Count == 1)
                {
                    return(weights[0].Explain(reader, doc));
                }
                ComplexExplanation result = new ComplexExplanation();
                float max = 0.0f, sum = 0.0f;

                result.Description = Enclosing_Instance.tieBreakerMultiplier == 0.0f?"max of:":"max plus " + Enclosing_Instance.tieBreakerMultiplier + " times others of:";
                foreach (Weight wt in weights)
                {
                    Explanation e = wt.Explain(reader, doc);
                    if (e.IsMatch)
                    {
                        System.Boolean tempAux = true;
                        result.Match = tempAux;
                        result.AddDetail(e);
                        sum += e.Value;
                        max  = System.Math.Max(max, e.Value);
                    }
                }
                result.Value = max + (sum - max) * Enclosing_Instance.tieBreakerMultiplier;
                return(result);
            }
Esempio n. 36
0
 /// <summary> Extracts all terms texts of a given Query into an array of WeightedTerms
 /// 
 /// </summary>
 /// <param name="query">Query to extract term texts from</param>
 /// <param name="reader">used to compute IDF which can be used to a) score selected fragments better 
 /// b) use graded highlights eg chaning intensity of font color</param>
 /// <param name="fieldName">the field on which Inverse Document Frequency (IDF) calculations are based</param>
 /// <returns> an array of the terms used in a query, plus their weights.</returns>
 public static WeightedTerm[] GetIdfWeightedTerms(Query query, IndexReader reader, string fieldName)
 {
     WeightedTerm[] terms = GetTerms(query, false, fieldName);
     int totalNumDocs = reader.NumDocs();
     foreach (WeightedTerm t in terms)
     {
         try
         {
             int docFreq = reader.DocFreq(new Term(fieldName, t.Term));
             // docFreq counts deletes
             if (totalNumDocs < docFreq)
             {
                 docFreq = totalNumDocs;
             }
             //IDF algorithm taken from DefaultSimilarity class
             var idf = (float)(Math.Log((float)totalNumDocs / (double)(docFreq + 1)) + 1.0);
             t.Weight *= idf;
         }
         catch (IOException e)
         {
             //ignore
         }
     }
     return terms;
 }
        public virtual void  TestEqualScores()
        {
            // NOTE: uses index build in *this* setUp

            IndexReader   reader = IndexReader.Open(small, true, null);
            IndexSearcher search = new IndexSearcher(reader);

            ScoreDoc[] result;

            // some hits match more terms then others, score should be the same

            result = search.Search(Csrq("data", "1", "6", T, T), null, 1000, null).ScoreDocs;
            int numHits = result.Length;

            AssertEquals("wrong number of results", 6, numHits);
            float score = result[0].Score;

            for (int i = 1; i < numHits; i++)
            {
                AssertEquals("score for " + i + " was not the same", score, result[i].Score);
            }

            result  = search.Search(Csrq("data", "1", "6", T, T, MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE), null, 1000, null).ScoreDocs;
            numHits = result.Length;
            AssertEquals("wrong number of results", 6, numHits);
            for (int i = 0; i < numHits; i++)
            {
                AssertEquals("score for " + i + " was not the same", score, result[i].Score);
            }
        }
Esempio n. 38
0
		/// <summary>Add an IndexReader whose stored fields will not be returned.  This can
		/// accellerate search when stored fields are only needed from a subset of
		/// the IndexReaders.
		/// 
		/// </summary>
		/// <throws>  IllegalArgumentException if not all indexes contain the same number  </throws>
		/// <summary>     of documents
		/// </summary>
		/// <throws>  IllegalArgumentException if not all indexes have the same value  </throws>
		/// <summary>     of {@link IndexReader#MaxDoc()}
		/// </summary>
		public virtual void  Add(IndexReader reader, bool ignoreStoredFields)
		{
			
			if (readers.Count == 0)
			{
				this.maxDoc = reader.MaxDoc();
				this.numDocs = reader.NumDocs();
				this.hasDeletions = reader.HasDeletions();
			}
			
			if (reader.MaxDoc() != maxDoc)
			// check compatibility
				throw new System.ArgumentException("All readers must have same maxDoc: " + maxDoc + "!=" + reader.MaxDoc());
			if (reader.NumDocs() != numDocs)
				throw new System.ArgumentException("All readers must have same numDocs: " + numDocs + "!=" + reader.NumDocs());
			
			System.Collections.IEnumerator i = reader.GetFieldNames(IndexReader.FieldOption.ALL).GetEnumerator();
			while (i.MoveNext())
			{
                System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) i.Current;

				// update fieldToReader map
				System.String field = fi.Key.ToString();
				if (fieldToReader[field] == null)
					fieldToReader[field] = reader;
			}
			
			if (!ignoreStoredFields)
				storedFieldReaders.Add(reader); // add to storedFieldReaders
			readers.Add(reader);
		}
		internal SegmentMergeInfo(int b, TermEnum te, IndexReader r)
		{
			base_Renamed = b;
			reader = r;
			termEnum = te;
			term = te.Term;
		}
        public DownloadsScoreProvider(IndexReader reader,
            IReadOnlyDictionary<string, int[]> idMapping,
            Downloads downloads,
            RankingResult ranking,
            QueryBoostingContext context,
            double baseBoost)
            : base(reader)
        {
            _idMapping = idMapping;
            _downloads = downloads;
            _baseBoost = baseBoost;
            _ranking = ranking;
            _context = context;

            // We need the reader name: Lucene *may* have multiple segments (which are smaller indices)
            // and RankingsHandler provides us with the per-segment document numbers.
            //
            // If no segments are present (small index) we use an empty string, which is what
            // Lucene also uses internally.
            var segmentReader = reader as SegmentReader;

            _readerName = segmentReader != null
                ? segmentReader.SegmentName
                : string.Empty;
        }
        public virtual void  TestBooleanOrderUnAffected()
        {
            // NOTE: uses index build in *this* setUp

            IndexReader   reader = IndexReader.Open(small, true, null);
            IndexSearcher search = new IndexSearcher(reader);

            // first do a regular TermRangeQuery which uses term expansion so
            // docs with more terms in range get higher scores

            Query rq = new TermRangeQuery("data", "1", "4", T, T);

            ScoreDoc[] expected = search.Search(rq, null, 1000, null).ScoreDocs;
            int        numHits  = expected.Length;

            // now do a boolean where which also contains a
            // ConstantScoreRangeQuery and make sure hte order is the same

            BooleanQuery q = new BooleanQuery();

            q.Add(rq, Occur.MUST);                           // T, F);
            q.Add(Csrq("data", "1", "6", T, T), Occur.MUST); // T, F);

            ScoreDoc[] actual = search.Search(q, null, 1000, null).ScoreDocs;

            AssertEquals("wrong numebr of hits", numHits, actual.Length);
            for (int i = 0; i < numHits; i++)
            {
                AssertEquals("mismatch in docid for hit#" + i, expected[i].Doc, actual[i].Doc);
            }
        }
        public void Process(IndexReader indexReader)
        {
            var perIndexDocumentNumber = 0;

            foreach (var handler in _handlers)
            {
                handler.Begin(indexReader);
            }

            if (_enumerateSubReaders && indexReader.GetSequentialSubReaders() != null)
            {
                foreach (SegmentReader segmentReader in indexReader.GetSequentialSubReaders())
                {
                    ProcessReader(segmentReader, segmentReader.SegmentName, ref perIndexDocumentNumber);
                }
            }
            else
            {
                ProcessReader(indexReader, string.Empty, ref perIndexDocumentNumber);
            }

            foreach (var handler in _handlers)
            {
                handler.End(indexReader);
            }
        }
        public virtual void  TestFarsi()
        {
            /* build an index */
            RAMDirectory farsiIndex = new RAMDirectory();
            IndexWriter  writer     = new IndexWriter(farsiIndex, new SimpleAnalyzer(), T, IndexWriter.MaxFieldLength.LIMITED, null);
            Document     doc        = new Document();

            doc.Add(new Field("content", "\u0633\u0627\u0628", Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("body", "body", Field.Store.YES, Field.Index.NOT_ANALYZED));
            writer.AddDocument(doc, null);

            writer.Optimize(null);
            writer.Close();

            IndexReader   reader = IndexReader.Open((Directory)farsiIndex, true, null);
            IndexSearcher search = new IndexSearcher(reader);

            // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
            // RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
            // characters properly.
            System.Globalization.CompareInfo c = new System.Globalization.CultureInfo("ar").CompareInfo;

            // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
            // orders the U+0698 character before the U+0633 character, so the single
            // index Term below should NOT be returned by a ConstantScoreRangeQuery
            // with a Farsi Collator (or an Arabic one for the case when Farsi is
            // not supported).
            ScoreDoc[] result = search.Search(Csrq("content", "\u062F", "\u0698", T, T, c), null, 1000, null).ScoreDocs;
            AssertEquals("The index Term should not be included.", 0, result.Length);

            result = search.Search(Csrq("content", "\u0633", "\u0638", T, T, c), null, 1000, null).ScoreDocs;
            AssertEquals("The index Term should be included.", 1, result.Length);
            search.Close();
        }
        private IList<SearchResult> RunQuery(Query query)
        {
            // If two threads ran this method simultaneously, there would be issues with this.IndexReader.
            // Alternatively, there could be one RAMDirectory per filesystem directory.
            lock (this)
            {
                IndexReader newReader = this.indexReader.Reopen();
                if (newReader != this.indexReader)
                {
                    this.indexReader.Dispose();
                    this.indexReader = newReader;
                }

                IndexSearcher searcher; searcher = new IndexSearcher(newReader);
                if (query == null)
                {
                    return new List<SearchResult>();
                }
                TopDocs hits = searcher.Search(query, 1000);

                return hits.ScoreDocs
                    .Select(scoreDoc => searcher.Doc(scoreDoc.Doc).Get(FullTextFieldType.Path.ToString()))
                    .Select(path => new SearchResult(path))
                    .ToList();
            }
        }
        public virtual void  TestDanish()
        {
            /* build an index */
            RAMDirectory danishIndex = new RAMDirectory();
            IndexWriter  writer      = new IndexWriter(danishIndex, new SimpleAnalyzer(), T, IndexWriter.MaxFieldLength.LIMITED, null);

            // Danish collation orders the words below in the given order
            // (example taken from TestSort.testInternationalSort() ).
            System.String[] words = new System.String[] { "H\u00D8T", "H\u00C5T", "MAND" };
            for (int docnum = 0; docnum < words.Length; ++docnum)
            {
                Document doc = new Document();
                doc.Add(new Field("content", words[docnum], Field.Store.YES, Field.Index.NOT_ANALYZED));
                doc.Add(new Field("body", "body", Field.Store.YES, Field.Index.NOT_ANALYZED));
                writer.AddDocument(doc, null);
            }
            writer.Optimize(null);
            writer.Close();

            IndexReader   reader = IndexReader.Open((Directory)danishIndex, true, null);
            IndexSearcher search = new IndexSearcher(reader);

            System.Globalization.CompareInfo c = new System.Globalization.CultureInfo("da" + "-" + "dk").CompareInfo;

            // Unicode order would not include "H\u00C5T" in [ "H\u00D8T", "MAND" ],
            // but Danish collation does.
            ScoreDoc[] result = search.Search(Csrq("content", "H\u00D8T", "MAND", F, F, c), null, 1000, null).ScoreDocs;
            AssertEquals("The index Term should be included.", 1, result.Length);

            result = search.Search(Csrq("content", "H\u00C5T", "MAND", F, F, c), null, 1000, null).ScoreDocs;
            AssertEquals("The index Term should not be included.", 0, result.Length);
            search.Close();
        }
        public void Process(IndexReader indexReader,
            string readerName,
            int perSegmentDocumentNumber,
            int perIndexDocumentNumber,
            Document document,
            string id,
            NuGetVersion version)
        {
            // main index docid
            if (id == null || version == null)
            {
                return;
            }

            List<RegistrationEntry> versions;
            if (!_registrations.TryGetValue(id, out versions))
            {
                versions = new List<RegistrationEntry>();
                _registrations.Add(id, versions);
            }

            var entry = new RegistrationEntry(perIndexDocumentNumber, version, GetListed(document));

            versions.Add(entry);
        }
Esempio n. 47
0
        public virtual void  TestRangeFilterRand()
        {
            IndexReader   reader = IndexReader.Open((Directory)signedIndex.index, true, null);
            IndexSearcher search = new IndexSearcher(reader);

            System.String minRP = Pad(signedIndex.minR);
            System.String maxRP = Pad(signedIndex.maxR);

            int numDocs = reader.NumDocs();

            Assert.AreEqual(numDocs, 1 + maxId - minId, "num of docs");

            ScoreDoc[] result;
            Query      q = new TermQuery(new Term("body", "body"));

            // test extremes, bounded on both ends

            result = search.Search(q, new TermRangeFilter("rand", minRP, maxRP, T, T), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs, result.Length, "find all");

            result = search.Search(q, new TermRangeFilter("rand", minRP, maxRP, T, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs - 1, result.Length, "all but biggest");

            result = search.Search(q, new TermRangeFilter("rand", minRP, maxRP, F, T), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs - 1, result.Length, "all but smallest");

            result = search.Search(q, new TermRangeFilter("rand", minRP, maxRP, F, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs - 2, result.Length, "all but extremes");

            // unbounded

            result = search.Search(q, new TermRangeFilter("rand", minRP, null, T, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs, result.Length, "smallest and up");

            result = search.Search(q, new TermRangeFilter("rand", null, maxRP, F, T), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs, result.Length, "biggest and down");

            result = search.Search(q, new TermRangeFilter("rand", minRP, null, F, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs - 1, result.Length, "not smallest, but up");

            result = search.Search(q, new TermRangeFilter("rand", null, maxRP, F, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(numDocs - 1, result.Length, "not biggest, but down");

            // very small sets

            result = search.Search(q, new TermRangeFilter("rand", minRP, minRP, F, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(0, result.Length, "min,min,F,F");
            result = search.Search(q, new TermRangeFilter("rand", maxRP, maxRP, F, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(0, result.Length, "max,max,F,F");

            result = search.Search(q, new TermRangeFilter("rand", minRP, minRP, T, T), numDocs, null).ScoreDocs;
            Assert.AreEqual(1, result.Length, "min,min,T,T");
            result = search.Search(q, new TermRangeFilter("rand", null, minRP, F, T), numDocs, null).ScoreDocs;
            Assert.AreEqual(1, result.Length, "nul,min,F,T");

            result = search.Search(q, new TermRangeFilter("rand", maxRP, maxRP, T, T), numDocs, null).ScoreDocs;
            Assert.AreEqual(1, result.Length, "max,max,T,T");
            result = search.Search(q, new TermRangeFilter("rand", maxRP, null, T, F), numDocs, null).ScoreDocs;
            Assert.AreEqual(1, result.Length, "max,nul,T,T");
        }
Esempio n. 48
0
 private void VerifyCount(IndexReader ir)
 {
     Fields fields = MultiFields.GetFields(ir);
     if (fields == null)
     {
         return;
     }
     foreach (string field in fields)
     {
         Terms terms = fields.Terms(field);
         if (terms == null)
         {
             continue;
         }
         int docCount = terms.DocCount;
         FixedBitSet visited = new FixedBitSet(ir.MaxDoc);
         TermsEnum te = terms.Iterator(null);
         while (te.Next() != null)
         {
             DocsEnum de = TestUtil.Docs(Random(), te, null, null, DocsEnum.FLAG_NONE);
             while (de.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
             {
                 visited.Set(de.DocID());
             }
         }
         Assert.AreEqual(visited.Cardinality(), docCount);
     }
 }
 public virtual void  Init(IndexReader reader)
 {
     this.reader = reader;
     timeElapsed = 0;
     t           = new ThreadClass(new System.Threading.ThreadStart(this.Run));
     t.Start();
 }
		public SuggestionQueryResult Query(SuggestionQuery suggestionQuery, IndexReader indexReader)
		{
			if (suggestionQuery.Accuracy.HasValue == false)
				throw new InvalidOperationException("SuggestionQuery.Accuracy must be specified.");

			if (suggestionQuery.Distance.HasValue == false)
				throw new InvalidOperationException("SuggestionQuery.Distance must be specified.");

			spellChecker.setStringDistance(SuggestionQueryRunner.GetStringDistance(suggestionQuery.Distance.Value));
			spellChecker.SetAccuracy(suggestionQuery.Accuracy.Value);

			if (suggestionQuery.Term.StartsWith("<<") && suggestionQuery.Term.EndsWith(">>"))
			{
				return QueryOverMultipleWords(suggestionQuery, indexReader,
					suggestionQuery.Term.Substring(2, suggestionQuery.Term.Length - 4));
			}
			if (suggestionQuery.Term.StartsWith("(") && suggestionQuery.Term.EndsWith(")"))
			{
				return QueryOverMultipleWords(suggestionQuery, indexReader,
					suggestionQuery.Term.Substring(1, suggestionQuery.Term.Length - 2));
			}
			string[] suggestions = spellChecker.SuggestSimilar(suggestionQuery.Term,
															   suggestionQuery.MaxSuggestions,
															   indexReader,
															   suggestionQuery.Field,
															   true);

			return new SuggestionQueryResult
			{
				Suggestions = suggestions
			};
		}
 public static OpenBitSet CreateBitSet(IndexReader reader, Filter filter)
 {
     IndexSearcher searcher = new IndexSearcher(reader);
     OpenBitSet result = new OpenBitSet();
     searcher.Search(new MatchAllDocsQuery(), filter, new BitSetCollector(result));
     return result;
 }
Esempio n. 52
0
        public override Query Rewrite(IndexReader reader)
        {
            CustomScoreQuery clone = null;

            Query sq = subQuery.Rewrite(reader);
            if (sq != subQuery)
            {
                clone = (CustomScoreQuery)Clone();
                clone.subQuery = sq;
            }

            for (int i = 0; i < scoringQueries.Length; i++)
            {
                Query v = scoringQueries[i].Rewrite(reader);
                if (v != scoringQueries[i])
                {
                    if (clone == null)
                    {
                        clone = (CustomScoreQuery)Clone();
                    }
                    clone.scoringQueries[i] = v;
                }
            }

            return clone ?? this;
        }
Esempio n. 53
0
        internal static CommitUserData ReadCommitUserData(IndexReader reader)
        {
            int lastActivityId = 0;
            var gap = new List<int>();

            var cud = reader.GetCommitUserData();
            if (cud != null)
            {
                if (cud.ContainsKey(IndexManager.LastActivityIdKey))
                {
                    var lastID = cud[IndexManager.LastActivityIdKey];
                    if (!string.IsNullOrEmpty(lastID))
                        int.TryParse(lastID, out lastActivityId);
                }
                if (cud.ContainsKey(IndexManager.MissingActivitiesKey))
                {
                    var gapstring = cud[IndexManager.MissingActivitiesKey];
                    int g;
                    if (!string.IsNullOrEmpty(gapstring))
                        foreach (var s in gapstring.Split(','))
                            if (Int32.TryParse(s, out g))
                                gap.Add(g);
                }
            }
            return new CommitUserData { LastActivityId = lastActivityId, Gap = gap };
        }
			internal MatchAllScorer(MatchAllDocsQuery enclosingInstance, IndexReader reader, Similarity similarity, Weight w, byte[] norms):base(similarity)
			{
				InitBlock(enclosingInstance);
				this.termDocs = reader.TermDocs(null);
				score = w.Value;
				this.norms = norms;
			}
				/// <summary>
				/// Returns a list of fields to search on
				/// </summary>
				/// <returns></returns>
				protected static string[] GetSearchFields(IndexReader reader)
				{
						//exclude the special index fields
						return reader.GetFieldNames(IndexReader.FieldOption.ALL)
								.Where(x => !x.StartsWith(LuceneIndexer.SpecialFieldPrefix))
								.ToArray();
				}
Esempio n. 56
0
            protected internal override float[] CreateValue(IndexReader reader, Entry entryKey, IState state)
            {
                Entry entry = entryKey;

                System.String field  = entry.field;
                FloatParser   parser = (FloatParser)entry.custom;

                if (parser == null)
                {
                    try
                    {
                        return(wrapper.GetFloats(reader, field, Lucene.Net.Search.FieldCache_Fields.DEFAULT_FLOAT_PARSER, state));
                    }
                    catch (System.FormatException)
                    {
                        return(wrapper.GetFloats(reader, field, Lucene.Net.Search.FieldCache_Fields.NUMERIC_UTILS_FLOAT_PARSER, state));
                    }
                }
                float[]  retArray = null;
                TermDocs termDocs = reader.TermDocs(state);
                TermEnum termEnum = reader.Terms(new Term(field), state);

                try
                {
                    do
                    {
                        Term term = termEnum.Term;
                        if (term == null || (System.Object)term.Field != (System.Object)field)
                        {
                            break;
                        }
                        float termval = parser.ParseFloat(term.Text);
                        if (retArray == null)
                        {
                            // late init
                            retArray = new float[reader.MaxDoc];
                        }
                        termDocs.Seek(termEnum, state);
                        while (termDocs.Next(state))
                        {
                            retArray[termDocs.Doc] = termval;
                        }
                    }while (termEnum.Next(state));
                }
                catch (StopFillCacheException)
                {
                }
                finally
                {
                    termDocs.Close();
                    termEnum.Close();
                }
                if (retArray == null)
                {
                    // no values
                    retArray = new float[reader.MaxDoc];
                }
                return(retArray);
            }
Esempio n. 57
0
        //function to initialize the spell checker functionality
        public void SpellCheckerInit()
        {
            spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(spellCheckIndexStorage);

            // To index a field of a user index:
            indexReader = writer.GetReader();
            spellchecker.IndexDictionary(new LuceneDictionary(indexReader, WORD_FN));
        }
Esempio n. 58
0
        public virtual void  TestPhrasePrefix()
        {
            RAMDirectory indexStore = new RAMDirectory();
            IndexWriter  writer     = new IndexWriter(indexStore, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);
            Document     doc1       = new Document();
            Document     doc2       = new Document();
            Document     doc3       = new Document();
            Document     doc4       = new Document();
            Document     doc5       = new Document();

            doc1.Add(new Field("body", "blueberry pie", Field.Store.YES, Field.Index.ANALYZED));
            doc2.Add(new Field("body", "blueberry strudel", Field.Store.YES, Field.Index.ANALYZED));
            doc3.Add(new Field("body", "blueberry pizza", Field.Store.YES, Field.Index.ANALYZED));
            doc4.Add(new Field("body", "blueberry chewing gum", Field.Store.YES, Field.Index.ANALYZED));
            doc5.Add(new Field("body", "piccadilly circus", Field.Store.YES, Field.Index.ANALYZED));
            writer.AddDocument(doc1, null);
            writer.AddDocument(doc2, null);
            writer.AddDocument(doc3, null);
            writer.AddDocument(doc4, null);
            writer.AddDocument(doc5, null);
            writer.Optimize(null);
            writer.Close();

            IndexSearcher searcher = new IndexSearcher(indexStore, true, null);

            //PhrasePrefixQuery query1 = new PhrasePrefixQuery();
            MultiPhraseQuery query1 = new MultiPhraseQuery();
            //PhrasePrefixQuery query2 = new PhrasePrefixQuery();
            MultiPhraseQuery query2 = new MultiPhraseQuery();

            query1.Add(new Term("body", "blueberry"));
            query2.Add(new Term("body", "strawberry"));

            System.Collections.ArrayList termsWithPrefix = new System.Collections.ArrayList();
            IndexReader ir = IndexReader.Open((Directory)indexStore, true, null);

            // this TermEnum gives "piccadilly", "pie" and "pizza".
            System.String prefix = "pi";
            TermEnum      te     = ir.Terms(new Term("body", prefix + "*"), null);

            do
            {
                if (te.Term.Text.StartsWith(prefix))
                {
                    termsWithPrefix.Add(te.Term);
                }
            }while (te.Next(null));

            query1.Add((Term[])termsWithPrefix.ToArray(typeof(Term)));
            query2.Add((Term[])termsWithPrefix.ToArray(typeof(Term)));

            ScoreDoc[] result;
            result = searcher.Search(query1, null, 1000, null).ScoreDocs;
            Assert.AreEqual(2, result.Length);

            result = searcher.Search(query2, null, 1000, null).ScoreDocs;
            Assert.AreEqual(0, result.Length);
        }
Esempio n. 59
0
        private void  SearchWithFilter(IndexReader reader, Weight weight, Filter filter, Collector collector, IState state)
        {
            System.Diagnostics.Debug.Assert(filter != null);

            Scorer scorer = weight.Scorer(reader, true, false, state);

            if (scorer == null)
            {
                return;
            }

            int docID = scorer.DocID();

            System.Diagnostics.Debug.Assert(docID == -1 || docID == DocIdSetIterator.NO_MORE_DOCS);

            // CHECKME: use ConjunctionScorer here?
            DocIdSet filterDocIdSet = filter.GetDocIdSet(reader, state);

            if (filterDocIdSet == null)
            {
                // this means the filter does not accept any documents.
                return;
            }

            DocIdSetIterator filterIter = filterDocIdSet.Iterator(state);

            if (filterIter == null)
            {
                // this means the filter does not accept any documents.
                return;
            }
            int filterDoc = filterIter.NextDoc(state);
            int scorerDoc = scorer.Advance(filterDoc, state);

            collector.SetScorer(scorer);
            while (true)
            {
                if (scorerDoc == filterDoc)
                {
                    // Check if scorer has exhausted, only before collecting.
                    if (scorerDoc == DocIdSetIterator.NO_MORE_DOCS)
                    {
                        break;
                    }
                    collector.Collect(scorerDoc, state);
                    filterDoc = filterIter.NextDoc(state);
                    scorerDoc = scorer.Advance(filterDoc, state);
                }
                else if (scorerDoc > filterDoc)
                {
                    filterDoc = filterIter.Advance(scorerDoc, state);
                }
                else
                {
                    scorerDoc = scorer.Advance(filterDoc, state);
                }
            }
        }
        public virtual void  TestBoost()
        {
            // NOTE: uses index build in *this* setUp

            IndexReader   reader = IndexReader.Open(small, true, null);
            IndexSearcher search = new IndexSearcher(reader);

            // test for correct application of query normalization
            // must use a non score normalizing method for this.
            Query q = Csrq("data", "1", "6", T, T);

            q.Boost = 100;
            search.Search(q, null, new AnonymousClassCollector(this), null);

            //
            // Ensure that boosting works to score one clause of a query higher
            // than another.
            //
            Query q1 = Csrq("data", "A", "A", T, T);             // matches document #0

            q1.Boost = .1f;
            Query        q2 = Csrq("data", "Z", "Z", T, T);      // matches document #1
            BooleanQuery bq = new BooleanQuery(true);

            bq.Add(q1, Occur.SHOULD);
            bq.Add(q2, Occur.SHOULD);

            ScoreDoc[] hits = search.Search(bq, null, 1000, null).ScoreDocs;
            Assert.AreEqual(1, hits[0].Doc);
            Assert.AreEqual(0, hits[1].Doc);
            Assert.IsTrue(hits[0].Score > hits[1].Score);

            q1       = Csrq("data", "A", "A", T, T, MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);       // matches document #0
            q1.Boost = .1f;
            q2       = Csrq("data", "Z", "Z", T, T, MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);       // matches document #1
            bq       = new BooleanQuery(true);
            bq.Add(q1, Occur.SHOULD);
            bq.Add(q2, Occur.SHOULD);

            hits = search.Search(bq, null, 1000, null).ScoreDocs;
            Assert.AreEqual(1, hits[0].Doc);
            Assert.AreEqual(0, hits[1].Doc);
            Assert.IsTrue(hits[0].Score > hits[1].Score);

            q1       = Csrq("data", "A", "A", T, T);       // matches document #0
            q1.Boost = 10f;
            q2       = Csrq("data", "Z", "Z", T, T);       // matches document #1
            bq       = new BooleanQuery(true);
            bq.Add(q1, Occur.SHOULD);
            bq.Add(q2, Occur.SHOULD);

            hits = search.Search(bq, null, 1000, null).ScoreDocs;
            Assert.AreEqual(0, hits[0].Doc);
            Assert.AreEqual(1, hits[1].Doc);
            Assert.IsTrue(hits[0].Score > hits[1].Score);
        }