public override FunctionValues GetValues(IDictionary context, AtomicReaderContext readerContext) { var searcher = (IndexSearcher)context["searcher"]; TFIDFSimilarity similarity = IDFValueSource.AsTFIDF(searcher.Similarity, m_field); if (similarity == null) { throw new NotSupportedException("requires a TFIDFSimilarity (such as DefaultSimilarity)"); } NumericDocValues norms = readerContext.AtomicReader.GetNormValues(m_field); if (norms == null) { return(new ConstDoubleDocValues(0.0, this)); } return(new SingleDocValuesAnonymousInnerClassHelper(this, this, similarity, norms)); }
public override void BeforeClass() // LUCENENET specific - renamed from BeforeClassSortingAtomicReaderTest() to ensure calling order vs base class { base.BeforeClass(); // sort the index by id (as integer, in NUMERIC_DV_FIELD) Sort sort = new Sort(new SortField(NUMERIC_DV_FIELD, SortFieldType.INT32)); Sorter.DocMap docMap = new Sorter(sort).Sort(reader); // Sorter.compute also sorts the values NumericDocValues dv = reader.GetNumericDocValues(NUMERIC_DV_FIELD); sortedValues = new int[reader.MaxDoc]; for (int i = 0; i < reader.MaxDoc; ++i) { sortedValues[docMap.OldToNew(i)] = (int)dv.Get(i); } if (VERBOSE) { Console.WriteLine("docMap: " + docMap); Console.WriteLine("sortedValues: " + Arrays.ToString(sortedValues)); } // sort the index by id (as integer, in NUMERIC_DV_FIELD) reader = SortingAtomicReader.Wrap(reader, sort); if (VERBOSE) { Console.WriteLine("mapped-deleted-docs: "); IBits mappedLiveDocs = reader.LiveDocs; for (int i = 0; i < mappedLiveDocs.Length; i++) { if (!mappedLiveDocs.Get(i)) { Console.WriteLine(i + " "); } } Console.WriteLine(); } TestUtil.CheckReader(reader); }
public static void BeforeClassSortingAtomicReaderTest() { // sort the index by id (as integer, in NUMERIC_DV_FIELD) Sort sort = new Sort(new SortField(NUMERIC_DV_FIELD, SortField.Type_e.INT)); Sorter.DocMap docMap = new Sorter(sort).Sort(reader); // Sorter.compute also sorts the values NumericDocValues dv = reader.GetNumericDocValues(NUMERIC_DV_FIELD); sortedValues = new int[reader.MaxDoc]; for (int i = 0; i < reader.MaxDoc; ++i) { sortedValues[docMap.OldToNew(i)] = (int)dv.Get(i); } if (VERBOSE) { Console.WriteLine("docMap: " + docMap); Console.WriteLine("sortedValues: " + Arrays.ToString(sortedValues)); } // sort the index by id (as integer, in NUMERIC_DV_FIELD) reader = SortingAtomicReader.Wrap(reader, sort); if (VERBOSE) { Console.WriteLine("mapped-deleted-docs: "); Bits mappedLiveDocs = reader.LiveDocs; for (int i = 0; i < mappedLiveDocs.Length(); i++) { if (!mappedLiveDocs.Get(i)) { Console.WriteLine(i + " "); } } Console.WriteLine(); } TestUtil.CheckReader(reader); }
public virtual void TestBasics() { // sanity check of norms writer // TODO: generalize AtomicReader slow = SlowCompositeReaderWrapper.Wrap(reader); NumericDocValues fooNorms = slow.GetNormValues("foo"); NumericDocValues barNorms = slow.GetNormValues("bar"); for (int i = 0; i < slow.MaxDoc; i++) { Assert.IsFalse(fooNorms.Get(i) == barNorms.Get(i)); } // sanity check of searching TopDocs foodocs = searcher.Search(new TermQuery(new Term("foo", "brown")), 10); Assert.IsTrue(foodocs.TotalHits > 0); TopDocs bardocs = searcher.Search(new TermQuery(new Term("bar", "brown")), 10); Assert.IsTrue(bardocs.TotalHits > 0); Assert.IsTrue(foodocs.ScoreDocs[0].Score < bardocs.ScoreDocs[0].Score); }
public override long GetValue(string field) { if (EnsureValidDisi()) { if (DocValuesCache.ContainsKey(field)) { return(DocValuesCache[field].get(CurrentIdIterator.docID())); } NumericDocValues docValues = CurrentDocs.readDocValues(field); DocValuesCache[field] = docValues; return(docValues.get(CurrentIdIterator.docID())); } else { // same as DocValues.emptyNumeric()#get // which means, getValue carries over the semantics of NDV // -1 would also be a possibility here. return(0); } }
/// <returns> the {@code NumericDocValues} for a given field </returns> /// <exception cref="IllegalArgumentException"> if this field is not indexed with numeric doc values </exception> public NumericDocValues ReadDocValues(string field) { try { NumericDocValues dv = Context.reader().getNumericDocValues(field); if (dv == null) { FieldInfo fi = Context.reader().FieldInfos.fieldInfo(field); DocValuesType actual = null; if (fi != null) { actual = fi.DocValuesType; } throw new System.InvalidOperationException("The field '" + field + "' is not indexed properly, expected NumericDV, but got '" + actual + "'"); } return(dv); } catch (IOException e) { throw new Exception(e); } }
private IEnumerable <long?> GetMergeNumericFieldEnumerable(FieldInfo fieldinfo, MergeState mergeState, IList <NumericDocValues> toMerge) { int readerUpto = -1; int docIDUpto = 0; AtomicReader currentReader = null; NumericDocValues currentValues = null; Bits currentLiveDocs = null; while (true) { if (readerUpto == toMerge.Count) { yield break; } if (currentReader == null || docIDUpto == currentReader.MaxDoc) { readerUpto++; if (readerUpto < toMerge.Count) { currentReader = mergeState.Readers[readerUpto]; currentValues = toMerge[readerUpto]; currentLiveDocs = currentReader.LiveDocs; } docIDUpto = 0; continue; } if (currentLiveDocs == null || currentLiveDocs.Get(docIDUpto)) { yield return(currentValues.Get(docIDUpto++)); continue; } docIDUpto++; } }
private Explanation ExplainScore(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms) { Explanation result = new Explanation(); result.Description = "score(doc=" + doc + ",freq=" + freq + "), product of:"; Explanation boostExpl = new Explanation(stats.QueryBoost * stats.TopLevelBoost, "boost"); if (boostExpl.Value != 1.0f) { result.AddDetail(boostExpl); } result.AddDetail(stats.Idf); Explanation tfNormExpl = new Explanation(); tfNormExpl.Description = "tfNorm, computed from:"; tfNormExpl.AddDetail(freq); tfNormExpl.AddDetail(new Explanation(k1, "parameter k1")); if (norms == null) { tfNormExpl.AddDetail(new Explanation(0, "parameter b (norms omitted for field)")); tfNormExpl.Value = (freq.Value * (k1 + 1)) / (freq.Value + k1); } else { float doclen = DecodeNormValue((byte)norms.Get(doc)); tfNormExpl.AddDetail(new Explanation(b, "parameter b")); tfNormExpl.AddDetail(new Explanation(stats.Avgdl, "avgFieldLength")); tfNormExpl.AddDetail(new Explanation(doclen, "fieldLength")); tfNormExpl.Value = (freq.Value * (k1 + 1)) / (freq.Value + k1 * (1 - b + b * doclen / stats.Avgdl)); } result.AddDetail(tfNormExpl); result.Value = boostExpl.Value * stats.Idf.Value * tfNormExpl.Value; return(result); }
public RandomAccessOrdsAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, SortedSetEntry entry, NumericDocValues docToOrdAddress, NumericDocValues ords, BinaryDocValues values) { this.entry = entry; this.docToOrdAddress = docToOrdAddress; this.ords = ords; this.values = values; }
public SortedDocValuesAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, SortedEntry entry, NumericDocValues docToOrd, BinaryDocValues values) { this.outerInstance = outerInstance; this.entry = entry; this.docToOrd = docToOrd; this.values = values; }
public SingleDocValuesAnonymousClass(NormValueSource @this, TFIDFSimilarity similarity, NumericDocValues norms) : base(@this) { this.similarity = similarity; this.norms = norms; }
public RandomAccessOrdsAnonymousClass(SortedSetEntry entry, NumericDocValues docToOrdAddress, NumericDocValues ords, BinaryDocValues values) { this.entry = entry; this.docToOrdAddress = docToOrdAddress; this.ords = ords; this.values = values; }
public SortedDocValuesAnonymousClass(SortedEntry entry, NumericDocValues docToOrd, BinaryDocValues values) { this.entry = entry; this.docToOrd = docToOrd; this.values = values; }
public SortedDocValuesAnonymousInnerClassHelper(Lucene42DocValuesProducer outerInstance, FSTEntry entry, NumericDocValues docToOrd, FST <long> fst, FST <long> .BytesReader @in, FST <long> .Arc <long> firstArc, FST <long> .Arc <long> scratchArc, IntsRef scratchInts, BytesRefFSTEnum <long> fstEnum) { this.OuterInstance = outerInstance; this.Entry = entry; this.DocToOrd = docToOrd; this.Fst = fst; this.@in = @in; this.FirstArc = firstArc; this.ScratchArc = scratchArc; this.ScratchInts = scratchInts; this.FstEnum = fstEnum; }
public SortedDocValuesAnonymousInnerClassHelper(FSTEntry entry, NumericDocValues docToOrd, FST<long?> fst, FST<long?>.BytesReader @in, FST<long?>.Arc<long?> firstArc, FST<long?>.Arc<long?> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<long?> fstEnum) { this.Entry = entry; this.DocToOrd = docToOrd; this.Fst = fst; this.@in = @in; this.FirstArc = firstArc; this.ScratchArc = scratchArc; this.ScratchInts = scratchInts; this.FstEnum = fstEnum; }
private void DuellReaders(CompositeReader other, AtomicReader memIndexReader) { AtomicReader competitor = SlowCompositeReaderWrapper.Wrap(other); Fields memFields = memIndexReader.Fields; foreach (string field in competitor.Fields) { Terms memTerms = memFields.GetTerms(field); Terms iwTerms = memIndexReader.GetTerms(field); if (iwTerms is null) { assertNull(memTerms); } else { NumericDocValues normValues = competitor.GetNormValues(field); NumericDocValues memNormValues = memIndexReader.GetNormValues(field); if (normValues != null) { // mem idx always computes norms on the fly assertNotNull(memNormValues); assertEquals(normValues.Get(0), memNormValues.Get(0)); } assertNotNull(memTerms); assertEquals(iwTerms.DocCount, memTerms.DocCount); assertEquals(iwTerms.SumDocFreq, memTerms.SumDocFreq); assertEquals(iwTerms.SumTotalTermFreq, memTerms.SumTotalTermFreq); TermsEnum iwTermsIter = iwTerms.GetEnumerator(); TermsEnum memTermsIter = memTerms.GetEnumerator(); if (iwTerms.HasPositions) { bool offsets = iwTerms.HasOffsets && memTerms.HasOffsets; while (iwTermsIter.MoveNext()) { assertTrue(memTermsIter.MoveNext()); assertEquals(iwTermsIter.Term, memTermsIter.Term); DocsAndPositionsEnum iwDocsAndPos = iwTermsIter.DocsAndPositions(null, null); DocsAndPositionsEnum memDocsAndPos = memTermsIter.DocsAndPositions(null, null); while (iwDocsAndPos.NextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS) { assertEquals(iwDocsAndPos.DocID, memDocsAndPos.NextDoc()); assertEquals(iwDocsAndPos.Freq, memDocsAndPos.Freq); for (int i = 0; i < iwDocsAndPos.Freq; i++) { assertEquals("term: " + iwTermsIter.Term.Utf8ToString(), iwDocsAndPos.NextPosition(), memDocsAndPos.NextPosition()); if (offsets) { assertEquals(iwDocsAndPos.StartOffset, memDocsAndPos.StartOffset); assertEquals(iwDocsAndPos.EndOffset, memDocsAndPos.EndOffset); } } } } } else { while (iwTermsIter.MoveNext()) { assertEquals(iwTermsIter.Term, memTermsIter.Term); DocsEnum iwDocsAndPos = iwTermsIter.Docs(null, null); DocsEnum memDocsAndPos = memTermsIter.Docs(null, null); while (iwDocsAndPos.NextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS) { assertEquals(iwDocsAndPos.DocID, memDocsAndPos.NextDoc()); assertEquals(iwDocsAndPos.Freq, memDocsAndPos.Freq); } } } } } }
public SortedDocValuesAnonymousInnerClassHelper(Lucene42DocValuesProducer outerInstance, FSTEntry entry, NumericDocValues docToOrd, FST<long> fst, FST<long>.BytesReader @in, FST<long>.Arc<long> firstArc, FST<long>.Arc<long> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<long> fstEnum) { this.OuterInstance = outerInstance; this.Entry = entry; this.DocToOrd = docToOrd; this.Fst = fst; this.@in = @in; this.FirstArc = firstArc; this.ScratchArc = scratchArc; this.ScratchInts = scratchInts; this.FstEnum = fstEnum; }
private Explanation ExplainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) { Explanation result = new Explanation(); // LUCENENET specific - using freq.Value is a change that was made in Lucene 5.0, but is included // in 4.8.0 to remove annoying newlines from the output. // See: https://github.com/apache/lucene-solr/commit/f0bfcbc7d8fbc5bb2791da60af559e8b0ad6eed6 result.Description = "score(doc=" + doc + ",freq=" + freq.Value + "), product of:"; // explain query weight Explanation queryExpl = new Explanation(); queryExpl.Description = "queryWeight, product of:"; Explanation boostExpl = new Explanation(stats.QueryBoost, "boost"); if (stats.QueryBoost != 1.0f) { queryExpl.AddDetail(boostExpl); } queryExpl.AddDetail(stats.Idf); Explanation queryNormExpl = new Explanation(stats.QueryNorm, "queryNorm"); queryExpl.AddDetail(queryNormExpl); queryExpl.Value = boostExpl.Value * stats.Idf.Value * queryNormExpl.Value; result.AddDetail(queryExpl); // explain field weight Explanation fieldExpl = new Explanation(); fieldExpl.Description = "fieldWeight in " + doc + ", product of:"; Explanation tfExplanation = new Explanation(); tfExplanation.Value = Tf(freq.Value); tfExplanation.Description = "tf(freq=" + freq.Value + "), with freq of:"; tfExplanation.AddDetail(freq); fieldExpl.AddDetail(tfExplanation); fieldExpl.AddDetail(stats.Idf); Explanation fieldNormExpl = new Explanation(); float fieldNorm = norms != null?DecodeNormValue(norms.Get(doc)) : 1.0f; fieldNormExpl.Value = fieldNorm; fieldNormExpl.Description = "fieldNorm(doc=" + doc + ")"; fieldExpl.AddDetail(fieldNormExpl); fieldExpl.Value = tfExplanation.Value * stats.Idf.Value * fieldNormExpl.Value; result.AddDetail(fieldExpl); // combine them result.Value = queryExpl.Value * fieldExpl.Value; if (queryExpl.Value == 1.0f) { return(fieldExpl); } return(result); }
//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET: //ORIGINAL LINE: public SortingNumericDocValues(final org.apache.lucene.index.NumericDocValues in, Sorter.DocMap docMap) public SortingNumericDocValues(NumericDocValues @in, Sorter.DocMap docMap) { this.@in = @in; this.docMap = docMap; }
public override long GetValue(string field) { NumericDocValues fieldValues = ReaderCache.computeIfAbsent(field, this.getDocValues); return(fieldValues.get(CurrentDocID)); }
public SortingNumericDocValues(NumericDocValues input, Sorter.DocMap docMap) { this.@in = input; this.docMap = docMap; }
public void Test() { RandomIndexWriter writer; DirectoryReader indexReader; int numParents = AtLeast(200); IndexWriterConfig cfg = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); cfg.SetMergePolicy(NewLogMergePolicy()); using (writer = new RandomIndexWriter(Random(), NewDirectory(), cfg)) { Document parentDoc = new Document(); NumericDocValuesField parentVal = new NumericDocValuesField("parent_val", 0L); parentDoc.Add(parentVal); StringField parent = new StringField("parent", "true", Field.Store.YES); parentDoc.Add(parent); for (int i = 0; i < numParents; ++i) { List <Document> documents = new List <Document>(); int numChildren = Random().nextInt(10); for (int j = 0; j < numChildren; ++j) { Document childDoc = new Document(); childDoc.Add(new NumericDocValuesField("child_val", Random().nextInt(5))); documents.Add(childDoc); } parentVal.SetInt64Value(Random().nextInt(50)); documents.Add(parentDoc); writer.AddDocuments(documents); } writer.ForceMerge(1); indexReader = writer.Reader; } AtomicReader reader = GetOnlySegmentReader(indexReader); Filter parentsFilter = new FixedBitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("parent", "true")))); FixedBitSet parentBits = (FixedBitSet)parentsFilter.GetDocIdSet(reader.AtomicContext, null); NumericDocValues parentValues = reader.GetNumericDocValues("parent_val"); NumericDocValues childValues = reader.GetNumericDocValues("child_val"); Sort parentSort = new Sort(new SortField("parent_val", SortFieldType.INT64)); Sort childSort = new Sort(new SortField("child_val", SortFieldType.INT64)); Sort sort = new Sort(new SortField("custom", new BlockJoinComparerSource(parentsFilter, parentSort, childSort))); Sorter sorter = new Sorter(sort); Sorter.DocMap docMap = sorter.Sort(reader); assertEquals(reader.MaxDoc, docMap.Count); int[] children = new int[1]; int numChildren2 = 0; int previousParent = -1; for (int i = 0; i < docMap.Count; ++i) { int oldID = docMap.NewToOld(i); if (parentBits.Get(oldID)) { // check that we have the right children for (int j = 0; j < numChildren2; ++j) { assertEquals(oldID, parentBits.NextSetBit(children[j])); } // check that children are sorted for (int j = 1; j < numChildren2; ++j) { int doc1 = children[j - 1]; int doc2 = children[j]; if (childValues.Get(doc1) == childValues.Get(doc2)) { assertTrue(doc1 < doc2); // sort is stable } else { assertTrue(childValues.Get(doc1) < childValues.Get(doc2)); } } // check that parents are sorted if (previousParent != -1) { if (parentValues.Get(previousParent) == parentValues.Get(oldID)) { assertTrue(previousParent < oldID); } else { assertTrue(parentValues.Get(previousParent) < parentValues.Get(oldID)); } } // reset previousParent = oldID; numChildren2 = 0; } else { children = ArrayUtil.Grow(children, numChildren2 + 1); children[numChildren2++] = oldID; } } indexReader.Dispose(); writer.w.Directory.Dispose(); }
private Explanation ExplainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) { Explanation result = new Explanation(); result.Description = "score(doc=" + doc + ",freq=" + freq + "), product of:"; // explain query weight Explanation queryExpl = new Explanation(); queryExpl.Description = "queryWeight, product of:"; Explanation boostExpl = new Explanation(stats.QueryBoost, "boost"); if (stats.QueryBoost != 1.0f) { queryExpl.AddDetail(boostExpl); } queryExpl.AddDetail(stats.Idf); Explanation queryNormExpl = new Explanation(stats.QueryNorm, "queryNorm"); queryExpl.AddDetail(queryNormExpl); queryExpl.Value = boostExpl.Value * stats.Idf.Value * queryNormExpl.Value; result.AddDetail(queryExpl); // explain field weight Explanation fieldExpl = new Explanation(); fieldExpl.Description = "fieldWeight in " + doc + ", product of:"; Explanation tfExplanation = new Explanation(); tfExplanation.Value = Tf(freq.Value); tfExplanation.Description = "tf(freq=" + freq.Value + "), with freq of:"; tfExplanation.AddDetail(freq); fieldExpl.AddDetail(tfExplanation); fieldExpl.AddDetail(stats.Idf); Explanation fieldNormExpl = new Explanation(); float fieldNorm = norms != null?DecodeNormValue(norms.Get(doc)) : 1.0f; fieldNormExpl.Value = fieldNorm; fieldNormExpl.Description = "fieldNorm(doc=" + doc + ")"; fieldExpl.AddDetail(fieldNormExpl); fieldExpl.Value = tfExplanation.Value * stats.Idf.Value * fieldNormExpl.Value; result.AddDetail(fieldExpl); // combine them result.Value = queryExpl.Value * fieldExpl.Value; if (queryExpl.Value == 1.0f) { return(fieldExpl); } return(result); }
public SingleDocValuesAnonymousInnerClassHelper(NormValueSource outerInstance, NormValueSource @this, TFIDFSimilarity similarity, NumericDocValues norms) : base(@this) { this.outerInstance = outerInstance; this.similarity = similarity; this.norms = norms; }
//JAVA TO C# CONVERTER WARNING: 'final' parameters are ignored unless the option to convert to C# 7.2 'in' parameters is selected: //ORIGINAL LINE: public IndexReaderStub(final org.apache.lucene.index.NumericDocValues ndv) public IndexReaderStub(NumericDocValues ndv) { this._ndvs = s => ndv; }