public override FunctionValues GetValues(IDictionary context, AtomicReaderContext readerContext)
{
var searcher = (IndexSearcher)context["searcher"];
TFIDFSimilarity similarity = IDFValueSource.AsTFIDF(searcher.Similarity, m_field);
if (similarity == null)
{
throw new NotSupportedException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
}
NumericDocValues norms = readerContext.AtomicReader.GetNormValues(m_field);
if (norms == null)
{
return(new ConstDoubleDocValues(0.0, this));
}
return(new SingleDocValuesAnonymousInnerClassHelper(this, this, similarity, norms));
}
public override void BeforeClass() // LUCENENET specific - renamed from BeforeClassSortingAtomicReaderTest() to ensure calling order vs base class
{
base.BeforeClass();
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
Sort sort = new Sort(new SortField(NUMERIC_DV_FIELD, SortFieldType.INT32));
Sorter.DocMap docMap = new Sorter(sort).Sort(reader);
// Sorter.compute also sorts the values
NumericDocValues dv = reader.GetNumericDocValues(NUMERIC_DV_FIELD);
sortedValues = new int[reader.MaxDoc];
for (int i = 0; i < reader.MaxDoc; ++i)
{
sortedValues[docMap.OldToNew(i)] = (int)dv.Get(i);
}
if (VERBOSE)
{
Console.WriteLine("docMap: " + docMap);
Console.WriteLine("sortedValues: " + Arrays.ToString(sortedValues));
}
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
reader = SortingAtomicReader.Wrap(reader, sort);
if (VERBOSE)
{
Console.WriteLine("mapped-deleted-docs: ");
IBits mappedLiveDocs = reader.LiveDocs;
for (int i = 0; i < mappedLiveDocs.Length; i++)
{
if (!mappedLiveDocs.Get(i))
{
Console.WriteLine(i + " ");
}
}
Console.WriteLine();
}
TestUtil.CheckReader(reader);
}
public static void BeforeClassSortingAtomicReaderTest()
{
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
Sort sort = new Sort(new SortField(NUMERIC_DV_FIELD, SortField.Type_e.INT));
Sorter.DocMap docMap = new Sorter(sort).Sort(reader);
// Sorter.compute also sorts the values
NumericDocValues dv = reader.GetNumericDocValues(NUMERIC_DV_FIELD);
sortedValues = new int[reader.MaxDoc];
for (int i = 0; i < reader.MaxDoc; ++i)
{
sortedValues[docMap.OldToNew(i)] = (int)dv.Get(i);
}
if (VERBOSE)
{
Console.WriteLine("docMap: " + docMap);
Console.WriteLine("sortedValues: " + Arrays.ToString(sortedValues));
}
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
reader = SortingAtomicReader.Wrap(reader, sort);
if (VERBOSE)
{
Console.WriteLine("mapped-deleted-docs: ");
Bits mappedLiveDocs = reader.LiveDocs;
for (int i = 0; i < mappedLiveDocs.Length(); i++)
{
if (!mappedLiveDocs.Get(i))
{
Console.WriteLine(i + " ");
}
}
Console.WriteLine();
}
TestUtil.CheckReader(reader);
}
public virtual void TestBasics()
{
// sanity check of norms writer
// TODO: generalize
AtomicReader slow = SlowCompositeReaderWrapper.Wrap(reader);
NumericDocValues fooNorms = slow.GetNormValues("foo");
NumericDocValues barNorms = slow.GetNormValues("bar");
for (int i = 0; i < slow.MaxDoc; i++)
{
Assert.IsFalse(fooNorms.Get(i) == barNorms.Get(i));
}
// sanity check of searching
TopDocs foodocs = searcher.Search(new TermQuery(new Term("foo", "brown")), 10);
Assert.IsTrue(foodocs.TotalHits > 0);
TopDocs bardocs = searcher.Search(new TermQuery(new Term("bar", "brown")), 10);
Assert.IsTrue(bardocs.TotalHits > 0);
Assert.IsTrue(foodocs.ScoreDocs[0].Score < bardocs.ScoreDocs[0].Score);
}
public override long GetValue(string field)
{
if (EnsureValidDisi())
{
if (DocValuesCache.ContainsKey(field))
{
return(DocValuesCache[field].get(CurrentIdIterator.docID()));
}
NumericDocValues docValues = CurrentDocs.readDocValues(field);
DocValuesCache[field] = docValues;
return(docValues.get(CurrentIdIterator.docID()));
}
else
{
// same as DocValues.emptyNumeric()#get
// which means, getValue carries over the semantics of NDV
// -1 would also be a possibility here.
return(0);
}
}
/// <returns> the {@code NumericDocValues} for a given field </returns>
/// <exception cref="IllegalArgumentException"> if this field is not indexed with numeric doc values </exception>
public NumericDocValues ReadDocValues(string field)
{
try
{
NumericDocValues dv = Context.reader().getNumericDocValues(field);
if (dv == null)
{
FieldInfo fi = Context.reader().FieldInfos.fieldInfo(field);
DocValuesType actual = null;
if (fi != null)
{
actual = fi.DocValuesType;
}
throw new System.InvalidOperationException("The field '" + field + "' is not indexed properly, expected NumericDV, but got '" + actual + "'");
}
return(dv);
}
catch (IOException e)
{
throw new Exception(e);
}
}
private IEnumerable <long?> GetMergeNumericFieldEnumerable(FieldInfo fieldinfo, MergeState mergeState, IList <NumericDocValues> toMerge)
{
int readerUpto = -1;
int docIDUpto = 0;
AtomicReader currentReader = null;
NumericDocValues currentValues = null;
Bits currentLiveDocs = null;
while (true)
{
if (readerUpto == toMerge.Count)
{
yield break;
}
if (currentReader == null || docIDUpto == currentReader.MaxDoc)
{
readerUpto++;
if (readerUpto < toMerge.Count)
{
currentReader = mergeState.Readers[readerUpto];
currentValues = toMerge[readerUpto];
currentLiveDocs = currentReader.LiveDocs;
}
docIDUpto = 0;
continue;
}
if (currentLiveDocs == null || currentLiveDocs.Get(docIDUpto))
{
yield return(currentValues.Get(docIDUpto++));
continue;
}
docIDUpto++;
}
}
private Explanation ExplainScore(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms)
{
Explanation result = new Explanation();
result.Description = "score(doc=" + doc + ",freq=" + freq + "), product of:";
Explanation boostExpl = new Explanation(stats.QueryBoost * stats.TopLevelBoost, "boost");
if (boostExpl.Value != 1.0f)
{
result.AddDetail(boostExpl);
}
result.AddDetail(stats.Idf);
Explanation tfNormExpl = new Explanation();
tfNormExpl.Description = "tfNorm, computed from:";
tfNormExpl.AddDetail(freq);
tfNormExpl.AddDetail(new Explanation(k1, "parameter k1"));
if (norms == null)
{
tfNormExpl.AddDetail(new Explanation(0, "parameter b (norms omitted for field)"));
tfNormExpl.Value = (freq.Value * (k1 + 1)) / (freq.Value + k1);
}
else
{
float doclen = DecodeNormValue((byte)norms.Get(doc));
tfNormExpl.AddDetail(new Explanation(b, "parameter b"));
tfNormExpl.AddDetail(new Explanation(stats.Avgdl, "avgFieldLength"));
tfNormExpl.AddDetail(new Explanation(doclen, "fieldLength"));
tfNormExpl.Value = (freq.Value * (k1 + 1)) / (freq.Value + k1 * (1 - b + b * doclen / stats.Avgdl));
}
result.AddDetail(tfNormExpl);
result.Value = boostExpl.Value * stats.Idf.Value * tfNormExpl.Value;
return(result);
}
public RandomAccessOrdsAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, SortedSetEntry entry, NumericDocValues docToOrdAddress, NumericDocValues ords, BinaryDocValues values)
{
this.entry = entry;
this.docToOrdAddress = docToOrdAddress;
this.ords = ords;
this.values = values;
}
public SortedDocValuesAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, SortedEntry entry, NumericDocValues docToOrd, BinaryDocValues values)
{
this.outerInstance = outerInstance;
this.entry = entry;
this.docToOrd = docToOrd;
this.values = values;
}
public SingleDocValuesAnonymousClass(NormValueSource @this, TFIDFSimilarity similarity, NumericDocValues norms)
: base(@this)
{
this.similarity = similarity;
this.norms = norms;
}
public RandomAccessOrdsAnonymousClass(SortedSetEntry entry, NumericDocValues docToOrdAddress, NumericDocValues ords, BinaryDocValues values)
{
this.entry = entry;
this.docToOrdAddress = docToOrdAddress;
this.ords = ords;
this.values = values;
}
public SortedDocValuesAnonymousClass(SortedEntry entry, NumericDocValues docToOrd, BinaryDocValues values)
{
this.entry = entry;
this.docToOrd = docToOrd;
this.values = values;
}
public SortedDocValuesAnonymousInnerClassHelper(Lucene42DocValuesProducer outerInstance, FSTEntry entry, NumericDocValues docToOrd, FST <long> fst, FST <long> .BytesReader @in, FST <long> .Arc <long> firstArc, FST <long> .Arc <long> scratchArc, IntsRef scratchInts, BytesRefFSTEnum <long> fstEnum)
{
this.OuterInstance = outerInstance;
this.Entry = entry;
this.DocToOrd = docToOrd;
this.Fst = fst;
this.@in = @in;
this.FirstArc = firstArc;
this.ScratchArc = scratchArc;
this.ScratchInts = scratchInts;
this.FstEnum = fstEnum;
}
public SortedDocValuesAnonymousInnerClassHelper(FSTEntry entry, NumericDocValues docToOrd, FST<long?> fst, FST<long?>.BytesReader @in, FST<long?>.Arc<long?> firstArc, FST<long?>.Arc<long?> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<long?> fstEnum)
{
this.Entry = entry;
this.DocToOrd = docToOrd;
this.Fst = fst;
this.@in = @in;
this.FirstArc = firstArc;
this.ScratchArc = scratchArc;
this.ScratchInts = scratchInts;
this.FstEnum = fstEnum;
}
private void DuellReaders(CompositeReader other, AtomicReader memIndexReader)
{
AtomicReader competitor = SlowCompositeReaderWrapper.Wrap(other);
Fields memFields = memIndexReader.Fields;
foreach (string field in competitor.Fields)
{
Terms memTerms = memFields.GetTerms(field);
Terms iwTerms = memIndexReader.GetTerms(field);
if (iwTerms is null)
{
assertNull(memTerms);
}
else
{
NumericDocValues normValues = competitor.GetNormValues(field);
NumericDocValues memNormValues = memIndexReader.GetNormValues(field);
if (normValues != null)
{
// mem idx always computes norms on the fly
assertNotNull(memNormValues);
assertEquals(normValues.Get(0), memNormValues.Get(0));
}
assertNotNull(memTerms);
assertEquals(iwTerms.DocCount, memTerms.DocCount);
assertEquals(iwTerms.SumDocFreq, memTerms.SumDocFreq);
assertEquals(iwTerms.SumTotalTermFreq, memTerms.SumTotalTermFreq);
TermsEnum iwTermsIter = iwTerms.GetEnumerator();
TermsEnum memTermsIter = memTerms.GetEnumerator();
if (iwTerms.HasPositions)
{
bool offsets = iwTerms.HasOffsets && memTerms.HasOffsets;
while (iwTermsIter.MoveNext())
{
assertTrue(memTermsIter.MoveNext());
assertEquals(iwTermsIter.Term, memTermsIter.Term);
DocsAndPositionsEnum iwDocsAndPos = iwTermsIter.DocsAndPositions(null, null);
DocsAndPositionsEnum memDocsAndPos = memTermsIter.DocsAndPositions(null, null);
while (iwDocsAndPos.NextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS)
{
assertEquals(iwDocsAndPos.DocID, memDocsAndPos.NextDoc());
assertEquals(iwDocsAndPos.Freq, memDocsAndPos.Freq);
for (int i = 0; i < iwDocsAndPos.Freq; i++)
{
assertEquals("term: " + iwTermsIter.Term.Utf8ToString(), iwDocsAndPos.NextPosition(), memDocsAndPos.NextPosition());
if (offsets)
{
assertEquals(iwDocsAndPos.StartOffset, memDocsAndPos.StartOffset);
assertEquals(iwDocsAndPos.EndOffset, memDocsAndPos.EndOffset);
}
}
}
}
}
else
{
while (iwTermsIter.MoveNext())
{
assertEquals(iwTermsIter.Term, memTermsIter.Term);
DocsEnum iwDocsAndPos = iwTermsIter.Docs(null, null);
DocsEnum memDocsAndPos = memTermsIter.Docs(null, null);
while (iwDocsAndPos.NextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS)
{
assertEquals(iwDocsAndPos.DocID, memDocsAndPos.NextDoc());
assertEquals(iwDocsAndPos.Freq, memDocsAndPos.Freq);
}
}
}
}
}
}
public SortedDocValuesAnonymousInnerClassHelper(Lucene42DocValuesProducer outerInstance, FSTEntry entry, NumericDocValues docToOrd, FST<long> fst, FST<long>.BytesReader @in, FST<long>.Arc<long> firstArc, FST<long>.Arc<long> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<long> fstEnum)
{
this.OuterInstance = outerInstance;
this.Entry = entry;
this.DocToOrd = docToOrd;
this.Fst = fst;
this.@in = @in;
this.FirstArc = firstArc;
this.ScratchArc = scratchArc;
this.ScratchInts = scratchInts;
this.FstEnum = fstEnum;
}
private Explanation ExplainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms)
{
Explanation result = new Explanation();
// LUCENENET specific - using freq.Value is a change that was made in Lucene 5.0, but is included
// in 4.8.0 to remove annoying newlines from the output.
// See: https://github.com/apache/lucene-solr/commit/f0bfcbc7d8fbc5bb2791da60af559e8b0ad6eed6
result.Description = "score(doc=" + doc + ",freq=" + freq.Value + "), product of:";
// explain query weight
Explanation queryExpl = new Explanation();
queryExpl.Description = "queryWeight, product of:";
Explanation boostExpl = new Explanation(stats.QueryBoost, "boost");
if (stats.QueryBoost != 1.0f)
{
queryExpl.AddDetail(boostExpl);
}
queryExpl.AddDetail(stats.Idf);
Explanation queryNormExpl = new Explanation(stats.QueryNorm, "queryNorm");
queryExpl.AddDetail(queryNormExpl);
queryExpl.Value = boostExpl.Value * stats.Idf.Value * queryNormExpl.Value;
result.AddDetail(queryExpl);
// explain field weight
Explanation fieldExpl = new Explanation();
fieldExpl.Description = "fieldWeight in " + doc + ", product of:";
Explanation tfExplanation = new Explanation();
tfExplanation.Value = Tf(freq.Value);
tfExplanation.Description = "tf(freq=" + freq.Value + "), with freq of:";
tfExplanation.AddDetail(freq);
fieldExpl.AddDetail(tfExplanation);
fieldExpl.AddDetail(stats.Idf);
Explanation fieldNormExpl = new Explanation();
float fieldNorm = norms != null?DecodeNormValue(norms.Get(doc)) : 1.0f;
fieldNormExpl.Value = fieldNorm;
fieldNormExpl.Description = "fieldNorm(doc=" + doc + ")";
fieldExpl.AddDetail(fieldNormExpl);
fieldExpl.Value = tfExplanation.Value * stats.Idf.Value * fieldNormExpl.Value;
result.AddDetail(fieldExpl);
// combine them
result.Value = queryExpl.Value * fieldExpl.Value;
if (queryExpl.Value == 1.0f)
{
return(fieldExpl);
}
return(result);
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
//ORIGINAL LINE: public SortingNumericDocValues(final org.apache.lucene.index.NumericDocValues in, Sorter.DocMap docMap)
public SortingNumericDocValues(NumericDocValues @in, Sorter.DocMap docMap)
{
this.@in = @in;
this.docMap = docMap;
}
public override long GetValue(string field)
{
NumericDocValues fieldValues = ReaderCache.computeIfAbsent(field, this.getDocValues);
return(fieldValues.get(CurrentDocID));
}
public SortingNumericDocValues(NumericDocValues input, Sorter.DocMap docMap)
{
this.@in = input;
this.docMap = docMap;
}
public void Test()
{
RandomIndexWriter writer;
DirectoryReader indexReader;
int numParents = AtLeast(200);
IndexWriterConfig cfg = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
cfg.SetMergePolicy(NewLogMergePolicy());
using (writer = new RandomIndexWriter(Random(), NewDirectory(), cfg))
{
Document parentDoc = new Document();
NumericDocValuesField parentVal = new NumericDocValuesField("parent_val", 0L);
parentDoc.Add(parentVal);
StringField parent = new StringField("parent", "true", Field.Store.YES);
parentDoc.Add(parent);
for (int i = 0; i < numParents; ++i)
{
List <Document> documents = new List <Document>();
int numChildren = Random().nextInt(10);
for (int j = 0; j < numChildren; ++j)
{
Document childDoc = new Document();
childDoc.Add(new NumericDocValuesField("child_val", Random().nextInt(5)));
documents.Add(childDoc);
}
parentVal.SetInt64Value(Random().nextInt(50));
documents.Add(parentDoc);
writer.AddDocuments(documents);
}
writer.ForceMerge(1);
indexReader = writer.Reader;
}
AtomicReader reader = GetOnlySegmentReader(indexReader);
Filter parentsFilter = new FixedBitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("parent", "true"))));
FixedBitSet parentBits = (FixedBitSet)parentsFilter.GetDocIdSet(reader.AtomicContext, null);
NumericDocValues parentValues = reader.GetNumericDocValues("parent_val");
NumericDocValues childValues = reader.GetNumericDocValues("child_val");
Sort parentSort = new Sort(new SortField("parent_val", SortFieldType.INT64));
Sort childSort = new Sort(new SortField("child_val", SortFieldType.INT64));
Sort sort = new Sort(new SortField("custom", new BlockJoinComparerSource(parentsFilter, parentSort, childSort)));
Sorter sorter = new Sorter(sort);
Sorter.DocMap docMap = sorter.Sort(reader);
assertEquals(reader.MaxDoc, docMap.Count);
int[] children = new int[1];
int numChildren2 = 0;
int previousParent = -1;
for (int i = 0; i < docMap.Count; ++i)
{
int oldID = docMap.NewToOld(i);
if (parentBits.Get(oldID))
{
// check that we have the right children
for (int j = 0; j < numChildren2; ++j)
{
assertEquals(oldID, parentBits.NextSetBit(children[j]));
}
// check that children are sorted
for (int j = 1; j < numChildren2; ++j)
{
int doc1 = children[j - 1];
int doc2 = children[j];
if (childValues.Get(doc1) == childValues.Get(doc2))
{
assertTrue(doc1 < doc2); // sort is stable
}
else
{
assertTrue(childValues.Get(doc1) < childValues.Get(doc2));
}
}
// check that parents are sorted
if (previousParent != -1)
{
if (parentValues.Get(previousParent) == parentValues.Get(oldID))
{
assertTrue(previousParent < oldID);
}
else
{
assertTrue(parentValues.Get(previousParent) < parentValues.Get(oldID));
}
}
// reset
previousParent = oldID;
numChildren2 = 0;
}
else
{
children = ArrayUtil.Grow(children, numChildren2 + 1);
children[numChildren2++] = oldID;
}
}
indexReader.Dispose();
writer.w.Directory.Dispose();
}
private Explanation ExplainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms)
{
Explanation result = new Explanation();
result.Description = "score(doc=" + doc + ",freq=" + freq + "), product of:";
// explain query weight
Explanation queryExpl = new Explanation();
queryExpl.Description = "queryWeight, product of:";
Explanation boostExpl = new Explanation(stats.QueryBoost, "boost");
if (stats.QueryBoost != 1.0f)
{
queryExpl.AddDetail(boostExpl);
}
queryExpl.AddDetail(stats.Idf);
Explanation queryNormExpl = new Explanation(stats.QueryNorm, "queryNorm");
queryExpl.AddDetail(queryNormExpl);
queryExpl.Value = boostExpl.Value * stats.Idf.Value * queryNormExpl.Value;
result.AddDetail(queryExpl);
// explain field weight
Explanation fieldExpl = new Explanation();
fieldExpl.Description = "fieldWeight in " + doc + ", product of:";
Explanation tfExplanation = new Explanation();
tfExplanation.Value = Tf(freq.Value);
tfExplanation.Description = "tf(freq=" + freq.Value + "), with freq of:";
tfExplanation.AddDetail(freq);
fieldExpl.AddDetail(tfExplanation);
fieldExpl.AddDetail(stats.Idf);
Explanation fieldNormExpl = new Explanation();
float fieldNorm = norms != null?DecodeNormValue(norms.Get(doc)) : 1.0f;
fieldNormExpl.Value = fieldNorm;
fieldNormExpl.Description = "fieldNorm(doc=" + doc + ")";
fieldExpl.AddDetail(fieldNormExpl);
fieldExpl.Value = tfExplanation.Value * stats.Idf.Value * fieldNormExpl.Value;
result.AddDetail(fieldExpl);
// combine them
result.Value = queryExpl.Value * fieldExpl.Value;
if (queryExpl.Value == 1.0f)
{
return(fieldExpl);
}
return(result);
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
//ORIGINAL LINE: public SortingNumericDocValues(final org.apache.lucene.index.NumericDocValues in, Sorter.DocMap docMap)
public SortingNumericDocValues(NumericDocValues @in, Sorter.DocMap docMap)
{
this.@in = @in;
this.docMap = docMap;
}
public SingleDocValuesAnonymousInnerClassHelper(NormValueSource outerInstance, NormValueSource @this, TFIDFSimilarity similarity, NumericDocValues norms)
: base(@this)
{
this.outerInstance = outerInstance;
this.similarity = similarity;
this.norms = norms;
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are ignored unless the option to convert to C# 7.2 'in' parameters is selected:
//ORIGINAL LINE: public IndexReaderStub(final org.apache.lucene.index.NumericDocValues ndv)
public IndexReaderStub(NumericDocValues ndv)
{
this._ndvs = s => ndv;
}
