/// <summary>
/// Returns a DocIdSet with documents that should be permitted in search
/// results.
/// </summary>
public override DocIdSet GetDocIdSet(AtomicReaderContext context, Bits acceptDocs)
{
SortedDocValues fcsi = FieldCache.DEFAULT.GetTermsIndex((context.AtomicReader), Query.field);
// Cannot use FixedBitSet because we require long index (ord):
LongBitSet termSet = new LongBitSet(fcsi.ValueCount);
TermsEnum termsEnum = Query.GetTermsEnum(new TermsAnonymousInnerClassHelper(this, fcsi));
Debug.Assert(termsEnum != null);
if (termsEnum.Next() != null)
{
// fill into a bitset
do
{
long ord = termsEnum.Ord();
if (ord >= 0)
{
termSet.Set(ord);
}
} while (termsEnum.Next() != null);
}
else
{
return(null);
}
return(new FieldCacheDocIdSetAnonymousInnerClassHelper(this, context.Reader.MaxDoc, acceptDocs, fcsi, termSet));
}
public override void AddSortedSetField(FieldInfo field, IEnumerable <BytesRef> values, IEnumerable <long?> docToOrdCount, IEnumerable <long?> ords)
{
long valueCount = 0;
BytesRef lastValue = null;
foreach (BytesRef b in values)
{
Debug.Assert(b != null);
Debug.Assert(b.Valid);
if (valueCount > 0)
{
Debug.Assert(b.CompareTo(lastValue) > 0);
}
lastValue = BytesRef.DeepCopyOf(b);
valueCount++;
}
int docCount = 0;
long ordCount = 0;
LongBitSet seenOrds = new LongBitSet(valueCount);
IEnumerator <long?> ordIterator = ords.GetEnumerator();
foreach (long?v in docToOrdCount)
{
Debug.Assert(v != null);
int count = (int)v.Value;
Debug.Assert(count >= 0);
docCount++;
ordCount += count;
long lastOrd = -1;
for (int i = 0; i < count; i++)
{
ordIterator.MoveNext();
long?o = ordIterator.Current;
Debug.Assert(o != null);
long ord = o.Value;
Debug.Assert(ord >= 0 && ord < valueCount);
Debug.Assert(ord > lastOrd, "ord=" + ord + ",lastOrd=" + lastOrd);
seenOrds.Set(ord);
lastOrd = ord;
}
}
Debug.Assert(ordIterator.MoveNext() == false);
Debug.Assert(docCount == MaxDoc);
Debug.Assert(seenOrds.Cardinality() == valueCount);
CheckIterator(values.GetEnumerator(), valueCount, false);
CheckIterator(docToOrdCount.GetEnumerator(), MaxDoc, false);
CheckIterator(ords.GetEnumerator(), ordCount, false);
@in.AddSortedSetField(field, values, docToOrdCount, ords);
}
/*
* private class IterableAnonymousInnerClassHelper3 : IEnumerable<BytesRef>
* {
* private readonly DocValuesConsumer OuterInstance;
*
* private SortedDocValues[] Dvs;
* private OrdinalMap Map;
*
* public IterableAnonymousInnerClassHelper3(DocValuesConsumer outerInstance, SortedDocValues[] dvs, OrdinalMap map)
* {
* this.OuterInstance = outerInstance;
* this.Dvs = dvs;
* this.Map = map;
* }
*
* // ord -> value
* public virtual IEnumerator<BytesRef> GetEnumerator()
* {
* return new IteratorAnonymousInnerClassHelper3(this);
* }
*
* private class IteratorAnonymousInnerClassHelper3 : IEnumerator<BytesRef>
* {
* private readonly IterableAnonymousInnerClassHelper3 OuterInstance;
*
* public IteratorAnonymousInnerClassHelper3(IterableAnonymousInnerClassHelper3 outerInstance)
* {
* this.OuterInstance = outerInstance;
* scratch = new BytesRef();
* }
*
* internal readonly BytesRef scratch;
* internal int currentOrd;
*
* public virtual bool HasNext()
* {
* return currentOrd < OuterInstance.Map.ValueCount;
* }
*
* public virtual BytesRef Next()
* {
* if (!HasNext())
* {
* throw new Exception();
* }
* int segmentNumber = OuterInstance.Map.GetFirstSegmentNumber(currentOrd);
* int segmentOrd = (int)OuterInstance.Map.GetFirstSegmentOrd(currentOrd);
* OuterInstance.Dvs[segmentNumber].LookupOrd(segmentOrd, scratch);
* currentOrd++;
* return scratch;
* }
*
* public virtual void Remove()
* {
* throw new System.NotSupportedException();
* }
* }
* }
*
* private class IterableAnonymousInnerClassHelper4 : IEnumerable<Number>
* {
* private readonly DocValuesConsumer OuterInstance;
*
* private AtomicReader[] Readers;
* private SortedDocValues[] Dvs;
* private OrdinalMap Map;
*
* public IterableAnonymousInnerClassHelper4(DocValuesConsumer outerInstance, AtomicReader[] readers, SortedDocValues[] dvs, OrdinalMap map)
* {
* this.OuterInstance = outerInstance;
* this.Readers = readers;
* this.Dvs = dvs;
* this.Map = map;
* }
*
* public virtual IEnumerator<Number> GetEnumerator()
* {
* return new IteratorAnonymousInnerClassHelper4(this);
* }
*
* private class IteratorAnonymousInnerClassHelper4 : IEnumerator<Number>
* {
* private readonly IterableAnonymousInnerClassHelper4 OuterInstance;
*
* public IteratorAnonymousInnerClassHelper4(IterableAnonymousInnerClassHelper4 outerInstance)
* {
* this.OuterInstance = outerInstance;
* readerUpto = -1;
* }
*
* internal int readerUpto;
* internal int docIDUpto;
* internal int nextValue;
* internal AtomicReader currentReader;
* internal Bits currentLiveDocs;
* internal bool nextIsSet;
*
* public virtual bool HasNext()
* {
* return nextIsSet || SetNext();
* }
*
* public virtual void Remove()
* {
* throw new System.NotSupportedException();
* }
*
* public virtual Number Next()
* {
* if (!HasNext())
* {
* throw new NoSuchElementException();
* }
* Debug.Assert(nextIsSet);
* nextIsSet = false;
* // TODO make a mutable number
* return nextValue;
* }
*
* private bool SetNext()
* {
* while (true)
* {
* if (readerUpto == OuterInstance.Readers.Length)
* {
* return false;
* }
*
* if (currentReader == null || docIDUpto == currentReader.MaxDoc)
* {
* readerUpto++;
* if (readerUpto < OuterInstance.Readers.Length)
* {
* currentReader = OuterInstance.Readers[readerUpto];
* currentLiveDocs = currentReader.LiveDocs;
* }
* docIDUpto = 0;
* continue;
* }
*
* if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto))
* {
* nextIsSet = true;
* int segOrd = OuterInstance.Dvs[readerUpto].GetOrd(docIDUpto);
* nextValue = segOrd == -1 ? - 1 : (int) OuterInstance.Map.GetGlobalOrd(readerUpto, segOrd);
* docIDUpto++;
* return true;
* }
*
* docIDUpto++;
* }
* }
* }
* }*/
/// <summary>
/// Merges the sortedset docvalues from <code>toMerge</code>.
/// <p>
/// The default implementation calls <seealso cref="#addSortedSetField"/>, passing
/// an Iterable that merges ordinals and values and filters deleted documents .
/// </summary>
public virtual void MergeSortedSetField(FieldInfo fieldInfo, MergeState mergeState, IList <SortedSetDocValues> toMerge)
{
var readers = mergeState.Readers.ToArray();
var dvs = toMerge.ToArray();
// step 1: iterate thru each sub and mark terms still in use
var liveTerms = new TermsEnum[dvs.Length];
for (int sub = 0; sub < liveTerms.Length; sub++)
{
var reader = readers[sub];
var dv = dvs[sub];
var liveDocs = reader.LiveDocs;
if (liveDocs == null)
{
liveTerms[sub] = dv.TermsEnum();
}
else
{
var bitset = new LongBitSet(dv.ValueCount);
for (int i = 0; i < reader.MaxDoc; i++)
{
if (liveDocs.Get(i))
{
dv.Document = i;
long ord;
while ((ord = dv.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
bitset.Set(ord);
}
}
}
liveTerms[sub] = new BitsFilteredTermsEnum(dv.TermsEnum(), bitset);
}
}
// step 2: create ordinal map (this conceptually does the "merging")
var map = new OrdinalMap(this, liveTerms);
// step 3: add field
AddSortedSetField(fieldInfo, GetMergeSortedSetValuesEnumerable(map, dvs),
// doc -> ord count
GetMergeSortedSetDocToOrdCountEnumerable(readers, dvs),
// ords
GetMergeSortedSetOrdsEnumerable(readers, dvs, map)
);
}
/// <summary>
/// Merges the sorted docvalues from <code>toMerge</code>.
/// <p>
/// The default implementation calls <seealso cref="#addSortedField"/>, passing
/// an Iterable that merges ordinals and values and filters deleted documents.</p>
/// </summary>
public virtual void MergeSortedField(FieldInfo fieldInfo, MergeState mergeState, IList <SortedDocValues> toMerge)
{
AtomicReader[] readers = mergeState.Readers.ToArray();
SortedDocValues[] dvs = toMerge.ToArray();
// step 1: iterate thru each sub and mark terms still in use
var liveTerms = new TermsEnum[dvs.Length];
for (int sub = 0; sub < liveTerms.Length; sub++)
{
AtomicReader reader = readers[sub];
SortedDocValues dv = dvs[sub];
Bits liveDocs = reader.LiveDocs;
if (liveDocs == null)
{
liveTerms[sub] = dv.TermsEnum();
}
else
{
var bitset = new LongBitSet(dv.ValueCount);
for (int i = 0; i < reader.MaxDoc; i++)
{
if (liveDocs.Get(i))
{
int ord = dv.GetOrd(i);
if (ord >= 0)
{
bitset.Set(ord);
}
}
}
liveTerms[sub] = new BitsFilteredTermsEnum(dv.TermsEnum(), bitset);
}
}
// step 2: create ordinal map (this conceptually does the "merging")
var map = new OrdinalMap(this, liveTerms);
// step 3: add field
AddSortedField(fieldInfo, GetMergeSortValuesEnumerable(map, dvs),
// doc -> ord
GetMergeSortedFieldDocToOrdEnumerable(readers, dvs, map)
);
}
/// <summary>
/// Returns a DocIdSet with documents that should be permitted in search
/// results.
/// </summary>
public override DocIdSet GetDocIdSet(AtomicReaderContext context, Bits acceptDocs)
{
SortedSetDocValues docTermOrds = FieldCache.DEFAULT.GetDocTermOrds((context.AtomicReader), Query.field);
// Cannot use FixedBitSet because we require long index (ord):
LongBitSet termSet = new LongBitSet(docTermOrds.ValueCount);
TermsEnum termsEnum = Query.GetTermsEnum(new TermsAnonymousInnerClassHelper(this, docTermOrds));
Debug.Assert(termsEnum != null);
if (termsEnum.Next() != null)
{
// fill into a bitset
do
{
termSet.Set(termsEnum.Ord());
} while (termsEnum.Next() != null);
}
else
{
return null;
}
return new FieldCacheDocIdSetAnonymousInnerClassHelper(this, context.Reader.MaxDoc, acceptDocs, docTermOrds, termSet);
}
/*
private class IterableAnonymousInnerClassHelper3 : IEnumerable<BytesRef>
{
private readonly DocValuesConsumer OuterInstance;
private SortedDocValues[] Dvs;
private OrdinalMap Map;
public IterableAnonymousInnerClassHelper3(DocValuesConsumer outerInstance, SortedDocValues[] dvs, OrdinalMap map)
{
this.OuterInstance = outerInstance;
this.Dvs = dvs;
this.Map = map;
}
// ord -> value
public virtual IEnumerator<BytesRef> GetEnumerator()
{
return new IteratorAnonymousInnerClassHelper3(this);
}
private class IteratorAnonymousInnerClassHelper3 : IEnumerator<BytesRef>
{
private readonly IterableAnonymousInnerClassHelper3 OuterInstance;
public IteratorAnonymousInnerClassHelper3(IterableAnonymousInnerClassHelper3 outerInstance)
{
this.OuterInstance = outerInstance;
scratch = new BytesRef();
}
internal readonly BytesRef scratch;
internal int currentOrd;
public virtual bool HasNext()
{
return currentOrd < OuterInstance.Map.ValueCount;
}
public virtual BytesRef Next()
{
if (!HasNext())
{
throw new Exception();
}
int segmentNumber = OuterInstance.Map.GetFirstSegmentNumber(currentOrd);
int segmentOrd = (int)OuterInstance.Map.GetFirstSegmentOrd(currentOrd);
OuterInstance.Dvs[segmentNumber].LookupOrd(segmentOrd, scratch);
currentOrd++;
return scratch;
}
public virtual void Remove()
{
throw new System.NotSupportedException();
}
}
}
private class IterableAnonymousInnerClassHelper4 : IEnumerable<Number>
{
private readonly DocValuesConsumer OuterInstance;
private AtomicReader[] Readers;
private SortedDocValues[] Dvs;
private OrdinalMap Map;
public IterableAnonymousInnerClassHelper4(DocValuesConsumer outerInstance, AtomicReader[] readers, SortedDocValues[] dvs, OrdinalMap map)
{
this.OuterInstance = outerInstance;
this.Readers = readers;
this.Dvs = dvs;
this.Map = map;
}
public virtual IEnumerator<Number> GetEnumerator()
{
return new IteratorAnonymousInnerClassHelper4(this);
}
private class IteratorAnonymousInnerClassHelper4 : IEnumerator<Number>
{
private readonly IterableAnonymousInnerClassHelper4 OuterInstance;
public IteratorAnonymousInnerClassHelper4(IterableAnonymousInnerClassHelper4 outerInstance)
{
this.OuterInstance = outerInstance;
readerUpto = -1;
}
internal int readerUpto;
internal int docIDUpto;
internal int nextValue;
internal AtomicReader currentReader;
internal Bits currentLiveDocs;
internal bool nextIsSet;
public virtual bool HasNext()
{
return nextIsSet || SetNext();
}
public virtual void Remove()
{
throw new System.NotSupportedException();
}
public virtual Number Next()
{
if (!HasNext())
{
throw new NoSuchElementException();
}
Debug.Assert(nextIsSet);
nextIsSet = false;
// TODO make a mutable number
return nextValue;
}
private bool SetNext()
{
while (true)
{
if (readerUpto == OuterInstance.Readers.Length)
{
return false;
}
if (currentReader == null || docIDUpto == currentReader.MaxDoc)
{
readerUpto++;
if (readerUpto < OuterInstance.Readers.Length)
{
currentReader = OuterInstance.Readers[readerUpto];
currentLiveDocs = currentReader.LiveDocs;
}
docIDUpto = 0;
continue;
}
if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto))
{
nextIsSet = true;
int segOrd = OuterInstance.Dvs[readerUpto].GetOrd(docIDUpto);
nextValue = segOrd == -1 ? - 1 : (int) OuterInstance.Map.GetGlobalOrd(readerUpto, segOrd);
docIDUpto++;
return true;
}
docIDUpto++;
}
}
}
}*/
/// <summary>
/// Merges the sortedset docvalues from <code>toMerge</code>.
/// <p>
/// The default implementation calls <seealso cref="#addSortedSetField"/>, passing
/// an Iterable that merges ordinals and values and filters deleted documents .
/// </summary>
public virtual void MergeSortedSetField(FieldInfo fieldInfo, MergeState mergeState, IList<SortedSetDocValues> toMerge)
{
var readers = mergeState.Readers.ToArray();
var dvs = toMerge.ToArray();
// step 1: iterate thru each sub and mark terms still in use
var liveTerms = new TermsEnum[dvs.Length];
for (int sub = 0; sub < liveTerms.Length; sub++)
{
var reader = readers[sub];
var dv = dvs[sub];
var liveDocs = reader.LiveDocs;
if (liveDocs == null)
{
liveTerms[sub] = dv.TermsEnum();
}
else
{
var bitset = new LongBitSet(dv.ValueCount);
for (int i = 0; i < reader.MaxDoc; i++)
{
if (liveDocs.Get(i))
{
dv.Document = i;
long ord;
while ((ord = dv.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
bitset.Set(ord);
}
}
}
liveTerms[sub] = new BitsFilteredTermsEnum(dv.TermsEnum(), bitset);
}
}
// step 2: create ordinal map (this conceptually does the "merging")
var map = new OrdinalMap(this, liveTerms);
// step 3: add field
AddSortedSetField(fieldInfo, GetMergeSortedSetValuesEnumerable(map, dvs),
// doc -> ord count
GetMergeSortedSetDocToOrdCountEnumerable(readers, dvs),
// ords
GetMergeSortedSetOrdsEnumerable(readers, dvs, map)
);
}
/// <summary>
/// Merges the sorted docvalues from <code>toMerge</code>.
/// <p>
/// The default implementation calls <seealso cref="#addSortedField"/>, passing
/// an Iterable that merges ordinals and values and filters deleted documents.</p>
/// </summary>
public virtual void MergeSortedField(FieldInfo fieldInfo, MergeState mergeState, IList<SortedDocValues> toMerge)
{
AtomicReader[] readers = mergeState.Readers.ToArray();
SortedDocValues[] dvs = toMerge.ToArray();
// step 1: iterate thru each sub and mark terms still in use
var liveTerms = new TermsEnum[dvs.Length];
for (int sub = 0; sub < liveTerms.Length; sub++)
{
AtomicReader reader = readers[sub];
SortedDocValues dv = dvs[sub];
Bits liveDocs = reader.LiveDocs;
if (liveDocs == null)
{
liveTerms[sub] = dv.TermsEnum();
}
else
{
var bitset = new LongBitSet(dv.ValueCount);
for (int i = 0; i < reader.MaxDoc; i++)
{
if (liveDocs.Get(i))
{
int ord = dv.GetOrd(i);
if (ord >= 0)
{
bitset.Set(ord);
}
}
}
liveTerms[sub] = new BitsFilteredTermsEnum(dv.TermsEnum(), bitset);
}
}
// step 2: create ordinal map (this conceptually does the "merging")
var map = new OrdinalMap(this, liveTerms);
// step 3: add field
AddSortedField(fieldInfo, GetMergeSortValuesEnumerable(map, dvs),
// doc -> ord
GetMergeSortedFieldDocToOrdEnumerable(readers, dvs, map)
);
}
private static void CheckSortedSetDocValues(string fieldName, AtomicReader reader, SortedSetDocValues dv, Bits docsWithField)
{
long maxOrd = dv.ValueCount - 1;
LongBitSet seenOrds = new LongBitSet(dv.ValueCount);
long maxOrd2 = -1;
for (int i = 0; i < reader.MaxDoc; i++)
{
dv.Document = i;
long lastOrd = -1;
long ord;
if (docsWithField.Get(i))
{
int ordCount = 0;
while ((ord = dv.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
if (ord <= lastOrd)
{
throw new Exception("ords out of order: " + ord + " <= " + lastOrd + " for doc: " + i);
}
if (ord < 0 || ord > maxOrd)
{
throw new Exception("ord out of bounds: " + ord);
}
if (dv is RandomAccessOrds)
{
long ord2 = ((RandomAccessOrds)dv).OrdAt(ordCount);
if (ord != ord2)
{
throw new Exception("ordAt(" + ordCount + ") inconsistent, expected=" + ord + ",got=" + ord2 + " for doc: " + i);
}
}
lastOrd = ord;
maxOrd2 = Math.Max(maxOrd2, ord);
seenOrds.Set(ord);
ordCount++;
}
if (ordCount == 0)
{
throw new Exception("dv for field: " + fieldName + " has no ordinals but is not marked missing for doc: " + i);
}
if (dv is RandomAccessOrds)
{
long ordCount2 = ((RandomAccessOrds)dv).Cardinality();
if (ordCount != ordCount2)
{
throw new Exception("cardinality inconsistent, expected=" + ordCount + ",got=" + ordCount2 + " for doc: " + i);
}
}
}
else
{
long o = dv.NextOrd();
if (o != SortedSetDocValues.NO_MORE_ORDS)
{
throw new Exception("dv for field: " + fieldName + " is marked missing but has ord=" + o + " for doc: " + i);
}
if (dv is RandomAccessOrds)
{
long ordCount2 = ((RandomAccessOrds)dv).Cardinality();
if (ordCount2 != 0)
{
throw new Exception("dv for field: " + fieldName + " is marked missing but has cardinality " + ordCount2 + " for doc: " + i);
}
}
}
}
if (maxOrd != maxOrd2)
{
throw new Exception("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2);
}
if (seenOrds.Cardinality() != dv.ValueCount)
{
throw new Exception("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.ValueCount + " but only used: " + seenOrds.Cardinality());
}
BytesRef lastValue = null;
BytesRef scratch = new BytesRef();
for (long i = 0; i <= maxOrd; i++)
{
dv.LookupOrd(i, scratch);
Debug.Assert(scratch.Valid);
if (lastValue != null)
{
if (scratch.CompareTo(lastValue) <= 0)
{
throw new Exception("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + scratch);
}
}
lastValue = BytesRef.DeepCopyOf(scratch);
}
}