public FieldCacheDocIdSetAnonymousInnerClassHelper(FieldCacheTermsFilter outerInstance, int maxDoc, Bits acceptDocs, SortedDocValues fcsi, FixedBitSet bits)
: base(maxDoc, acceptDocs)
{
this.OuterInstance = outerInstance;
this.Fcsi = fcsi;
this.Bits = bits;
}
public IntDocValuesAnonymousInnerClassHelper(ReverseOrdFieldSource outerInstance, ReverseOrdFieldSource @this, int off, SortedDocValues sindex, int end)
: base(@this)
{
this.outerInstance = outerInstance;
this.off = off;
this.sindex = sindex;
this.end = end;
}
protected DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, string field)
{
try
{
termsIndex = FieldCache.DEFAULT.GetTermsIndex(context.AtomicReader, field);
}
catch (Exception e)
{
throw new DocTermsIndexException(field, e);
}
this.vs = vs;
}
public override DocIdSet GetDocIdSet(AtomicReaderContext context, Bits acceptDocs)
{
SortedDocValues fcsi = FieldCache.GetTermsIndex((context.AtomicReader), Field);
FixedBitSet bits = new FixedBitSet(fcsi.ValueCount);
for (int i = 0; i < Terms.Length; i++)
{
int ord = fcsi.LookupTerm(Terms[i]);
if (ord >= 0)
{
bits.Set(ord);
}
}
return(new FieldCacheDocIdSetAnonymousInnerClassHelper(this, context.Reader.MaxDoc, acceptDocs, fcsi, bits));
}
/// <exception cref="System.IO.IOException"></exception>
public override void SetNextReader(AtomicReaderContext context)
{
if (segmentFacetCounts != null)
{
segmentResults.AddItem(((TermGroupFacetCollector.SV.SegmentResult)CreateSegmentResult
()));
}
groupFieldTermsIndex = FieldCache.DEFAULT.GetTermsIndex(((AtomicReader)context.Reader
()), groupField);
facetFieldTermsIndex = FieldCache.DEFAULT.GetTermsIndex(((AtomicReader)context.Reader
()), facetField);
// 1+ to allow for the -1 "not set":
segmentFacetCounts = new int[facetFieldTermsIndex.GetValueCount() + 1];
segmentTotalCount = 0;
segmentGroupedFacetHits.Clear();
foreach (GroupedFacetHit groupedFacetHit in groupedFacetHits)
{
int facetOrd = groupedFacetHit.facetValue == null ? -1 : facetFieldTermsIndex.LookupTerm
(groupedFacetHit.facetValue);
if (groupedFacetHit.facetValue != null && facetOrd < 0)
{
continue;
}
int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.LookupTerm
(groupedFacetHit.groupValue);
if (groupedFacetHit.groupValue != null && groupOrd < 0)
{
continue;
}
int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.GetValueCount()
+ 1) + facetOrd;
segmentGroupedFacetHits.Put(segmentGroupedFacetsIndex);
}
if (facetPrefix != null)
{
startFacetOrd = facetFieldTermsIndex.LookupTerm(facetPrefix);
if (startFacetOrd < 0)
{
// Points to the ord one higher than facetPrefix
startFacetOrd = -startFacetOrd - 1;
}
BytesRef facetEndPrefix = BytesRef.DeepCopyOf(facetPrefix);
facetEndPrefix.Append(UnicodeUtil.BIG_TERM);
endFacetOrd = facetFieldTermsIndex.LookupTerm(facetEndPrefix);
endFacetOrd < 0 = -endFacetOrd - 1;
}
else
{
// Points to the ord one higher than facetEndPrefix
startFacetOrd = -1;
endFacetOrd = facetFieldTermsIndex.GetValueCount();
}
}
public override FieldComparator SetNextReader(AtomicReaderContext context)
{
idIndex = FieldCache.DEFAULT.GetTermsIndex(context.AtomicReader, Fieldname);
return this;
}
public AssertingSortedDocValues(SortedDocValues @in, int maxDoc)
{
this.@in = @in;
this.MaxDoc = maxDoc;
this.ValueCount_Renamed = @in.ValueCount;
Debug.Assert(ValueCount_Renamed >= 0 && ValueCount_Renamed <= maxDoc);
}
public BitsAnonymousInnerClassHelper(SortedDocValues dv, int maxDoc)
{
this.dv = dv;
this.maxDoc = maxDoc;
}
/// <summary>
/// Returns a multi-valued view over the provided <see cref="SortedDocValues"/>
/// </summary>
public static SortedSetDocValues Singleton(SortedDocValues dv)
{
return(new SingletonSortedSetDocValues(dv));
}
/// <summary>
/// Returns a SortedDocValues for a reader's docvalues (potentially doing extremely slow things).
/// <p>
/// this is an extremely slow way to access sorted values. Instead, access them per-segment
/// with <seealso cref="AtomicReader#getSortedDocValues(String)"/>
/// </p>
/// </summary>
public static SortedDocValues GetSortedValues(IndexReader r, string field)
{
IList<AtomicReaderContext> leaves = r.Leaves;
int size = leaves.Count;
if (size == 0)
{
return null;
}
else if (size == 1)
{
return leaves[0].AtomicReader.GetSortedDocValues(field);
}
bool anyReal = false;
var values = new SortedDocValues[size];
int[] starts = new int[size + 1];
for (int i = 0; i < size; i++)
{
AtomicReaderContext context = leaves[i];
SortedDocValues v = context.AtomicReader.GetSortedDocValues(field);
if (v == null)
{
v = DocValues.EMPTY_SORTED;
}
else
{
anyReal = true;
}
values[i] = v;
starts[i] = context.DocBase;
}
starts[size] = r.MaxDoc;
if (!anyReal)
{
return null;
}
else
{
TermsEnum[] enums = new TermsEnum[values.Length];
for (int i = 0; i < values.Length; i++)
{
enums[i] = values[i].TermsEnum();
}
OrdinalMap mapping = new OrdinalMap(r.CoreCacheKey, enums);
return new MultiSortedDocValues(values, starts, mapping);
}
}
private void AssertEquals(int maxDoc, SortedDocValues expected, SortedDocValues actual)
{
AssertEquals(maxDoc, new SingletonSortedSetDocValues(expected), new SingletonSortedSetDocValues(actual));
}
public FieldCacheDocIdSetAnonymousInnerClassHelper(MultiTermQueryFieldCacheWrapperFilter outerInstance, int maxDoc, Bits acceptDocs, SortedDocValues fcsi, LongBitSet termSet)
: base(maxDoc, acceptDocs)
{
this.OuterInstance = outerInstance;
this.Fcsi = fcsi;
this.TermSet = termSet;
}
public TermsAnonymousInnerClassHelper(MultiTermQueryFieldCacheWrapperFilter outerInstance, SortedDocValues fcsi)
{
this.OuterInstance = outerInstance;
this.Fcsi = fcsi;
}
public FieldCacheDocIdSetAnonymousInnerClassHelper(FieldCacheTermsFilter outerInstance, int maxDoc, Bits acceptDocs, SortedDocValues fcsi, FixedBitSet bits)
: base(maxDoc, acceptDocs)
{
this.OuterInstance = outerInstance;
this.Fcsi = fcsi;
this.Bits = bits;
}
public virtual void SearchIndex(Directory dir, string oldName)
{
//QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));
//Query query = parser.parse("handle:1");
IndexReader reader = DirectoryReader.Open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
TestUtil.CheckIndex(dir);
// true if this is a 4.0+ index
bool is40Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("content5") != null;
IBits liveDocs = MultiFields.GetLiveDocs(reader);
for (int i = 0; i < 35; i++)
{
if (liveDocs.Get(i))
{
Document d = reader.Document(i);
IList <IIndexableField> fields = d.Fields;
bool isProxDoc = d.GetField("content3") == null;
if (isProxDoc)
{
int numFields = is40Index ? 7 : 5;
Assert.AreEqual(numFields, fields.Count);
IIndexableField f = d.GetField("id");
Assert.AreEqual("" + i, f.GetStringValue());
f = d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.GetStringValue());
f = d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.GetStringValue());
f = d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.GetStringValue());
f = d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.GetStringValue());
}
Fields tfvFields = reader.GetTermVectors(i);
Assert.IsNotNull(tfvFields, "i=" + i);
Terms tfv = tfvFields.GetTerms("utf8");
Assert.IsNotNull(tfv, "docID=" + i + " index=" + oldName);
}
else
{
// Only ID 7 is deleted
Assert.AreEqual(7, i);
}
}
if (is40Index)
{
// check docvalues fields
NumericDocValues dvByte = MultiDocValues.GetNumericValues(reader, "dvByte");
BinaryDocValues dvBytesDerefFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefFixed");
BinaryDocValues dvBytesDerefVar = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefVar");
SortedDocValues dvBytesSortedFixed = MultiDocValues.GetSortedValues(reader, "dvBytesSortedFixed");
SortedDocValues dvBytesSortedVar = MultiDocValues.GetSortedValues(reader, "dvBytesSortedVar");
BinaryDocValues dvBytesStraightFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightFixed");
BinaryDocValues dvBytesStraightVar = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightVar");
NumericDocValues dvDouble = MultiDocValues.GetNumericValues(reader, "dvDouble");
NumericDocValues dvFloat = MultiDocValues.GetNumericValues(reader, "dvFloat");
NumericDocValues dvInt = MultiDocValues.GetNumericValues(reader, "dvInt");
NumericDocValues dvLong = MultiDocValues.GetNumericValues(reader, "dvLong");
NumericDocValues dvPacked = MultiDocValues.GetNumericValues(reader, "dvPacked");
NumericDocValues dvShort = MultiDocValues.GetNumericValues(reader, "dvShort");
for (int i = 0; i < 35; i++)
{
int id = Convert.ToInt32(reader.Document(i).Get("id"));
Assert.AreEqual(id, dvByte.Get(i));
sbyte[] bytes = new sbyte[] { (sbyte)((int)((uint)id >> 24)), (sbyte)((int)((uint)id >> 16)), (sbyte)((int)((uint)id >> 8)), (sbyte)id };
BytesRef expectedRef = new BytesRef((byte[])(Array)bytes);
BytesRef scratch = new BytesRef();
dvBytesDerefFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesDerefVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
Assert.AreEqual((double)id, J2N.BitConversion.Int64BitsToDouble(dvDouble.Get(i)), 0D);
Assert.AreEqual((float)id, J2N.BitConversion.Int32BitsToSingle((int)dvFloat.Get(i)), 0F);
Assert.AreEqual(id, dvInt.Get(i));
Assert.AreEqual(id, dvLong.Get(i));
Assert.AreEqual(id, dvPacked.Get(i));
Assert.AreEqual(id, dvShort.Get(i));
}
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
// First document should be #21 since it's norm was
// increased:
Document d_ = searcher.IndexReader.Document(hits[0].Doc);
assertEquals("didn't get the right document first", "21", d_.Get("id"));
DoTestHits(hits, 34, searcher.IndexReader);
if (is40Index)
{
hits = searcher.Search(new TermQuery(new Term("content5", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
hits = searcher.Search(new TermQuery(new Term("content6", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
}
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
reader.Dispose();
}
/// <summary>
/// Creates a multi-valued view over the provided <see cref="Index.SortedDocValues"/> </summary>
public SingletonSortedSetDocValues(SortedDocValues @in)
{
this.@in = @in;
Debug.Assert(NO_MORE_ORDS == -1); // this allows our nextOrd() to work for missing values without a check
}
/// <summary>
/// Creates a new MultiSortedDocValues over <code>values</code> </summary>
internal MultiSortedDocValues(SortedDocValues[] values, int[] docStarts, OrdinalMap mapping)
{
Debug.Assert(values.Length == mapping.OrdDeltas.Length);
Debug.Assert(docStarts.Length == values.Length + 1);
this.Values = values;
this.DocStarts = docStarts;
this.Mapping = mapping;
}
/// <summary>
/// Returns a <see cref="IBits"/> representing all documents from <paramref name="dv"/> that have a value.
/// </summary>
public static IBits DocsWithValue(SortedDocValues dv, int maxDoc)
{
return(new BitsAnonymousInnerClassHelper(dv, maxDoc));
}
/// <summary>
/// Returns a Bits representing all documents from <code>dv</code> that have a value.
/// </summary>
public static Bits DocsWithValue(SortedDocValues dv, int maxDoc)
{
return new BitsAnonymousInnerClassHelper(dv, maxDoc);
}
public override SortedDocValues GetSortedDocValues(string field)
{
EnsureOpen();
OrdinalMap map = null;
lock (CachedOrdMaps)
{
if (!CachedOrdMaps.TryGetValue(field, out map))
{
// uncached, or not a multi dv
SortedDocValues dv = MultiDocValues.GetSortedValues(@in, field);
MultiSortedDocValues docValues = dv as MultiSortedDocValues;
if (docValues != null)
{
map = docValues.Mapping;
if (map.Owner == CoreCacheKey)
{
CachedOrdMaps[field] = map;
}
}
return dv;
}
}
// cached ordinal map
if (FieldInfos.FieldInfo(field).DocValuesType != DocValuesType.SORTED)
{
return null;
}
int size = @in.Leaves().Count;
SortedDocValues[] values = new SortedDocValues[size];
int[] starts = new int[size + 1];
for (int i = 0; i < size; i++)
{
AtomicReaderContext context = @in.Leaves()[i];
SortedDocValues v = context.AtomicReader.GetSortedDocValues(field) ?? DocValues.EMPTY_SORTED;
values[i] = v;
starts[i] = context.DocBase;
}
starts[size] = MaxDoc();
return new MultiSortedDocValues(values, starts, map);
}
/// <summary>
/// Returns a multi-valued view over the provided SortedDocValues
/// </summary>
public static SortedSetDocValues Singleton(SortedDocValues dv)
{
return new SingletonSortedSetDocValues(dv);
}
private static void CheckSortedDocValues(string fieldName, AtomicReader reader, SortedDocValues dv, Bits docsWithField)
{
CheckBinaryDocValues(fieldName, reader, dv, docsWithField);
int maxOrd = dv.ValueCount - 1;
FixedBitSet seenOrds = new FixedBitSet(dv.ValueCount);
int maxOrd2 = -1;
for (int i = 0; i < reader.MaxDoc; i++)
{
int ord = dv.GetOrd(i);
if (ord == -1)
{
if (docsWithField.Get(i))
{
throw new Exception("dv for field: " + fieldName + " has -1 ord but is not marked missing for doc: " + i);
}
}
else if (ord < -1 || ord > maxOrd)
{
throw new Exception("ord out of bounds: " + ord);
}
else
{
if (!docsWithField.Get(i))
{
throw new Exception("dv for field: " + fieldName + " is missing but has ord=" + ord + " for doc: " + i);
}
maxOrd2 = Math.Max(maxOrd2, ord);
seenOrds.Set(ord);
}
}
if (maxOrd != maxOrd2)
{
throw new Exception("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2);
}
if (seenOrds.Cardinality() != dv.ValueCount)
{
throw new Exception("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.ValueCount + " but only used: " + seenOrds.Cardinality());
}
BytesRef lastValue = null;
BytesRef scratch = new BytesRef();
for (int i = 0; i <= maxOrd; i++)
{
dv.LookupOrd(i, scratch);
Debug.Assert(scratch.Valid);
if (lastValue != null)
{
if (scratch.CompareTo(lastValue) <= 0)
{
throw new Exception("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + scratch);
}
}
lastValue = BytesRef.DeepCopyOf(scratch);
}
}
private void MergeDocValues(SegmentWriteState segmentWriteState)
{
DocValuesConsumer consumer = codec.DocValuesFormat.FieldsConsumer(segmentWriteState);
bool success = false;
try
{
foreach (FieldInfo field in mergeState.FieldInfos)
{
DocValuesType type = field.DocValuesType;
if (type != DocValuesType.NONE)
{
if (type == DocValuesType.NUMERIC)
{
IList <NumericDocValues> toMerge = new List <NumericDocValues>();
IList <IBits> docsWithField = new List <IBits>();
foreach (AtomicReader reader in mergeState.Readers)
{
NumericDocValues values = reader.GetNumericDocValues(field.Name);
IBits bits = reader.GetDocsWithField(field.Name);
if (values == null)
{
values = DocValues.EMPTY_NUMERIC;
bits = new Lucene.Net.Util.Bits.MatchNoBits(reader.MaxDoc);
}
toMerge.Add(values);
docsWithField.Add(bits);
}
consumer.MergeNumericField(field, mergeState, toMerge, docsWithField);
}
else if (type == DocValuesType.BINARY)
{
IList <BinaryDocValues> toMerge = new List <BinaryDocValues>();
IList <IBits> docsWithField = new List <IBits>();
foreach (AtomicReader reader in mergeState.Readers)
{
BinaryDocValues values = reader.GetBinaryDocValues(field.Name);
IBits bits = reader.GetDocsWithField(field.Name);
if (values == null)
{
values = DocValues.EMPTY_BINARY;
bits = new Lucene.Net.Util.Bits.MatchNoBits(reader.MaxDoc);
}
toMerge.Add(values);
docsWithField.Add(bits);
}
consumer.MergeBinaryField(field, mergeState, toMerge, docsWithField);
}
else if (type == DocValuesType.SORTED)
{
IList <SortedDocValues> toMerge = new List <SortedDocValues>();
foreach (AtomicReader reader in mergeState.Readers)
{
SortedDocValues values = reader.GetSortedDocValues(field.Name);
if (values == null)
{
values = DocValues.EMPTY_SORTED;
}
toMerge.Add(values);
}
consumer.MergeSortedField(field, mergeState, toMerge);
}
else if (type == DocValuesType.SORTED_SET)
{
IList <SortedSetDocValues> toMerge = new List <SortedSetDocValues>();
foreach (AtomicReader reader in mergeState.Readers)
{
SortedSetDocValues values = reader.GetSortedSetDocValues(field.Name);
if (values == null)
{
values = DocValues.EMPTY_SORTED_SET;
}
toMerge.Add(values);
}
consumer.MergeSortedSetField(field, mergeState, toMerge);
}
else
{
throw new InvalidOperationException("type=" + type);
}
}
}
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(consumer);
}
else
{
IOUtils.DisposeWhileHandlingException(consumer);
}
}
}
/// <summary>
/// Creates a new TermsEnum over the provided values </summary>
public SortedDocValuesTermsEnum(SortedDocValues values)
{
this.Values = values;
}
public IntDocValuesAnonymousInnerClassHelper(OrdFieldSource outerInstance, OrdFieldSource @this, int off, SortedDocValues sindex)
: base(@this)
{
this.outerInstance = outerInstance;
this.off = off;
this.sindex = sindex;
}
/// <summary>
/// Creates a new TermsEnum over the provided values </summary>
public SortedDocValuesTermsEnum(SortedDocValues values)
{
this.Values = values;
}
/// <summary>
/// Creates a multi-valued view over the provided SortedDocValues </summary>
public SingletonSortedSetDocValues(SortedDocValues @in)
{
this.@in = @in;
Debug.Assert(NO_MORE_ORDS == -1); // this allows our nextOrd() to work for missing values without a check
}
