public FieldCacheDocIdSetAnonymousInnerClassHelper(FieldCacheTermsFilter outerInstance, int maxDoc, Bits acceptDocs, SortedDocValues fcsi, FixedBitSet bits) : base(maxDoc, acceptDocs) { this.OuterInstance = outerInstance; this.Fcsi = fcsi; this.Bits = bits; }
public IntDocValuesAnonymousInnerClassHelper(ReverseOrdFieldSource outerInstance, ReverseOrdFieldSource @this, int off, SortedDocValues sindex, int end) : base(@this) { this.outerInstance = outerInstance; this.off = off; this.sindex = sindex; this.end = end; }
protected DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, string field) { try { termsIndex = FieldCache.DEFAULT.GetTermsIndex(context.AtomicReader, field); } catch (Exception e) { throw new DocTermsIndexException(field, e); } this.vs = vs; }
public override DocIdSet GetDocIdSet(AtomicReaderContext context, Bits acceptDocs) { SortedDocValues fcsi = FieldCache.GetTermsIndex((context.AtomicReader), Field); FixedBitSet bits = new FixedBitSet(fcsi.ValueCount); for (int i = 0; i < Terms.Length; i++) { int ord = fcsi.LookupTerm(Terms[i]); if (ord >= 0) { bits.Set(ord); } } return(new FieldCacheDocIdSetAnonymousInnerClassHelper(this, context.Reader.MaxDoc, acceptDocs, fcsi, bits)); }
/// <exception cref="System.IO.IOException"></exception> public override void SetNextReader(AtomicReaderContext context) { if (segmentFacetCounts != null) { segmentResults.AddItem(((TermGroupFacetCollector.SV.SegmentResult)CreateSegmentResult ())); } groupFieldTermsIndex = FieldCache.DEFAULT.GetTermsIndex(((AtomicReader)context.Reader ()), groupField); facetFieldTermsIndex = FieldCache.DEFAULT.GetTermsIndex(((AtomicReader)context.Reader ()), facetField); // 1+ to allow for the -1 "not set": segmentFacetCounts = new int[facetFieldTermsIndex.GetValueCount() + 1]; segmentTotalCount = 0; segmentGroupedFacetHits.Clear(); foreach (GroupedFacetHit groupedFacetHit in groupedFacetHits) { int facetOrd = groupedFacetHit.facetValue == null ? -1 : facetFieldTermsIndex.LookupTerm (groupedFacetHit.facetValue); if (groupedFacetHit.facetValue != null && facetOrd < 0) { continue; } int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.LookupTerm (groupedFacetHit.groupValue); if (groupedFacetHit.groupValue != null && groupOrd < 0) { continue; } int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.GetValueCount() + 1) + facetOrd; segmentGroupedFacetHits.Put(segmentGroupedFacetsIndex); } if (facetPrefix != null) { startFacetOrd = facetFieldTermsIndex.LookupTerm(facetPrefix); if (startFacetOrd < 0) { // Points to the ord one higher than facetPrefix startFacetOrd = -startFacetOrd - 1; } BytesRef facetEndPrefix = BytesRef.DeepCopyOf(facetPrefix); facetEndPrefix.Append(UnicodeUtil.BIG_TERM); endFacetOrd = facetFieldTermsIndex.LookupTerm(facetEndPrefix); endFacetOrd < 0 = -endFacetOrd - 1; } else { // Points to the ord one higher than facetEndPrefix startFacetOrd = -1; endFacetOrd = facetFieldTermsIndex.GetValueCount(); } }
public override FieldComparator SetNextReader(AtomicReaderContext context) { idIndex = FieldCache.DEFAULT.GetTermsIndex(context.AtomicReader, Fieldname); return this; }
public AssertingSortedDocValues(SortedDocValues @in, int maxDoc) { this.@in = @in; this.MaxDoc = maxDoc; this.ValueCount_Renamed = @in.ValueCount; Debug.Assert(ValueCount_Renamed >= 0 && ValueCount_Renamed <= maxDoc); }
public BitsAnonymousInnerClassHelper(SortedDocValues dv, int maxDoc) { this.dv = dv; this.maxDoc = maxDoc; }
/// <summary> /// Returns a multi-valued view over the provided <see cref="SortedDocValues"/> /// </summary> public static SortedSetDocValues Singleton(SortedDocValues dv) { return(new SingletonSortedSetDocValues(dv)); }
/// <summary> /// Returns a SortedDocValues for a reader's docvalues (potentially doing extremely slow things). /// <p> /// this is an extremely slow way to access sorted values. Instead, access them per-segment /// with <seealso cref="AtomicReader#getSortedDocValues(String)"/> /// </p> /// </summary> public static SortedDocValues GetSortedValues(IndexReader r, string field) { IList<AtomicReaderContext> leaves = r.Leaves; int size = leaves.Count; if (size == 0) { return null; } else if (size == 1) { return leaves[0].AtomicReader.GetSortedDocValues(field); } bool anyReal = false; var values = new SortedDocValues[size]; int[] starts = new int[size + 1]; for (int i = 0; i < size; i++) { AtomicReaderContext context = leaves[i]; SortedDocValues v = context.AtomicReader.GetSortedDocValues(field); if (v == null) { v = DocValues.EMPTY_SORTED; } else { anyReal = true; } values[i] = v; starts[i] = context.DocBase; } starts[size] = r.MaxDoc; if (!anyReal) { return null; } else { TermsEnum[] enums = new TermsEnum[values.Length]; for (int i = 0; i < values.Length; i++) { enums[i] = values[i].TermsEnum(); } OrdinalMap mapping = new OrdinalMap(r.CoreCacheKey, enums); return new MultiSortedDocValues(values, starts, mapping); } }
private void AssertEquals(int maxDoc, SortedDocValues expected, SortedDocValues actual) { AssertEquals(maxDoc, new SingletonSortedSetDocValues(expected), new SingletonSortedSetDocValues(actual)); }
public FieldCacheDocIdSetAnonymousInnerClassHelper(MultiTermQueryFieldCacheWrapperFilter outerInstance, int maxDoc, Bits acceptDocs, SortedDocValues fcsi, LongBitSet termSet) : base(maxDoc, acceptDocs) { this.OuterInstance = outerInstance; this.Fcsi = fcsi; this.TermSet = termSet; }
public TermsAnonymousInnerClassHelper(MultiTermQueryFieldCacheWrapperFilter outerInstance, SortedDocValues fcsi) { this.OuterInstance = outerInstance; this.Fcsi = fcsi; }
public FieldCacheDocIdSetAnonymousInnerClassHelper(FieldCacheTermsFilter outerInstance, int maxDoc, Bits acceptDocs, SortedDocValues fcsi, FixedBitSet bits) : base(maxDoc, acceptDocs) { this.OuterInstance = outerInstance; this.Fcsi = fcsi; this.Bits = bits; }
public virtual void SearchIndex(Directory dir, string oldName) { //QueryParser parser = new QueryParser("contents", new MockAnalyzer(random)); //Query query = parser.parse("handle:1"); IndexReader reader = DirectoryReader.Open(dir); IndexSearcher searcher = new IndexSearcher(reader); TestUtil.CheckIndex(dir); // true if this is a 4.0+ index bool is40Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("content5") != null; IBits liveDocs = MultiFields.GetLiveDocs(reader); for (int i = 0; i < 35; i++) { if (liveDocs.Get(i)) { Document d = reader.Document(i); IList <IIndexableField> fields = d.Fields; bool isProxDoc = d.GetField("content3") == null; if (isProxDoc) { int numFields = is40Index ? 7 : 5; Assert.AreEqual(numFields, fields.Count); IIndexableField f = d.GetField("id"); Assert.AreEqual("" + i, f.GetStringValue()); f = d.GetField("utf8"); Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.GetStringValue()); f = d.GetField("autf8"); Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.GetStringValue()); f = d.GetField("content2"); Assert.AreEqual("here is more content with aaa aaa aaa", f.GetStringValue()); f = d.GetField("fie\u2C77ld"); Assert.AreEqual("field with non-ascii name", f.GetStringValue()); } Fields tfvFields = reader.GetTermVectors(i); Assert.IsNotNull(tfvFields, "i=" + i); Terms tfv = tfvFields.GetTerms("utf8"); Assert.IsNotNull(tfv, "docID=" + i + " index=" + oldName); } else { // Only ID 7 is deleted Assert.AreEqual(7, i); } } if (is40Index) { // check docvalues fields NumericDocValues dvByte = MultiDocValues.GetNumericValues(reader, "dvByte"); BinaryDocValues dvBytesDerefFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefFixed"); BinaryDocValues dvBytesDerefVar = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefVar"); SortedDocValues dvBytesSortedFixed = MultiDocValues.GetSortedValues(reader, "dvBytesSortedFixed"); SortedDocValues dvBytesSortedVar = MultiDocValues.GetSortedValues(reader, "dvBytesSortedVar"); BinaryDocValues dvBytesStraightFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightFixed"); BinaryDocValues dvBytesStraightVar = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightVar"); NumericDocValues dvDouble = MultiDocValues.GetNumericValues(reader, "dvDouble"); NumericDocValues dvFloat = MultiDocValues.GetNumericValues(reader, "dvFloat"); NumericDocValues dvInt = MultiDocValues.GetNumericValues(reader, "dvInt"); NumericDocValues dvLong = MultiDocValues.GetNumericValues(reader, "dvLong"); NumericDocValues dvPacked = MultiDocValues.GetNumericValues(reader, "dvPacked"); NumericDocValues dvShort = MultiDocValues.GetNumericValues(reader, "dvShort"); for (int i = 0; i < 35; i++) { int id = Convert.ToInt32(reader.Document(i).Get("id")); Assert.AreEqual(id, dvByte.Get(i)); sbyte[] bytes = new sbyte[] { (sbyte)((int)((uint)id >> 24)), (sbyte)((int)((uint)id >> 16)), (sbyte)((int)((uint)id >> 8)), (sbyte)id }; BytesRef expectedRef = new BytesRef((byte[])(Array)bytes); BytesRef scratch = new BytesRef(); dvBytesDerefFixed.Get(i, scratch); Assert.AreEqual(expectedRef, scratch); dvBytesDerefVar.Get(i, scratch); Assert.AreEqual(expectedRef, scratch); dvBytesSortedFixed.Get(i, scratch); Assert.AreEqual(expectedRef, scratch); dvBytesSortedVar.Get(i, scratch); Assert.AreEqual(expectedRef, scratch); dvBytesStraightFixed.Get(i, scratch); Assert.AreEqual(expectedRef, scratch); dvBytesStraightVar.Get(i, scratch); Assert.AreEqual(expectedRef, scratch); Assert.AreEqual((double)id, J2N.BitConversion.Int64BitsToDouble(dvDouble.Get(i)), 0D); Assert.AreEqual((float)id, J2N.BitConversion.Int32BitsToSingle((int)dvFloat.Get(i)), 0F); Assert.AreEqual(id, dvInt.Get(i)); Assert.AreEqual(id, dvLong.Get(i)); Assert.AreEqual(id, dvPacked.Get(i)); Assert.AreEqual(id, dvShort.Get(i)); } } ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs; // First document should be #21 since it's norm was // increased: Document d_ = searcher.IndexReader.Document(hits[0].Doc); assertEquals("didn't get the right document first", "21", d_.Get("id")); DoTestHits(hits, 34, searcher.IndexReader); if (is40Index) { hits = searcher.Search(new TermQuery(new Term("content5", "aaa")), null, 1000).ScoreDocs; DoTestHits(hits, 34, searcher.IndexReader); hits = searcher.Search(new TermQuery(new Term("content6", "aaa")), null, 1000).ScoreDocs; DoTestHits(hits, 34, searcher.IndexReader); } hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs; Assert.AreEqual(34, hits.Length); hits = searcher.Search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs; Assert.AreEqual(34, hits.Length); hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs; Assert.AreEqual(34, hits.Length); reader.Dispose(); }
/// <summary> /// Creates a multi-valued view over the provided <see cref="Index.SortedDocValues"/> </summary> public SingletonSortedSetDocValues(SortedDocValues @in) { this.@in = @in; Debug.Assert(NO_MORE_ORDS == -1); // this allows our nextOrd() to work for missing values without a check }
/// <summary> /// Creates a new MultiSortedDocValues over <code>values</code> </summary> internal MultiSortedDocValues(SortedDocValues[] values, int[] docStarts, OrdinalMap mapping) { Debug.Assert(values.Length == mapping.OrdDeltas.Length); Debug.Assert(docStarts.Length == values.Length + 1); this.Values = values; this.DocStarts = docStarts; this.Mapping = mapping; }
/// <summary> /// Returns a <see cref="IBits"/> representing all documents from <paramref name="dv"/> that have a value. /// </summary> public static IBits DocsWithValue(SortedDocValues dv, int maxDoc) { return(new BitsAnonymousInnerClassHelper(dv, maxDoc)); }
/// <summary> /// Returns a Bits representing all documents from <code>dv</code> that have a value. /// </summary> public static Bits DocsWithValue(SortedDocValues dv, int maxDoc) { return new BitsAnonymousInnerClassHelper(dv, maxDoc); }
public override SortedDocValues GetSortedDocValues(string field) { EnsureOpen(); OrdinalMap map = null; lock (CachedOrdMaps) { if (!CachedOrdMaps.TryGetValue(field, out map)) { // uncached, or not a multi dv SortedDocValues dv = MultiDocValues.GetSortedValues(@in, field); MultiSortedDocValues docValues = dv as MultiSortedDocValues; if (docValues != null) { map = docValues.Mapping; if (map.Owner == CoreCacheKey) { CachedOrdMaps[field] = map; } } return dv; } } // cached ordinal map if (FieldInfos.FieldInfo(field).DocValuesType != DocValuesType.SORTED) { return null; } int size = @in.Leaves().Count; SortedDocValues[] values = new SortedDocValues[size]; int[] starts = new int[size + 1]; for (int i = 0; i < size; i++) { AtomicReaderContext context = @in.Leaves()[i]; SortedDocValues v = context.AtomicReader.GetSortedDocValues(field) ?? DocValues.EMPTY_SORTED; values[i] = v; starts[i] = context.DocBase; } starts[size] = MaxDoc(); return new MultiSortedDocValues(values, starts, map); }
/// <summary> /// Returns a multi-valued view over the provided SortedDocValues /// </summary> public static SortedSetDocValues Singleton(SortedDocValues dv) { return new SingletonSortedSetDocValues(dv); }
private static void CheckSortedDocValues(string fieldName, AtomicReader reader, SortedDocValues dv, Bits docsWithField) { CheckBinaryDocValues(fieldName, reader, dv, docsWithField); int maxOrd = dv.ValueCount - 1; FixedBitSet seenOrds = new FixedBitSet(dv.ValueCount); int maxOrd2 = -1; for (int i = 0; i < reader.MaxDoc; i++) { int ord = dv.GetOrd(i); if (ord == -1) { if (docsWithField.Get(i)) { throw new Exception("dv for field: " + fieldName + " has -1 ord but is not marked missing for doc: " + i); } } else if (ord < -1 || ord > maxOrd) { throw new Exception("ord out of bounds: " + ord); } else { if (!docsWithField.Get(i)) { throw new Exception("dv for field: " + fieldName + " is missing but has ord=" + ord + " for doc: " + i); } maxOrd2 = Math.Max(maxOrd2, ord); seenOrds.Set(ord); } } if (maxOrd != maxOrd2) { throw new Exception("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2); } if (seenOrds.Cardinality() != dv.ValueCount) { throw new Exception("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.ValueCount + " but only used: " + seenOrds.Cardinality()); } BytesRef lastValue = null; BytesRef scratch = new BytesRef(); for (int i = 0; i <= maxOrd; i++) { dv.LookupOrd(i, scratch); Debug.Assert(scratch.Valid); if (lastValue != null) { if (scratch.CompareTo(lastValue) <= 0) { throw new Exception("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + scratch); } } lastValue = BytesRef.DeepCopyOf(scratch); } }
private void MergeDocValues(SegmentWriteState segmentWriteState) { DocValuesConsumer consumer = codec.DocValuesFormat.FieldsConsumer(segmentWriteState); bool success = false; try { foreach (FieldInfo field in mergeState.FieldInfos) { DocValuesType type = field.DocValuesType; if (type != DocValuesType.NONE) { if (type == DocValuesType.NUMERIC) { IList <NumericDocValues> toMerge = new List <NumericDocValues>(); IList <IBits> docsWithField = new List <IBits>(); foreach (AtomicReader reader in mergeState.Readers) { NumericDocValues values = reader.GetNumericDocValues(field.Name); IBits bits = reader.GetDocsWithField(field.Name); if (values == null) { values = DocValues.EMPTY_NUMERIC; bits = new Lucene.Net.Util.Bits.MatchNoBits(reader.MaxDoc); } toMerge.Add(values); docsWithField.Add(bits); } consumer.MergeNumericField(field, mergeState, toMerge, docsWithField); } else if (type == DocValuesType.BINARY) { IList <BinaryDocValues> toMerge = new List <BinaryDocValues>(); IList <IBits> docsWithField = new List <IBits>(); foreach (AtomicReader reader in mergeState.Readers) { BinaryDocValues values = reader.GetBinaryDocValues(field.Name); IBits bits = reader.GetDocsWithField(field.Name); if (values == null) { values = DocValues.EMPTY_BINARY; bits = new Lucene.Net.Util.Bits.MatchNoBits(reader.MaxDoc); } toMerge.Add(values); docsWithField.Add(bits); } consumer.MergeBinaryField(field, mergeState, toMerge, docsWithField); } else if (type == DocValuesType.SORTED) { IList <SortedDocValues> toMerge = new List <SortedDocValues>(); foreach (AtomicReader reader in mergeState.Readers) { SortedDocValues values = reader.GetSortedDocValues(field.Name); if (values == null) { values = DocValues.EMPTY_SORTED; } toMerge.Add(values); } consumer.MergeSortedField(field, mergeState, toMerge); } else if (type == DocValuesType.SORTED_SET) { IList <SortedSetDocValues> toMerge = new List <SortedSetDocValues>(); foreach (AtomicReader reader in mergeState.Readers) { SortedSetDocValues values = reader.GetSortedSetDocValues(field.Name); if (values == null) { values = DocValues.EMPTY_SORTED_SET; } toMerge.Add(values); } consumer.MergeSortedSetField(field, mergeState, toMerge); } else { throw new InvalidOperationException("type=" + type); } } } success = true; } finally { if (success) { IOUtils.Dispose(consumer); } else { IOUtils.DisposeWhileHandlingException(consumer); } } }
/// <summary> /// Creates a new TermsEnum over the provided values </summary> public SortedDocValuesTermsEnum(SortedDocValues values) { this.Values = values; }
public IntDocValuesAnonymousInnerClassHelper(OrdFieldSource outerInstance, OrdFieldSource @this, int off, SortedDocValues sindex) : base(@this) { this.outerInstance = outerInstance; this.off = off; this.sindex = sindex; }
/// <summary> /// Creates a new TermsEnum over the provided values </summary> public SortedDocValuesTermsEnum(SortedDocValues values) { this.Values = values; }
/// <summary> /// Creates a multi-valued view over the provided SortedDocValues </summary> public SingletonSortedSetDocValues(SortedDocValues @in) { this.@in = @in; Debug.Assert(NO_MORE_ORDS == -1); // this allows our nextOrd() to work for missing values without a check }