private IEnumerable <BytesRef> GetBytesIterator(int maxDocParam) { // Use yield return instead of ucsom IEnumerable BytesRef value = new BytesRef(); AppendingDeltaPackedLongBuffer.Iterator lengthsIterator = (AppendingDeltaPackedLongBuffer.Iterator)Lengths.GetIterator(); int size = (int)Lengths.Size(); DataInput bytesIterator = Bytes.DataInput; int maxDoc = maxDocParam; int upto = 0; long byteOffset = 0L; while (upto < maxDoc) { if (upto < size) { int length = (int)lengthsIterator.Next(); value.Grow(length); value.Length = length; //LUCENE TODO: This modification is slightly fishy, 4x port uses ByteBlockPool bytesIterator.ReadBytes(/*byteOffset,*/ value.Bytes, value.Offset, value.Length); byteOffset += length; } else { // This is to handle last N documents not having // this DV field in the end of the segment: value.Length = 0; } upto++; yield return(value); } }
private IEnumerable <BytesRef> GetBytesIterator(int maxDocParam) { // Use yield return instead of ucsom IEnumerable AppendingDeltaPackedLongBuffer.Iterator lengthsIterator = Lengths.GetIterator(); int size = (int)Lengths.Size(); DataInput bytesIterator = Bytes.DataInput; int maxDoc = maxDocParam; int upto = 0; while (upto < maxDoc) { BytesRef v = null; if (upto < size) { int length = (int)lengthsIterator.Next(); var value = new BytesRef(); value.Grow(length); value.Length = length; bytesIterator.ReadBytes(value.Bytes, value.Offset, value.Length); if (DocsWithField.Get(upto)) { v = value; } } upto++; yield return(v); } }
public virtual void AddValue(int docID, long value) { if (docID < Pending.Size()) { throw new System.ArgumentException("DocValuesField \"" + FieldInfo.Name + "\" appears more than once in this document (only one value is allowed per field)"); } // Fill in any holes: for (int i = (int)Pending.Size(); i < docID; ++i) { Pending.Add(MISSING); } Pending.Add(value); if (DocsWithField != null) { DocsWithField = FixedBitSet.EnsureCapacity(DocsWithField, docID); DocsWithField.Set(docID); } UpdateBytesUsed(); }
public virtual void AddValue(int docID, BytesRef value) { if (docID < Pending.Size()) { throw new System.ArgumentException("DocValuesField \"" + FieldInfo.Name + "\" appears more than once in this document (only one value is allowed per field)"); } if (value == null) { throw new System.ArgumentException("field \"" + FieldInfo.Name + "\": null value not allowed"); } if (value.Length > (ByteBlockPool.BYTE_BLOCK_SIZE - 2)) { throw new System.ArgumentException("DocValuesField \"" + FieldInfo.Name + "\" is too large, must be <= " + (ByteBlockPool.BYTE_BLOCK_SIZE - 2)); } // Fill in any holes: while (Pending.Size() < docID) { Pending.Add(EMPTY_ORD); } AddOneValue(value); }
internal override void Flush(SegmentWriteState state, DocValuesConsumer dvConsumer) { int maxDoc = state.SegmentInfo.DocCount; int maxCountPerDoc = MaxCount; Debug.Assert(PendingCounts.Size() == maxDoc); int valueCount = Hash.Size(); int[] sortedValues = Hash.Sort(BytesRef.UTF8SortedAsUnicodeComparer); int[] ordMap = new int[valueCount]; for (int ord = 0; ord < valueCount; ord++) { ordMap[sortedValues[ord]] = ord; } dvConsumer.AddSortedSetField(FieldInfo, GetBytesRefEnumberable(valueCount, sortedValues), // doc -> ordCount GetOrdsEnumberable(maxDoc), // ords GetOrdCountEnumberable(maxCountPerDoc, ordMap)); }