public PrimitiveColumnContainer(T[] values) { values = values ?? throw new ArgumentNullException(nameof(values)); long length = values.LongLength; DataFrameBuffer <T> curBuffer; if (Buffers.Count == 0) { curBuffer = new DataFrameBuffer <T>(); Buffers.Add(curBuffer); NullBitMapBuffers.Add(new DataFrameBuffer <byte>()); } else { curBuffer = (DataFrameBuffer <T>)Buffers[Buffers.Count - 1]; } for (long i = 0; i < length; i++) { if (curBuffer.Length == ReadOnlyDataFrameBuffer <T> .MaxCapacity) { curBuffer = new DataFrameBuffer <T>(); Buffers.Add(curBuffer); NullBitMapBuffers.Add(new DataFrameBuffer <byte>()); } curBuffer.Append(values[i]); SetValidityBit(Length, true); Length++; } }
public PrimitiveColumnContainer(long length = 0) { while (length > 0) { if (Buffers.Count == 0) { Buffers.Add(new DataFrameBuffer <T>()); NullBitMapBuffers.Add(new DataFrameBuffer <byte>()); } DataFrameBuffer <T> lastBuffer = (DataFrameBuffer <T>)Buffers[Buffers.Count - 1]; if (lastBuffer.Length == ReadOnlyDataFrameBuffer <T> .MaxCapacity) { lastBuffer = new DataFrameBuffer <T>(); Buffers.Add(lastBuffer); NullBitMapBuffers.Add(new DataFrameBuffer <byte>()); } int allocatable = (int)Math.Min(length, ReadOnlyDataFrameBuffer <T> .MaxCapacity); lastBuffer.EnsureCapacity(allocatable); DataFrameBuffer <byte> lastNullBitMapBuffer = (DataFrameBuffer <byte>)(NullBitMapBuffers[NullBitMapBuffers.Count - 1]); int nullBufferAllocatable = (allocatable + 7) / 8; lastNullBitMapBuffer.EnsureCapacity(nullBufferAllocatable); lastBuffer.Length = allocatable; lastNullBitMapBuffer.Length = nullBufferAllocatable; length -= allocatable; Length += lastBuffer.Length; NullCount += lastBuffer.Length; } }
public void CumulativeMax(PrimitiveColumnContainer <DateTime> column) { var ret = column.Buffers[0].ReadOnlySpan[0]; for (int b = 0; b < column.Buffers.Count; b++) { var buffer = column.Buffers[b]; var mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; for (int i = 0; i < readOnlySpan.Length; i++) { var val = readOnlySpan[i]; if (val > ret) { ret = val; } mutableSpan[i] = ret; } column.Buffers[b] = mutableBuffer; } }
public PrimitiveColumnContainer(ReadOnlyMemory <byte> buffer, ReadOnlyMemory <byte> nullBitMap, int length, int nullCount) { ReadOnlyDataFrameBuffer <T> dataBuffer; if (buffer.IsEmpty) { DataFrameBuffer <T> mutableBuffer = new DataFrameBuffer <T>(); mutableBuffer.EnsureCapacity(length); mutableBuffer.Length = length; mutableBuffer.RawSpan.Fill(default(T)); dataBuffer = mutableBuffer; } else { dataBuffer = new ReadOnlyDataFrameBuffer <T>(buffer, length); } Buffers.Add(dataBuffer); int bitMapBufferLength = (length + 7) / 8; ReadOnlyDataFrameBuffer <byte> nullDataFrameBuffer; if (nullBitMap.IsEmpty) { if (nullCount != 0) { throw new ArgumentNullException(Strings.InconsistentNullBitMapAndNullCount, nameof(nullBitMap)); } if (!buffer.IsEmpty) { // Create a new bitMap with all the bits up to length set var bitMap = new byte[bitMapBufferLength]; bitMap.AsSpan().Fill(255); int lastByte = 1 << (length - (bitMapBufferLength - 1) * 8); bitMap[bitMapBufferLength - 1] = (byte)(lastByte - 1); nullDataFrameBuffer = new DataFrameBuffer <byte>(bitMap, bitMapBufferLength); } else { nullDataFrameBuffer = new DataFrameBuffer <byte>(); } } else { if (nullBitMap.Length < bitMapBufferLength) { throw new ArgumentException(Strings.InconsistentNullBitMapAndLength, nameof(nullBitMap)); } nullDataFrameBuffer = new ReadOnlyDataFrameBuffer <byte>(nullBitMap, bitMapBufferLength); } NullBitMapBuffers.Add(nullDataFrameBuffer); Length = length; NullCount = nullCount; }
public void CumulativeMax(PrimitiveColumnContainer <DateTime> column, IEnumerable <long> rows) { var ret = default(DateTime); var mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(column.Buffers[0]); var span = mutableBuffer.Span; long minRange = 0; long maxRange = ReadOnlyDataFrameBuffer <DateTime> .MaxCapacity; long maxCapacity = maxRange; IEnumerator <long> enumerator = rows.GetEnumerator(); if (enumerator.MoveNext()) { long row = enumerator.Current; if (row < minRange || row >= maxRange) { int bufferIndex = (int)(row / maxCapacity); mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(column.Buffers[bufferIndex]); span = mutableBuffer.Span; minRange = checked (bufferIndex * maxCapacity); maxRange = checked ((bufferIndex + 1) * maxCapacity); } row -= minRange; ret = span[(int)row]; } while (enumerator.MoveNext()) { long row = enumerator.Current; if (row < minRange || row >= maxRange) { int bufferIndex = (int)(row / maxCapacity); mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(column.Buffers[bufferIndex]); span = mutableBuffer.Span; minRange = checked (bufferIndex * maxCapacity); maxRange = checked ((bufferIndex + 1) * maxCapacity); } row -= minRange; var val = span[(int)row]; if (val > ret) { ret = val; } span[(int)row] = ret; } }
private void SetValidityBit(long index, bool value) { if ((ulong)index > (ulong)Length) { throw new ArgumentOutOfRangeException(nameof(index)); } // First find the right bitMapBuffer int bitMapIndex = GetBufferIndexContainingRowIndex(index, out int indexInBuffer); Debug.Assert(_nullBitMapBuffers.Count > bitMapIndex); DataFrameBuffer <byte> bitMapBuffer = (DataFrameBuffer <byte>)_nullBitMapBuffers[bitMapIndex]; // Set the bit int bitMapBufferIndex = (int)((uint)indexInBuffer / 8); Debug.Assert(bitMapBuffer.Length >= bitMapBufferIndex); if (bitMapBuffer.Length == bitMapBufferIndex) { bitMapBuffer.Append(0); } byte curBitMap = bitMapBuffer[bitMapBufferIndex]; byte newBitMap; if (value) { newBitMap = (byte)(curBitMap | (byte)(1 << (indexInBuffer & 7))); //bit hack for index % 8 if (((curBitMap >> (indexInBuffer & 7)) & 1) == 0 && indexInBuffer < Length - 1 && NullCount > 0) { // Old value was null. _nullCount--; } } else { if (((curBitMap >> (indexInBuffer & 7)) & 1) == 1 && indexInBuffer < Length) { // old value was NOT null and new value is null _nullCount++; } else if (indexInBuffer == Length - 1) { // New entry from an append _nullCount++; } newBitMap = (byte)(curBitMap & (byte)~(1 << (int)((uint)indexInBuffer & 7))); } bitMapBuffer[bitMapBufferIndex] = newBitMap; }
// This is an immutable column, however this method exists to support Clone(). Keep this method private // Appending a default string is equivalent to appending null. It increases the NullCount and sets a null bitmap bit // Appending an empty string is valid. It does NOT affect the NullCount. It instead adds a new offset entry private void Append(ReadOnlySpan <byte> value) { if (_dataBuffers.Count == 0) { _dataBuffers.Add(new DataFrameBuffer <byte>()); _nullBitMapBuffers.Add(new DataFrameBuffer <byte>()); _offsetsBuffers.Add(new DataFrameBuffer <int>()); } DataFrameBuffer <int> mutableOffsetsBuffer = (DataFrameBuffer <int>)_offsetsBuffers[_offsetsBuffers.Count - 1]; if (mutableOffsetsBuffer.Length == 0) { mutableOffsetsBuffer.Append(0); } Length++; if (value == default) { mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1]); } else { DataFrameBuffer <byte> mutableDataBuffer = (DataFrameBuffer <byte>)_dataBuffers[_dataBuffers.Count - 1]; if (mutableDataBuffer.Length == ReadOnlyDataFrameBuffer <byte> .MaxCapacity) { mutableDataBuffer = new DataFrameBuffer <byte>(); _dataBuffers.Add(mutableDataBuffer); _nullBitMapBuffers.Add(new DataFrameBuffer <byte>()); mutableOffsetsBuffer = new DataFrameBuffer <int>(); _offsetsBuffers.Add(mutableOffsetsBuffer); mutableOffsetsBuffer.Append(0); } mutableDataBuffer.EnsureCapacity(value.Length); value.CopyTo(mutableDataBuffer.RawSpan.Slice(mutableDataBuffer.Length)); mutableDataBuffer.Length += value.Length; mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1] + value.Length); } SetValidityBit(Length - 1, value != default); }