public PrimitiveColumnContainer(T[] values)
{
values = values ?? throw new ArgumentNullException(nameof(values));
long length = values.LongLength;
DataFrameBuffer <T> curBuffer;
if (Buffers.Count == 0)
{
curBuffer = new DataFrameBuffer <T>();
Buffers.Add(curBuffer);
NullBitMapBuffers.Add(new DataFrameBuffer <byte>());
}
else
{
curBuffer = (DataFrameBuffer <T>)Buffers[Buffers.Count - 1];
}
for (long i = 0; i < length; i++)
{
if (curBuffer.Length == ReadOnlyDataFrameBuffer <T> .MaxCapacity)
{
curBuffer = new DataFrameBuffer <T>();
Buffers.Add(curBuffer);
NullBitMapBuffers.Add(new DataFrameBuffer <byte>());
}
curBuffer.Append(values[i]);
SetValidityBit(Length, true);
Length++;
}
}
public PrimitiveColumnContainer(long length = 0)
{
while (length > 0)
{
if (Buffers.Count == 0)
{
Buffers.Add(new DataFrameBuffer <T>());
NullBitMapBuffers.Add(new DataFrameBuffer <byte>());
}
DataFrameBuffer <T> lastBuffer = (DataFrameBuffer <T>)Buffers[Buffers.Count - 1];
if (lastBuffer.Length == ReadOnlyDataFrameBuffer <T> .MaxCapacity)
{
lastBuffer = new DataFrameBuffer <T>();
Buffers.Add(lastBuffer);
NullBitMapBuffers.Add(new DataFrameBuffer <byte>());
}
int allocatable = (int)Math.Min(length, ReadOnlyDataFrameBuffer <T> .MaxCapacity);
lastBuffer.EnsureCapacity(allocatable);
DataFrameBuffer <byte> lastNullBitMapBuffer = (DataFrameBuffer <byte>)(NullBitMapBuffers[NullBitMapBuffers.Count - 1]);
int nullBufferAllocatable = (allocatable + 7) / 8;
lastNullBitMapBuffer.EnsureCapacity(nullBufferAllocatable);
lastBuffer.Length = allocatable;
lastNullBitMapBuffer.Length = nullBufferAllocatable;
length -= allocatable;
Length += lastBuffer.Length;
NullCount += lastBuffer.Length;
}
}
public void CumulativeMax(PrimitiveColumnContainer <DateTime> column)
{
var ret = column.Buffers[0].ReadOnlySpan[0];
for (int b = 0; b < column.Buffers.Count; b++)
{
var buffer = column.Buffers[b];
var mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(buffer);
var mutableSpan = mutableBuffer.Span;
var readOnlySpan = buffer.ReadOnlySpan;
for (int i = 0; i < readOnlySpan.Length; i++)
{
var val = readOnlySpan[i];
if (val > ret)
{
ret = val;
}
mutableSpan[i] = ret;
}
column.Buffers[b] = mutableBuffer;
}
}
public PrimitiveColumnContainer(ReadOnlyMemory <byte> buffer, ReadOnlyMemory <byte> nullBitMap, int length, int nullCount)
{
ReadOnlyDataFrameBuffer <T> dataBuffer;
if (buffer.IsEmpty)
{
DataFrameBuffer <T> mutableBuffer = new DataFrameBuffer <T>();
mutableBuffer.EnsureCapacity(length);
mutableBuffer.Length = length;
mutableBuffer.RawSpan.Fill(default(T));
dataBuffer = mutableBuffer;
}
else
{
dataBuffer = new ReadOnlyDataFrameBuffer <T>(buffer, length);
}
Buffers.Add(dataBuffer);
int bitMapBufferLength = (length + 7) / 8;
ReadOnlyDataFrameBuffer <byte> nullDataFrameBuffer;
if (nullBitMap.IsEmpty)
{
if (nullCount != 0)
{
throw new ArgumentNullException(Strings.InconsistentNullBitMapAndNullCount, nameof(nullBitMap));
}
if (!buffer.IsEmpty)
{
// Create a new bitMap with all the bits up to length set
var bitMap = new byte[bitMapBufferLength];
bitMap.AsSpan().Fill(255);
int lastByte = 1 << (length - (bitMapBufferLength - 1) * 8);
bitMap[bitMapBufferLength - 1] = (byte)(lastByte - 1);
nullDataFrameBuffer = new DataFrameBuffer <byte>(bitMap, bitMapBufferLength);
}
else
{
nullDataFrameBuffer = new DataFrameBuffer <byte>();
}
}
else
{
if (nullBitMap.Length < bitMapBufferLength)
{
throw new ArgumentException(Strings.InconsistentNullBitMapAndLength, nameof(nullBitMap));
}
nullDataFrameBuffer = new ReadOnlyDataFrameBuffer <byte>(nullBitMap, bitMapBufferLength);
}
NullBitMapBuffers.Add(nullDataFrameBuffer);
Length = length;
NullCount = nullCount;
}
public void CumulativeMax(PrimitiveColumnContainer <DateTime> column, IEnumerable <long> rows)
{
var ret = default(DateTime);
var mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(column.Buffers[0]);
var span = mutableBuffer.Span;
long minRange = 0;
long maxRange = ReadOnlyDataFrameBuffer <DateTime> .MaxCapacity;
long maxCapacity = maxRange;
IEnumerator <long> enumerator = rows.GetEnumerator();
if (enumerator.MoveNext())
{
long row = enumerator.Current;
if (row < minRange || row >= maxRange)
{
int bufferIndex = (int)(row / maxCapacity);
mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(column.Buffers[bufferIndex]);
span = mutableBuffer.Span;
minRange = checked (bufferIndex * maxCapacity);
maxRange = checked ((bufferIndex + 1) * maxCapacity);
}
row -= minRange;
ret = span[(int)row];
}
while (enumerator.MoveNext())
{
long row = enumerator.Current;
if (row < minRange || row >= maxRange)
{
int bufferIndex = (int)(row / maxCapacity);
mutableBuffer = DataFrameBuffer <DateTime> .GetMutableBuffer(column.Buffers[bufferIndex]);
span = mutableBuffer.Span;
minRange = checked (bufferIndex * maxCapacity);
maxRange = checked ((bufferIndex + 1) * maxCapacity);
}
row -= minRange;
var val = span[(int)row];
if (val > ret)
{
ret = val;
}
span[(int)row] = ret;
}
}
private void SetValidityBit(long index, bool value)
{
if ((ulong)index > (ulong)Length)
{
throw new ArgumentOutOfRangeException(nameof(index));
}
// First find the right bitMapBuffer
int bitMapIndex = GetBufferIndexContainingRowIndex(index, out int indexInBuffer);
Debug.Assert(_nullBitMapBuffers.Count > bitMapIndex);
DataFrameBuffer <byte> bitMapBuffer = (DataFrameBuffer <byte>)_nullBitMapBuffers[bitMapIndex];
// Set the bit
int bitMapBufferIndex = (int)((uint)indexInBuffer / 8);
Debug.Assert(bitMapBuffer.Length >= bitMapBufferIndex);
if (bitMapBuffer.Length == bitMapBufferIndex)
{
bitMapBuffer.Append(0);
}
byte curBitMap = bitMapBuffer[bitMapBufferIndex];
byte newBitMap;
if (value)
{
newBitMap = (byte)(curBitMap | (byte)(1 << (indexInBuffer & 7))); //bit hack for index % 8
if (((curBitMap >> (indexInBuffer & 7)) & 1) == 0 && indexInBuffer < Length - 1 && NullCount > 0)
{
// Old value was null.
_nullCount--;
}
}
else
{
if (((curBitMap >> (indexInBuffer & 7)) & 1) == 1 && indexInBuffer < Length)
{
// old value was NOT null and new value is null
_nullCount++;
}
else if (indexInBuffer == Length - 1)
{
// New entry from an append
_nullCount++;
}
newBitMap = (byte)(curBitMap & (byte)~(1 << (int)((uint)indexInBuffer & 7)));
}
bitMapBuffer[bitMapBufferIndex] = newBitMap;
}
// This is an immutable column, however this method exists to support Clone(). Keep this method private
// Appending a default string is equivalent to appending null. It increases the NullCount and sets a null bitmap bit
// Appending an empty string is valid. It does NOT affect the NullCount. It instead adds a new offset entry
private void Append(ReadOnlySpan <byte> value)
{
if (_dataBuffers.Count == 0)
{
_dataBuffers.Add(new DataFrameBuffer <byte>());
_nullBitMapBuffers.Add(new DataFrameBuffer <byte>());
_offsetsBuffers.Add(new DataFrameBuffer <int>());
}
DataFrameBuffer <int> mutableOffsetsBuffer = (DataFrameBuffer <int>)_offsetsBuffers[_offsetsBuffers.Count - 1];
if (mutableOffsetsBuffer.Length == 0)
{
mutableOffsetsBuffer.Append(0);
}
Length++;
if (value == default)
{
mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1]);
}
else
{
DataFrameBuffer <byte> mutableDataBuffer = (DataFrameBuffer <byte>)_dataBuffers[_dataBuffers.Count - 1];
if (mutableDataBuffer.Length == ReadOnlyDataFrameBuffer <byte> .MaxCapacity)
{
mutableDataBuffer = new DataFrameBuffer <byte>();
_dataBuffers.Add(mutableDataBuffer);
_nullBitMapBuffers.Add(new DataFrameBuffer <byte>());
mutableOffsetsBuffer = new DataFrameBuffer <int>();
_offsetsBuffers.Add(mutableOffsetsBuffer);
mutableOffsetsBuffer.Append(0);
}
mutableDataBuffer.EnsureCapacity(value.Length);
value.CopyTo(mutableDataBuffer.RawSpan.Slice(mutableDataBuffer.Length));
mutableDataBuffer.Length += value.Length;
mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1] + value.Length);
}
SetValidityBit(Length - 1, value != default);
}