private ArrayData GetDecimalArrayData(FixedSizeBinaryType type)
{
ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount);
var json = JsonFieldData.Data.GetRawText();
string[] values = JsonSerializer.Deserialize <string[]>(json, s_options);
Span <byte> buffer = stackalloc byte[type.ByteWidth];
ArrowBuffer.Builder <byte> valueBuilder = new ArrowBuffer.Builder <byte>();
foreach (string value in values)
{
buffer.Fill(0);
BigInteger bigInteger = BigInteger.Parse(value);
if (!bigInteger.TryWriteBytes(buffer, out int bytesWritten, false, !BitConverter.IsLittleEndian))
{
throw new InvalidDataException($"Decimal data was too big to fit into {type.BitWidth} bits.");
}
if (bigInteger.Sign == -1)
{
buffer.Slice(bytesWritten).Fill(255);
}
valueBuilder.Append(buffer);
}
ArrowBuffer valueBuffer = valueBuilder.Build(default);
protected BuilderBase(IArrowType dataType, int byteWidth)
{
DataType = dataType;
ByteWidth = byteWidth;
ValueBuffer = new ArrowBuffer.Builder <byte>();
ValidityBuffer = new ArrowBuffer.BitmapBuilder();
}
internal static DoubleArray ComputeTotal(DoubleArray price, DoubleArray discount, DoubleArray tax)
{
if ((price.Length != discount.Length) || (price.Length != tax.Length))
{
throw new ArgumentException("Arrays need to be the same length");
}
int length = price.Length;
var builder = new ArrowBuffer.Builder <double>(length);
ReadOnlySpan <double> prices = price.Values;
ReadOnlySpan <double> discounts = discount.Values;
ReadOnlySpan <double> taxes = tax.Values;
for (int i = 0; i < length; ++i)
{
builder.Append(prices[i] * (1 - discounts[i]) * (1 + taxes[i]));
}
return(new DoubleArray(
builder.Build(),
nullBitmapBuffer: ArrowBuffer.Empty,
length: length,
nullCount: 0,
offset: 0));
}
private static Action <byte[]> CreateBinaryArray(int length, out Func <IArrowArray> build)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
var offset = 0;
build = () =>
{
valueOffsets.Append(offset);
var result = new BinaryArray(
new ArrayData(
BinaryType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() }));
valueOffsets = null;
valueBuffer = null;
offset = 0;
return(result);
};
return(bytes =>
{
valueOffsets.Append(offset);
valueBuffer.Append(bytes);
offset += bytes.Length;
});
}
public void DoesNotThrowWithNullParameters()
{
var builder = new ArrowBuffer.Builder <int>();
builder.AppendRange(null);
builder.Append((Func <IEnumerable <int> >)null);
}
private static IArrowArray ToBooleanArray(bool[] array)
{
byte[] rawBytes = CreateRawBytesForBoolArray(array.Length);
for (int i = 0; i < array.Length; ++i)
{
// only need to set true values since rawBytes is zeroed
// by the .NET runtime.
if (array[i])
{
BitUtility.SetBit(rawBytes, i);
}
}
var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length);
builder.AppendRange(rawBytes);
var data = new ArrayData(
BooleanType.Default,
array.Length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
}
private static Action <string> CreateStringArray(int length, out Func <IArrowArray> build)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
var offset = 0;
build = () =>
{
valueOffsets.Append(offset);
var result = new StringArray(
new ArrayData(
StringType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() }));
valueOffsets = null;
valueBuffer = null;
offset = 0;
return(result);
};
return(str =>
{
var bytes = Encoding.UTF8.GetBytes(str);
valueOffsets.Append(offset);
// TODO: Anyway to use the span-based GetBytes to write directly to
// the value buffer?
valueBuffer.Append(bytes);
offset += bytes.Length;
});
}
private static Action <bool> CreateBooleanArray(
int length,
out Func <IArrowArray> build)
{
byte[] rawBytes = CreateRawBytesForBoolArray(length);
build = () =>
{
var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length);
builder.AppendRange(rawBytes);
var data = new ArrayData(
BooleanType.Default,
length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
};
int currentIndex = 0;
return(item =>
{
// Only need to set true values since rawBytes is zeroed by the .NET runtime.
if (item)
{
BitUtility.SetBit(rawBytes, currentIndex);
}
++currentIndex;
});
}
public void IsValidValue()
{
const int totalValueCount = 8;
const byte nullBitmap = 0b_11110011;
var nullBitmapBuffer = new ArrowBuffer.Builder <byte>().Append(nullBitmap).Build();
var valueBuffer = new ArrowBuffer.Builder <long>().Append(0).Append(1).Append(4).Append(5).Append(6).Append(7).Append(8).Build();
//Check all offset and length
for (var offset = 0; offset < totalValueCount; offset++)
{
var nullCount = totalValueCount - offset - BitUtility.CountBits(nullBitmapBuffer.Span, offset);
for (var length = 1; length + offset < totalValueCount; length++)
{
TestIsValid(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
}
}
void TestIsValid(ArrowBuffer valueBuf, ArrowBuffer nullBitmapBuf, int length, int nullCount, int offset)
{
var array = new Int64Array(valueBuf, nullBitmapBuf, length, nullCount, offset);
for (var i = 0; i < length; i++)
{
if (BitUtility.GetBit(nullBitmap, i + offset))
{
Assert.True(array.IsValid(i));
}
else
{
Assert.False(array.IsValid(i));
}
}
}
}
private static IArrowArray ToStringArrowArray(string[] array)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
int offset = 0;
// TODO: Use array pool and encode directly into the array.
foreach (string str in array)
{
byte[] bytes = Encoding.UTF8.GetBytes(str);
valueOffsets.Append(offset);
// TODO: Anyway to use the span-based GetBytes to write directly to
// the value buffer?
valueBuffer.Append(bytes);
offset += bytes.Length;
}
valueOffsets.Append(offset);
return(new StringArray(
new ArrayData(
StringType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() })));
}
public void ThrowsWhenIndexOutOfBounds()
{
Assert.Throws <IndexOutOfRangeException>(() =>
{
var builder = new ArrowBuffer.Builder <int>();
builder.Span[100] = 100;
});
}
public void LengthHasExpectedValueAfterResize()
{
var builder = new ArrowBuffer.Builder <int>();
builder.Resize(8);
Assert.True(builder.Capacity >= 8);
Assert.Equal(8, builder.Length);
}
public unsafe void AllocatesAlignedToMultipleOf64(int size)
{
var buffer = new ArrowBuffer.Builder <byte>(size).Build();
fixed(byte *ptr = &buffer.Span.GetPinnableReference())
{
Assert.True(new IntPtr(ptr).ToInt64() % 64 == 0);
}
}
public void LengthGrowsBySpanLength()
{
var builder = new ArrowBuffer.Builder <int>(1);
var data = Enumerable.Range(0, 10).Select(x => x).ToArray();
builder.Append(data);
Assert.Equal(10, builder.Length);
}
public void CapacityOnlyGrowsWhenLengthWillExceedCapacity()
{
var builder = new ArrowBuffer.Builder <int>(1);
var capacity = builder.Capacity;
builder.Append(1);
Assert.Equal(capacity, builder.Capacity);
}
public void LengthIncrementsAfterAppend()
{
var builder = new ArrowBuffer.Builder <int>(1);
var length = builder.Length;
builder.Append(1);
Assert.Equal(length + 1, builder.Length);
}
public void HasZeroPadding()
{
var buffer = new ArrowBuffer.Builder <byte>(10).Append(0).Build();
foreach (var b in buffer.Span)
{
Assert.Equal(0, b);
}
}
public void CapacityGrowsAfterAppendSpan()
{
var builder = new ArrowBuffer.Builder <int>(1);
var capacity = builder.Capacity;
var data = Enumerable.Range(0, 10).Select(x => x).ToArray();
builder.Append(data);
Assert.True(builder.Capacity > capacity);
}
public void CapacityGrowsAfterAppendWhenLengthExceedsCapacity()
{
var builder = new ArrowBuffer.Builder <int>(1);
var capacity = builder.Capacity;
builder.Append(1);
builder.Append(2);
Assert.True(builder.Capacity > capacity);
}
public void Visit(BooleanType type)
{
ArrowBuffer.Builder <bool> builder = new ArrowBuffer.Builder <bool>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append(i % 2 == 0);
}
Buffer = builder.Build();
}
public void NegativeLengthResizesToZero()
{
var builder = new ArrowBuffer.Builder <int>();
builder.Append(10);
builder.Append(20);
builder.Resize(-1);
Assert.Equal(0, builder.Length);
}
public void Visit(UInt8Type type)
{
ArrowBuffer.Builder <byte> builder = new ArrowBuffer.Builder <byte>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((byte)i);
}
Buffer = builder.Build();
}
public void Visit(UInt16Type type)
{
ArrowBuffer.Builder <ushort> builder = new ArrowBuffer.Builder <ushort>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((ushort)i);
}
Buffer = builder.Build();
}
private void CreateNumberArray <T>(IArrowType type)
where T : struct
{
ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((T)Convert.ChangeType(i, typeof(T)));
}
Buffer = builder.Build();
}
public void NegativeLengthThrows()
{
// Arrange
var builder = new ArrowBuffer.Builder <int>();
builder.Append(10);
builder.Append(20);
// Act/Assert
Assert.Throws <ArgumentOutOfRangeException>(() => builder.Resize(-1));
}
public void LengthGrowsByEnumerableCount()
{
var builder = new ArrowBuffer.Builder <int>(1);
var length = builder.Length;
var data = Enumerable.Range(0, 10).Select(x => x).ToArray();
var count = data.Length;
builder.AppendRange(data);
Assert.Equal(length + count, builder.Length);
}
/// <inheritdoc />
public StringDictionaryBuilder(Encoding encoding = null, IEqualityComparer <string> comparer = null, HashFunctionDelegate hashFunc = null) : base(comparer, hashFunc)
{
if (encoding == null)
{
encoding = DefaultEncoding;
}
Encoding = encoding;
ValueOffsets = new ArrowBuffer.Builder <int>();
ValueBuffer = new ArrowBuffer.Builder <byte>();
}
public static Array MakeIntArray(int length)
{
// The following should be improved once the ArrayBuilder PR goes in
var intBuilder = new ArrowBuffer.Builder <int>();
intBuilder.AppendRange(Enumerable.Range(0, length).Select(x => x));
ArrowBuffer buffer = intBuilder.Build();
ArrayData intData = new ArrayData(Int32Type.Default, length, 0, 0, new[] { ArrowBuffer.Empty, buffer });
Array intArray = ArrowArrayFactory.BuildArray(intData) as Array;
return(intArray);
}
public void AllocatesWithExpectedPadding(int size, int expectedCapacity)
{
var builder = new ArrowBuffer.Builder <byte>(size);
for (int i = 0; i < size; i++)
{
builder.Append(0);
}
var buffer = builder.Build();
Assert.Equal(expectedCapacity, buffer.Length);
}
public void Visit(FixedSizeBinaryType type)
{
ArrowBuffer.Builder <byte> valueBuilder = new ArrowBuffer.Builder <byte>();
int valueSize = type.BitWidth;
for (int i = 0; i < Length; i++)
{
valueBuilder.Append(Enumerable.Repeat((byte)i, valueSize).ToArray());
}
ArrowBuffer validityBuffer = ArrowBuffer.Empty;
ArrowBuffer valueBuffer = valueBuilder.Build(default);
