private static IArrowArray ToStringArrowArray(string[] array)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
int offset = 0;
// TODO: Use array pool and encode directly into the array.
foreach (string str in array)
{
byte[] bytes = Encoding.UTF8.GetBytes(str);
valueOffsets.Append(offset);
// TODO: Anyway to use the span-based GetBytes to write directly to
// the value buffer?
valueBuffer.Append(bytes);
offset += bytes.Length;
}
valueOffsets.Append(offset);
return(new StringArray(
new ArrayData(
StringType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() })));
}
public void LengthIncrementsAfterAppend()
{
var builder = new ArrowBuffer.Builder <int>(1);
var length = builder.Length;
builder.Append(1);
Assert.Equal(length + 1, builder.Length);
}
public void LengthGrowsBySpanLength()
{
var builder = new ArrowBuffer.Builder <int>(1);
var data = Enumerable.Range(0, 10).Select(x => x).ToArray();
builder.Append(data);
Assert.Equal(10, builder.Length);
}
public void CapacityOnlyGrowsWhenLengthWillExceedCapacity()
{
var builder = new ArrowBuffer.Builder <int>(1);
var capacity = builder.Capacity;
builder.Append(1);
Assert.Equal(capacity, builder.Capacity);
}
public void Visit(UInt16Type type)
{
ArrowBuffer.Builder <ushort> builder = new ArrowBuffer.Builder <ushort>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((ushort)i);
}
Buffer = builder.Build();
}
public void CapacityGrowsAfterAppendSpan()
{
var builder = new ArrowBuffer.Builder <int>(1);
var capacity = builder.Capacity;
var data = Enumerable.Range(0, 10).Select(x => x).ToArray();
builder.Append(data);
Assert.True(builder.Capacity > capacity);
}
public void Visit(UInt8Type type)
{
ArrowBuffer.Builder <byte> builder = new ArrowBuffer.Builder <byte>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((byte)i);
}
Buffer = builder.Build();
}
public void Visit(BooleanType type)
{
ArrowBuffer.Builder <bool> builder = new ArrowBuffer.Builder <bool>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append(i % 2 == 0);
}
Buffer = builder.Build();
}
private void CreateNumberArray <T>(IArrowType type)
where T : struct
{
ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((T)Convert.ChangeType(i, typeof(T)));
}
Buffer = builder.Build();
}
public void AllocatesWithExpectedPadding(int size, int expectedCapacity)
{
var builder = new ArrowBuffer.Builder <byte>(size);
for (int i = 0; i < size; i++)
{
builder.Append(0);
}
var buffer = builder.Build();
Assert.Equal(expectedCapacity, buffer.Length);
}
public void Visit(FixedSizeBinaryType type)
{
ArrowBuffer.Builder <byte> valueBuilder = new ArrowBuffer.Builder <byte>();
int valueSize = type.BitWidth;
for (int i = 0; i < Length; i++)
{
valueBuilder.Append(Enumerable.Repeat((byte)i, valueSize).ToArray());
}
ArrowBuffer validityBuffer = ArrowBuffer.Empty;
ArrowBuffer valueBuffer = valueBuilder.Build(default);
public unsafe void AllocatesAlignedToMultipleOf64(int size)
{
var builder = new ArrowBuffer.Builder <byte>(size);
for (int i = 0; i < size; i++)
{
builder.Append(0);
}
var buffer = builder.Build();
fixed(byte *ptr = &buffer.Span.GetPinnableReference())
{
Assert.True(new IntPtr(ptr).ToInt64() % 64 == 0);
}
}
private static IArrowArray ToBinaryArrowArray(byte[][] array)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
int offset = 0;
foreach (byte[] bytes in array)
{
valueOffsets.Append(offset);
// TODO: Anyway to use the span-based GetBytes to write directly to
// the value buffer?
valueBuffer.Append(bytes);
offset += bytes.Length;
}
valueOffsets.Append(offset);
return(new StringArray(
new ArrayData(
BinaryType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() })));
}
public TBuilder Append(TFrom value)
{
var convertedVal = ConvertTo(value);
var temp = new DictionaryEntry(convertedVal, Comparer, HashFunction);
if (!Entries.TryGetValue(temp, out var index))
{
index = NextIndex++;
Entries.Add(temp, index);
ValuesBuffer.Append(convertedVal);
}
IndicesBuffer.Append(index);
return(Instance);
}
public void Visit(ListType type)
{
//Todo : Use ListArray.Builder
var children = new[] { CreateArray(type.ValueField, Length).Data };
ArrowBuffer.Builder <int> builder = new ArrowBuffer.Builder <int>(Length);
for (int i = 0; i < Length; i++)
{
builder.Append(i);
}
var valueOffsetsBuffer = builder.Build();
Array = new ListArray(new ArrayData(type, Length, 0, 0,
new[] { ArrowBuffer.Empty, valueOffsetsBuffer }, children));
}
public static IArrowArray ToPrimitiveArrowArray <T>(T[] array) where T : struct
{
var builder = new ArrowBuffer.Builder <T>(array.Length);
// TODO: The builder should have an API for blitting an array, or its IEnumerable
// AppendRange should special-case T[] to do that directly when possible.
foreach (T item in array)
{
builder.Append(item);
}
var data = new ArrayData(
GetArrowType <T>(),
array.Length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
}
public void SetsAllValuesToDefault(int sizeBeforeClear)
{
var builder = new ArrowBuffer.Builder <int>(1);
var data = Enumerable.Range(0, sizeBeforeClear).Select(x => x).ToArray();
builder.AppendRange(data);
builder.Clear();
builder.Append(0);
var buffer = builder.Build();
// No matter the sizeBeforeClear, we only appended a single 0,
// so the buffer length should be the smallest possible.
Assert.Equal(64, buffer.Length);
// check all 16 int elements are default
var zeros = Enumerable.Range(0, 16).Select(x => 0).ToArray();
var values = buffer.Span.CastTo <int>().Slice(0, 16).ToArray();
Assert.True(zeros.SequenceEqual(values));
}
private static Action <T> CreatePrimitiveArrowArray <T>(
int length,
out Func <IArrowArray> build) where T : struct
{
var builder = new ArrowBuffer.Builder <T>(length);
build = () =>
{
var data = new ArrayData(
GetArrowType <T>(),
length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
IArrowArray result = ArrowArrayFactory.BuildArray(data);
builder = null;
return(result);
};
return(item => builder.Append(item));
}
private Tuple <Field, ListArray> CreateDictionaryTypeListArrayTestData(StringArray dictionary)
{
Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();
//DictionaryArray has no Builder for now, so creating ListArray directly.
var dictionaryType = new DictionaryType(Int32Type.Default, StringType.Default, false);
var dictionaryArray = new DictionaryArray(dictionaryType, indiceArray, dictionary);
var valueOffsetsBufferBuilder = new ArrowBuffer.Builder <int>();
var validityBufferBuilder = new ArrowBuffer.BitmapBuilder();
foreach (int i in Enumerable.Range(0, dictionary.Length + 1))
{
valueOffsetsBufferBuilder.Append(i);
validityBufferBuilder.Append(true);
}
var dictionaryField = new Field("dictionaryField_list", dictionaryType, false);
var listType = new ListType(dictionaryField);
var listArray = new ListArray(listType, valueOffsetsBufferBuilder.Length - 1, valueOffsetsBufferBuilder.Build(), dictionaryArray, valueOffsetsBufferBuilder.Build());
return(Tuple.Create(new Field($"listField_{listType.ValueDataType.Name}", listType, false), listArray));
}
internal static DoubleArray ComputeDiscountPrice(DoubleArray price, DoubleArray discount)
{
if (price.Length != discount.Length)
{
throw new ArgumentException("Arrays need to be the same length");
}
int length = price.Length;
var builder = new ArrowBuffer.Builder <double>(length);
ReadOnlySpan <double> prices = price.Values;
ReadOnlySpan <double> discounts = discount.Values;
for (int i = 0; i < length; ++i)
{
builder.Append(prices[i] * (1 - discounts[i]));
}
return(new DoubleArray(
builder.Build(),
nullBitmapBuffer: ArrowBuffer.Empty,
length: length,
nullCount: 0,
offset: 0));
}
public IArrowArray BuildArray()
{
offsetBuilder.Append(currentOffset);
return(new ListArray(_valueType, count, offsetBuilder.Build(), _childEncoder.BuildArray(), nullBitmap.Build(), nullCount));
}