private static Action <string> CreateStringArray(int length, out Func <IArrowArray> build)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
var offset = 0;
build = () =>
{
valueOffsets.Append(offset);
var result = new StringArray(
new ArrayData(
StringType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() }));
valueOffsets = null;
valueBuffer = null;
offset = 0;
return(result);
};
return(str =>
{
var bytes = Encoding.UTF8.GetBytes(str);
valueOffsets.Append(offset);
// TODO: Anyway to use the span-based GetBytes to write directly to
// the value buffer?
valueBuffer.Append(bytes);
offset += bytes.Length;
});
}
private static Action <byte[]> CreateBinaryArray(int length, out Func <IArrowArray> build)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
var offset = 0;
build = () =>
{
valueOffsets.Append(offset);
var result = new BinaryArray(
new ArrayData(
BinaryType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() }));
valueOffsets = null;
valueBuffer = null;
offset = 0;
return(result);
};
return(bytes =>
{
valueOffsets.Append(offset);
valueBuffer.Append(bytes);
offset += bytes.Length;
});
}
private static IArrowArray ToBooleanArray(bool[] array)
{
byte[] rawBytes = CreateRawBytesForBoolArray(array.Length);
for (int i = 0; i < array.Length; ++i)
{
// only need to set true values since rawBytes is zeroed
// by the .NET runtime.
if (array[i])
{
BitUtility.SetBit(rawBytes, i);
}
}
var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length);
builder.AppendRange(rawBytes);
var data = new ArrayData(
BooleanType.Default,
array.Length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
}
private static IArrowArray ToStringArrowArray(string[] array)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
int offset = 0;
// TODO: Use array pool and encode directly into the array.
foreach (string str in array)
{
byte[] bytes = Encoding.UTF8.GetBytes(str);
valueOffsets.Append(offset);
// TODO: Anyway to use the span-based GetBytes to write directly to
// the value buffer?
valueBuffer.Append(bytes);
offset += bytes.Length;
}
valueOffsets.Append(offset);
return(new StringArray(
new ArrayData(
StringType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() })));
}
private static Action <bool> CreateBooleanArray(
int length,
out Func <IArrowArray> build)
{
byte[] rawBytes = CreateRawBytesForBoolArray(length);
build = () =>
{
var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length);
builder.AppendRange(rawBytes);
var data = new ArrayData(
BooleanType.Default,
length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
};
int currentIndex = 0;
return(item =>
{
// Only need to set true values since rawBytes is zeroed by the .NET runtime.
if (item)
{
BitUtility.SetBit(rawBytes, currentIndex);
}
++currentIndex;
});
}
internal static DoubleArray ComputeTotal(DoubleArray price, DoubleArray discount, DoubleArray tax)
{
if ((price.Length != discount.Length) || (price.Length != tax.Length))
{
throw new ArgumentException("Arrays need to be the same length");
}
int length = price.Length;
var builder = new ArrowBuffer.Builder <double>(length);
ReadOnlySpan <double> prices = price.Values;
ReadOnlySpan <double> discounts = discount.Values;
ReadOnlySpan <double> taxes = tax.Values;
for (int i = 0; i < length; ++i)
{
builder.Append(prices[i] * (1 - discounts[i]) * (1 + taxes[i]));
}
return(new DoubleArray(
builder.Build(),
nullBitmapBuffer: ArrowBuffer.Empty,
length: length,
nullCount: 0,
offset: 0));
}
public void Visit(UInt16Type type)
{
ArrowBuffer.Builder <ushort> builder = new ArrowBuffer.Builder <ushort>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((ushort)i);
}
Buffer = builder.Build();
}
public void Visit(UInt8Type type)
{
ArrowBuffer.Builder <byte> builder = new ArrowBuffer.Builder <byte>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((byte)i);
}
Buffer = builder.Build();
}
public void Visit(BooleanType type)
{
ArrowBuffer.Builder <bool> builder = new ArrowBuffer.Builder <bool>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append(i % 2 == 0);
}
Buffer = builder.Build();
}
private void CreateNumberArray <T>(IArrowType type)
where T : struct
{
ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>(_length);
for (int i = 0; i < _length; i++)
{
builder.Append((T)Convert.ChangeType(i, typeof(T)));
}
Buffer = builder.Build();
}
public void AllocatesWithExpectedPadding(int size, int expectedCapacity)
{
var builder = new ArrowBuffer.Builder <byte>(size);
for (int i = 0; i < size; i++)
{
builder.Append(0);
}
var buffer = builder.Build();
Assert.Equal(expectedCapacity, buffer.Length);
}
public static Array MakeIntArray(int length)
{
// The following should be improved once the ArrayBuilder PR goes in
var intBuilder = new ArrowBuffer.Builder <int>();
intBuilder.AppendRange(Enumerable.Range(0, length).Select(x => x));
ArrowBuffer buffer = intBuilder.Build();
ArrayData intData = new ArrayData(Int32Type.Default, length, 0, 0, new[] { ArrowBuffer.Empty, buffer });
Array intArray = ArrowArrayFactory.BuildArray(intData) as Array;
return(intArray);
}
public void SetsAllValuesToDefault()
{
var builder = new ArrowBuffer.Builder <int>(1);
var data = Enumerable.Range(0, 10).Select(x => x).ToArray();
builder.AppendRange(data);
builder.Clear();
var buffer = builder.Build();
var zeros = Enumerable.Range(0, 10).Select(x => 0).ToArray();
var values = buffer.Span.CastTo <int>().Slice(0, 10).ToArray();
Assert.True(zeros.SequenceEqual(values));
}
public void BufferHasExpectedValues()
{
var builder = new ArrowBuffer.Builder <int>(1);
builder.Append(10);
builder.Append(20);
var buffer = builder.Build();
var span = buffer.Span.CastTo <int>();
Assert.Equal(10, span[0]);
Assert.Equal(20, span[1]);
Assert.Equal(0, span[2]);
}
public unsafe void AllocatesAlignedToMultipleOf64(int size)
{
var builder = new ArrowBuffer.Builder <byte>(size);
for (int i = 0; i < size; i++)
{
builder.Append(0);
}
var buffer = builder.Build();
fixed(byte *ptr = &buffer.Span.GetPinnableReference())
{
Assert.True(new IntPtr(ptr).ToInt64() % 64 == 0);
}
}
public void BufferHasExpectedValues()
{
var builder = new ArrowBuffer.Builder <int>(1);
var data = Enumerable.Range(0, 10).Select(x => x).ToArray();
builder.AppendRange(data);
var buffer = builder.Build();
var span = buffer.Span.CastTo <int>();
for (var i = 0; i < 10; i++)
{
Assert.Equal(i, span[i]);
}
}
public void Visit(ListType type)
{
//Todo : Use ListArray.Builder
var children = new[] { CreateArray(type.ValueField, Length).Data };
ArrowBuffer.Builder <int> builder = new ArrowBuffer.Builder <int>(Length);
for (int i = 0; i < Length; i++)
{
builder.Append(i);
}
var valueOffsetsBuffer = builder.Build();
Array = new ListArray(new ArrayData(type, Length, 0, 0,
new[] { ArrowBuffer.Empty, valueOffsetsBuffer }, children));
}
public static IArrowArray ToPrimitiveArrowArray <T>(T[] array) where T : struct
{
var builder = new ArrowBuffer.Builder <T>(array.Length);
// TODO: The builder should have an API for blitting an array, or its IEnumerable
// AppendRange should special-case T[] to do that directly when possible.
foreach (T item in array)
{
builder.Append(item);
}
var data = new ArrayData(
GetArrowType <T>(),
array.Length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
}
private static Action <T> CreatePrimitiveArrowArray <T>(
int length,
out Func <IArrowArray> build) where T : struct
{
var builder = new ArrowBuffer.Builder <T>(length);
build = () =>
{
var data = new ArrayData(
GetArrowType <T>(),
length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
IArrowArray result = ArrowArrayFactory.BuildArray(data);
builder = null;
return(result);
};
return(item => builder.Append(item));
}
public void SetsAllValuesToDefault(int sizeBeforeClear)
{
var builder = new ArrowBuffer.Builder <int>(1);
var data = Enumerable.Range(0, sizeBeforeClear).Select(x => x).ToArray();
builder.AppendRange(data);
builder.Clear();
builder.Append(0);
var buffer = builder.Build();
// No matter the sizeBeforeClear, we only appended a single 0,
// so the buffer length should be the smallest possible.
Assert.Equal(64, buffer.Length);
// check all 16 int elements are default
var zeros = Enumerable.Range(0, 16).Select(x => 0).ToArray();
var values = buffer.Span.CastTo <int>().Slice(0, 16).ToArray();
Assert.True(zeros.SequenceEqual(values));
}
private static IArrowArray ToBinaryArrowArray(byte[][] array)
{
var valueOffsets = new ArrowBuffer.Builder <int>();
var valueBuffer = new ArrowBuffer.Builder <byte>();
int offset = 0;
foreach (byte[] bytes in array)
{
valueOffsets.Append(offset);
// TODO: Anyway to use the span-based GetBytes to write directly to
// the value buffer?
valueBuffer.Append(bytes);
offset += bytes.Length;
}
valueOffsets.Append(offset);
return(new StringArray(
new ArrayData(
BinaryType.Default,
valueOffsets.Length - 1,
0,
0,
new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() })));
}
public static Array MakeArrayBuffer <T>(IArrowType dataType,
int length,
IEnumerable <T> data)
where T : struct
{
ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>();
builder.AppendRange(data);
ArrowBuffer buffer = builder.Build();
ArrayData arrayData = new ArrayData(dataType,
length,
0,
0,
new[]
{
buffer
});
Array array = ArrowArrayFactory.BuildArray(arrayData) as Array;
return(array);
}
private Tuple <Field, ListArray> CreateDictionaryTypeListArrayTestData(StringArray dictionary)
{
Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();
//DictionaryArray has no Builder for now, so creating ListArray directly.
var dictionaryType = new DictionaryType(Int32Type.Default, StringType.Default, false);
var dictionaryArray = new DictionaryArray(dictionaryType, indiceArray, dictionary);
var valueOffsetsBufferBuilder = new ArrowBuffer.Builder <int>();
var validityBufferBuilder = new ArrowBuffer.BitmapBuilder();
foreach (int i in Enumerable.Range(0, dictionary.Length + 1))
{
valueOffsetsBufferBuilder.Append(i);
validityBufferBuilder.Append(true);
}
var dictionaryField = new Field("dictionaryField_list", dictionaryType, false);
var listType = new ListType(dictionaryField);
var listArray = new ListArray(listType, valueOffsetsBufferBuilder.Length - 1, valueOffsetsBufferBuilder.Build(), dictionaryArray, valueOffsetsBufferBuilder.Build());
return(Tuple.Create(new Field($"listField_{listType.ValueDataType.Name}", listType, false), listArray));
}
public IArrowArray BuildArray()
{
offsetBuilder.Append(currentOffset);
return(new ListArray(_valueType, count, offsetBuilder.Build(), _childEncoder.BuildArray(), nullBitmap.Build(), nullCount));
}