private static Action <bool> CreateBooleanArray(
int length,
out Func <IArrowArray> build)
{
byte[] rawBytes = CreateRawBytesForBoolArray(length);
build = () =>
{
var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length);
builder.AppendRange(rawBytes);
var data = new ArrayData(
BooleanType.Default,
length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
};
int currentIndex = 0;
return(item =>
{
// Only need to set true values since rawBytes is zeroed by the .NET runtime.
if (item)
{
BitUtility.SetBit(rawBytes, currentIndex);
}
++currentIndex;
});
}
private static IArrowArray ToBooleanArray(bool[] array)
{
byte[] rawBytes = CreateRawBytesForBoolArray(array.Length);
for (int i = 0; i < array.Length; ++i)
{
// only need to set true values since rawBytes is zeroed
// by the .NET runtime.
if (array[i])
{
BitUtility.SetBit(rawBytes, i);
}
}
var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length);
builder.AppendRange(rawBytes);
var data = new ArrayData(
BooleanType.Default,
array.Length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
}
private List <IArrowArray> BuildArrays(
Schema schema,
ByteBuffer messageBuffer,
Flatbuf.RecordBatch recordBatchMessage)
{
var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength);
if (recordBatchMessage.NodesLength == 0)
{
return(arrays);
}
var recordBatchEnumerator = new RecordBatchEnumerator(in recordBatchMessage);
int schemaFieldIndex = 0;
do
{
Field field = schema.GetFieldByIndex(schemaFieldIndex++);
Flatbuf.FieldNode fieldNode = recordBatchEnumerator.CurrentNode;
ArrayData arrayData = field.DataType.IsFixedPrimitive()
? LoadPrimitiveField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer)
: LoadVariableField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer);
arrays.Add(ArrowArrayFactory.BuildArray(arrayData));
} while (recordBatchEnumerator.MoveNextNode());
return(arrays);
}
public void TestSingleElement()
{
Int32Array array = new Int32Array.Builder().Append(1).Append(2).Build();
ArrayData actualArray = ArrayDataConcatenatorReflector.InvokeConcatenate(new[] { array.Data });
ArrowReaderVerifier.CompareArrays(array, ArrowArrayFactory.BuildArray(actualArray));
}
private static IArrowArray CreateArray(Field field, int length)
{
var creator = new ArrayBufferCreator(length);
field.DataType.Accept(creator);
ArrayData data = new ArrayData(field.DataType, length, 0, 0,
new[] { ArrowBuffer.Empty, creator.Buffer });
return(ArrowArrayFactory.BuildArray(data));
}
public static Array MakeIntArray(int length)
{
// The following should be improved once the ArrayBuilder PR goes in
var intBuilder = new ArrowBuffer.Builder <int>();
intBuilder.AppendRange(Enumerable.Range(0, length).Select(x => x));
ArrowBuffer buffer = intBuilder.Build();
ArrayData intData = new ArrayData(Int32Type.Default, length, 0, 0, new[] { ArrowBuffer.Empty, buffer });
Array intArray = ArrowArrayFactory.BuildArray(intData) as Array;
return(intArray);
}
public static IArrowArray ToPrimitiveArrowArray <T>(T[] array) where T : struct
{
var builder = new ArrowBuffer.Builder <T>(array.Length);
// TODO: The builder should have an API for blitting an array, or its IEnumerable
// AppendRange should special-case T[] to do that directly when possible.
foreach (T item in array)
{
builder.Append(item);
}
var data = new ArrayData(
GetArrowType <T>(),
array.Length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
return(ArrowArrayFactory.BuildArray(data));
}
private static Action <T> CreatePrimitiveArrowArray <T>(
int length,
out Func <IArrowArray> build) where T : struct
{
var builder = new ArrowBuffer.Builder <T>(length);
build = () =>
{
var data = new ArrayData(
GetArrowType <T>(),
length,
0,
0,
new[] { ArrowBuffer.Empty, builder.Build() });
IArrowArray result = ArrowArrayFactory.BuildArray(data);
builder = null;
return(result);
};
return(item => builder.Append(item));
}
private List <IArrowArray> BuildArrays(
Schema schema,
ByteBuffer messageBuffer,
Flatbuf.RecordBatch recordBatchMessage)
{
var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength);
int bufferIndex = 0;
for (var n = 0; n < recordBatchMessage.NodesLength; n++)
{
Field field = schema.GetFieldByIndex(n);
Flatbuf.FieldNode fieldNode = recordBatchMessage.Nodes(n).GetValueOrDefault();
ArrayData arrayData = field.DataType.IsFixedPrimitive() ?
LoadPrimitiveField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex) :
LoadVariableField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex);
arrays.Add(ArrowArrayFactory.BuildArray(arrayData));
}
return(arrays);
}
public static Array MakeArrayBuffer <T>(IArrowType dataType,
int length,
IEnumerable <T> data)
where T : struct
{
ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>();
builder.AppendRange(data);
ArrowBuffer buffer = builder.Build();
ArrayData arrayData = new ArrayData(dataType,
length,
0,
0,
new[]
{
buffer
});
Array array = ArrowArrayFactory.BuildArray(arrayData) as Array;
return(array);
}
public static IArrowArray CreateEmptyArray(IArrowType arrowType)
{
ArrayData data = BuildEmptyArrayDataFromArrowType(arrowType);
return(ArrowArrayFactory.BuildArray(data));
}
public static IArrowArray CreateEmptyArray <T>()
{
ArrayData data = BuildEmptyArrayDataFromArrayType <T>();
return(ArrowArrayFactory.BuildArray(data));
}
