private static Action <bool> CreateBooleanArray( int length, out Func <IArrowArray> build) { byte[] rawBytes = CreateRawBytesForBoolArray(length); build = () => { var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length); builder.AppendRange(rawBytes); var data = new ArrayData( BooleanType.Default, length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); return(ArrowArrayFactory.BuildArray(data)); }; int currentIndex = 0; return(item => { // Only need to set true values since rawBytes is zeroed by the .NET runtime. if (item) { BitUtility.SetBit(rawBytes, currentIndex); } ++currentIndex; }); }
private static IArrowArray ToBooleanArray(bool[] array) { byte[] rawBytes = CreateRawBytesForBoolArray(array.Length); for (int i = 0; i < array.Length; ++i) { // only need to set true values since rawBytes is zeroed // by the .NET runtime. if (array[i]) { BitUtility.SetBit(rawBytes, i); } } var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length); builder.AppendRange(rawBytes); var data = new ArrayData( BooleanType.Default, array.Length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); return(ArrowArrayFactory.BuildArray(data)); }
private List <IArrowArray> BuildArrays( Schema schema, ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength); if (recordBatchMessage.NodesLength == 0) { return(arrays); } var recordBatchEnumerator = new RecordBatchEnumerator(in recordBatchMessage); int schemaFieldIndex = 0; do { Field field = schema.GetFieldByIndex(schemaFieldIndex++); Flatbuf.FieldNode fieldNode = recordBatchEnumerator.CurrentNode; ArrayData arrayData = field.DataType.IsFixedPrimitive() ? LoadPrimitiveField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer) : LoadVariableField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer); arrays.Add(ArrowArrayFactory.BuildArray(arrayData)); } while (recordBatchEnumerator.MoveNextNode()); return(arrays); }
public void TestSingleElement() { Int32Array array = new Int32Array.Builder().Append(1).Append(2).Build(); ArrayData actualArray = ArrayDataConcatenatorReflector.InvokeConcatenate(new[] { array.Data }); ArrowReaderVerifier.CompareArrays(array, ArrowArrayFactory.BuildArray(actualArray)); }
private static IArrowArray CreateArray(Field field, int length) { var creator = new ArrayBufferCreator(length); field.DataType.Accept(creator); ArrayData data = new ArrayData(field.DataType, length, 0, 0, new[] { ArrowBuffer.Empty, creator.Buffer }); return(ArrowArrayFactory.BuildArray(data)); }
public static Array MakeIntArray(int length) { // The following should be improved once the ArrayBuilder PR goes in var intBuilder = new ArrowBuffer.Builder <int>(); intBuilder.AppendRange(Enumerable.Range(0, length).Select(x => x)); ArrowBuffer buffer = intBuilder.Build(); ArrayData intData = new ArrayData(Int32Type.Default, length, 0, 0, new[] { ArrowBuffer.Empty, buffer }); Array intArray = ArrowArrayFactory.BuildArray(intData) as Array; return(intArray); }
public static IArrowArray ToPrimitiveArrowArray <T>(T[] array) where T : struct { var builder = new ArrowBuffer.Builder <T>(array.Length); // TODO: The builder should have an API for blitting an array, or its IEnumerable // AppendRange should special-case T[] to do that directly when possible. foreach (T item in array) { builder.Append(item); } var data = new ArrayData( GetArrowType <T>(), array.Length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); return(ArrowArrayFactory.BuildArray(data)); }
private static Action <T> CreatePrimitiveArrowArray <T>( int length, out Func <IArrowArray> build) where T : struct { var builder = new ArrowBuffer.Builder <T>(length); build = () => { var data = new ArrayData( GetArrowType <T>(), length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); IArrowArray result = ArrowArrayFactory.BuildArray(data); builder = null; return(result); }; return(item => builder.Append(item)); }
private List <IArrowArray> BuildArrays( Schema schema, ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength); int bufferIndex = 0; for (var n = 0; n < recordBatchMessage.NodesLength; n++) { Field field = schema.GetFieldByIndex(n); Flatbuf.FieldNode fieldNode = recordBatchMessage.Nodes(n).GetValueOrDefault(); ArrayData arrayData = field.DataType.IsFixedPrimitive() ? LoadPrimitiveField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex) : LoadVariableField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex); arrays.Add(ArrowArrayFactory.BuildArray(arrayData)); } return(arrays); }
public static Array MakeArrayBuffer <T>(IArrowType dataType, int length, IEnumerable <T> data) where T : struct { ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>(); builder.AppendRange(data); ArrowBuffer buffer = builder.Build(); ArrayData arrayData = new ArrayData(dataType, length, 0, 0, new[] { buffer }); Array array = ArrowArrayFactory.BuildArray(arrayData) as Array; return(array); }
public static IArrowArray CreateEmptyArray(IArrowType arrowType) { ArrayData data = BuildEmptyArrayDataFromArrowType(arrowType); return(ArrowArrayFactory.BuildArray(data)); }
public static IArrowArray CreateEmptyArray <T>() { ArrayData data = BuildEmptyArrayDataFromArrayType <T>(); return(ArrowArrayFactory.BuildArray(data)); }