private static Action <string> CreateStringArray(int length, out Func <IArrowArray> build) { var valueOffsets = new ArrowBuffer.Builder <int>(); var valueBuffer = new ArrowBuffer.Builder <byte>(); var offset = 0; build = () => { valueOffsets.Append(offset); var result = new StringArray( new ArrayData( StringType.Default, valueOffsets.Length - 1, 0, 0, new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() })); valueOffsets = null; valueBuffer = null; offset = 0; return(result); }; return(str => { var bytes = Encoding.UTF8.GetBytes(str); valueOffsets.Append(offset); // TODO: Anyway to use the span-based GetBytes to write directly to // the value buffer? valueBuffer.Append(bytes); offset += bytes.Length; }); }
private static Action <byte[]> CreateBinaryArray(int length, out Func <IArrowArray> build) { var valueOffsets = new ArrowBuffer.Builder <int>(); var valueBuffer = new ArrowBuffer.Builder <byte>(); var offset = 0; build = () => { valueOffsets.Append(offset); var result = new BinaryArray( new ArrayData( BinaryType.Default, valueOffsets.Length - 1, 0, 0, new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() })); valueOffsets = null; valueBuffer = null; offset = 0; return(result); }; return(bytes => { valueOffsets.Append(offset); valueBuffer.Append(bytes); offset += bytes.Length; }); }
private static IArrowArray ToBooleanArray(bool[] array) { byte[] rawBytes = CreateRawBytesForBoolArray(array.Length); for (int i = 0; i < array.Length; ++i) { // only need to set true values since rawBytes is zeroed // by the .NET runtime. if (array[i]) { BitUtility.SetBit(rawBytes, i); } } var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length); builder.AppendRange(rawBytes); var data = new ArrayData( BooleanType.Default, array.Length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); return(ArrowArrayFactory.BuildArray(data)); }
private static IArrowArray ToStringArrowArray(string[] array) { var valueOffsets = new ArrowBuffer.Builder <int>(); var valueBuffer = new ArrowBuffer.Builder <byte>(); int offset = 0; // TODO: Use array pool and encode directly into the array. foreach (string str in array) { byte[] bytes = Encoding.UTF8.GetBytes(str); valueOffsets.Append(offset); // TODO: Anyway to use the span-based GetBytes to write directly to // the value buffer? valueBuffer.Append(bytes); offset += bytes.Length; } valueOffsets.Append(offset); return(new StringArray( new ArrayData( StringType.Default, valueOffsets.Length - 1, 0, 0, new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() }))); }
private static Action <bool> CreateBooleanArray( int length, out Func <IArrowArray> build) { byte[] rawBytes = CreateRawBytesForBoolArray(length); build = () => { var builder = new ArrowBuffer.Builder <byte>(rawBytes.Length); builder.AppendRange(rawBytes); var data = new ArrayData( BooleanType.Default, length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); return(ArrowArrayFactory.BuildArray(data)); }; int currentIndex = 0; return(item => { // Only need to set true values since rawBytes is zeroed by the .NET runtime. if (item) { BitUtility.SetBit(rawBytes, currentIndex); } ++currentIndex; }); }
internal static DoubleArray ComputeTotal(DoubleArray price, DoubleArray discount, DoubleArray tax) { if ((price.Length != discount.Length) || (price.Length != tax.Length)) { throw new ArgumentException("Arrays need to be the same length"); } int length = price.Length; var builder = new ArrowBuffer.Builder <double>(length); ReadOnlySpan <double> prices = price.Values; ReadOnlySpan <double> discounts = discount.Values; ReadOnlySpan <double> taxes = tax.Values; for (int i = 0; i < length; ++i) { builder.Append(prices[i] * (1 - discounts[i]) * (1 + taxes[i])); } return(new DoubleArray( builder.Build(), nullBitmapBuffer: ArrowBuffer.Empty, length: length, nullCount: 0, offset: 0)); }
public void Visit(UInt16Type type) { ArrowBuffer.Builder <ushort> builder = new ArrowBuffer.Builder <ushort>(_length); for (int i = 0; i < _length; i++) { builder.Append((ushort)i); } Buffer = builder.Build(); }
public void Visit(UInt8Type type) { ArrowBuffer.Builder <byte> builder = new ArrowBuffer.Builder <byte>(_length); for (int i = 0; i < _length; i++) { builder.Append((byte)i); } Buffer = builder.Build(); }
public void Visit(BooleanType type) { ArrowBuffer.Builder <bool> builder = new ArrowBuffer.Builder <bool>(_length); for (int i = 0; i < _length; i++) { builder.Append(i % 2 == 0); } Buffer = builder.Build(); }
private void CreateNumberArray <T>(IArrowType type) where T : struct { ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>(_length); for (int i = 0; i < _length; i++) { builder.Append((T)Convert.ChangeType(i, typeof(T))); } Buffer = builder.Build(); }
public void AllocatesWithExpectedPadding(int size, int expectedCapacity) { var builder = new ArrowBuffer.Builder <byte>(size); for (int i = 0; i < size; i++) { builder.Append(0); } var buffer = builder.Build(); Assert.Equal(expectedCapacity, buffer.Length); }
public static Array MakeIntArray(int length) { // The following should be improved once the ArrayBuilder PR goes in var intBuilder = new ArrowBuffer.Builder <int>(); intBuilder.AppendRange(Enumerable.Range(0, length).Select(x => x)); ArrowBuffer buffer = intBuilder.Build(); ArrayData intData = new ArrayData(Int32Type.Default, length, 0, 0, new[] { ArrowBuffer.Empty, buffer }); Array intArray = ArrowArrayFactory.BuildArray(intData) as Array; return(intArray); }
public void SetsAllValuesToDefault() { var builder = new ArrowBuffer.Builder <int>(1); var data = Enumerable.Range(0, 10).Select(x => x).ToArray(); builder.AppendRange(data); builder.Clear(); var buffer = builder.Build(); var zeros = Enumerable.Range(0, 10).Select(x => 0).ToArray(); var values = buffer.Span.CastTo <int>().Slice(0, 10).ToArray(); Assert.True(zeros.SequenceEqual(values)); }
public void BufferHasExpectedValues() { var builder = new ArrowBuffer.Builder <int>(1); builder.Append(10); builder.Append(20); var buffer = builder.Build(); var span = buffer.Span.CastTo <int>(); Assert.Equal(10, span[0]); Assert.Equal(20, span[1]); Assert.Equal(0, span[2]); }
public unsafe void AllocatesAlignedToMultipleOf64(int size) { var builder = new ArrowBuffer.Builder <byte>(size); for (int i = 0; i < size; i++) { builder.Append(0); } var buffer = builder.Build(); fixed(byte *ptr = &buffer.Span.GetPinnableReference()) { Assert.True(new IntPtr(ptr).ToInt64() % 64 == 0); } }
public void BufferHasExpectedValues() { var builder = new ArrowBuffer.Builder <int>(1); var data = Enumerable.Range(0, 10).Select(x => x).ToArray(); builder.AppendRange(data); var buffer = builder.Build(); var span = buffer.Span.CastTo <int>(); for (var i = 0; i < 10; i++) { Assert.Equal(i, span[i]); } }
public void Visit(ListType type) { //Todo : Use ListArray.Builder var children = new[] { CreateArray(type.ValueField, Length).Data }; ArrowBuffer.Builder <int> builder = new ArrowBuffer.Builder <int>(Length); for (int i = 0; i < Length; i++) { builder.Append(i); } var valueOffsetsBuffer = builder.Build(); Array = new ListArray(new ArrayData(type, Length, 0, 0, new[] { ArrowBuffer.Empty, valueOffsetsBuffer }, children)); }
public static IArrowArray ToPrimitiveArrowArray <T>(T[] array) where T : struct { var builder = new ArrowBuffer.Builder <T>(array.Length); // TODO: The builder should have an API for blitting an array, or its IEnumerable // AppendRange should special-case T[] to do that directly when possible. foreach (T item in array) { builder.Append(item); } var data = new ArrayData( GetArrowType <T>(), array.Length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); return(ArrowArrayFactory.BuildArray(data)); }
private static Action <T> CreatePrimitiveArrowArray <T>( int length, out Func <IArrowArray> build) where T : struct { var builder = new ArrowBuffer.Builder <T>(length); build = () => { var data = new ArrayData( GetArrowType <T>(), length, 0, 0, new[] { ArrowBuffer.Empty, builder.Build() }); IArrowArray result = ArrowArrayFactory.BuildArray(data); builder = null; return(result); }; return(item => builder.Append(item)); }
public void SetsAllValuesToDefault(int sizeBeforeClear) { var builder = new ArrowBuffer.Builder <int>(1); var data = Enumerable.Range(0, sizeBeforeClear).Select(x => x).ToArray(); builder.AppendRange(data); builder.Clear(); builder.Append(0); var buffer = builder.Build(); // No matter the sizeBeforeClear, we only appended a single 0, // so the buffer length should be the smallest possible. Assert.Equal(64, buffer.Length); // check all 16 int elements are default var zeros = Enumerable.Range(0, 16).Select(x => 0).ToArray(); var values = buffer.Span.CastTo <int>().Slice(0, 16).ToArray(); Assert.True(zeros.SequenceEqual(values)); }
private static IArrowArray ToBinaryArrowArray(byte[][] array) { var valueOffsets = new ArrowBuffer.Builder <int>(); var valueBuffer = new ArrowBuffer.Builder <byte>(); int offset = 0; foreach (byte[] bytes in array) { valueOffsets.Append(offset); // TODO: Anyway to use the span-based GetBytes to write directly to // the value buffer? valueBuffer.Append(bytes); offset += bytes.Length; } valueOffsets.Append(offset); return(new StringArray( new ArrayData( BinaryType.Default, valueOffsets.Length - 1, 0, 0, new[] { ArrowBuffer.Empty, valueOffsets.Build(), valueBuffer.Build() }))); }
public static Array MakeArrayBuffer <T>(IArrowType dataType, int length, IEnumerable <T> data) where T : struct { ArrowBuffer.Builder <T> builder = new ArrowBuffer.Builder <T>(); builder.AppendRange(data); ArrowBuffer buffer = builder.Build(); ArrayData arrayData = new ArrayData(dataType, length, 0, 0, new[] { buffer }); Array array = ArrowArrayFactory.BuildArray(arrayData) as Array; return(array); }
private Tuple <Field, ListArray> CreateDictionaryTypeListArrayTestData(StringArray dictionary) { Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build(); //DictionaryArray has no Builder for now, so creating ListArray directly. var dictionaryType = new DictionaryType(Int32Type.Default, StringType.Default, false); var dictionaryArray = new DictionaryArray(dictionaryType, indiceArray, dictionary); var valueOffsetsBufferBuilder = new ArrowBuffer.Builder <int>(); var validityBufferBuilder = new ArrowBuffer.BitmapBuilder(); foreach (int i in Enumerable.Range(0, dictionary.Length + 1)) { valueOffsetsBufferBuilder.Append(i); validityBufferBuilder.Append(true); } var dictionaryField = new Field("dictionaryField_list", dictionaryType, false); var listType = new ListType(dictionaryField); var listArray = new ListArray(listType, valueOffsetsBufferBuilder.Length - 1, valueOffsetsBufferBuilder.Build(), dictionaryArray, valueOffsetsBufferBuilder.Build()); return(Tuple.Create(new Field($"listField_{listType.ValueDataType.Name}", listType, false), listArray)); }
public IArrowArray BuildArray() { offsetBuilder.Append(currentOffset); return(new ListArray(_valueType, count, offsetBuilder.Build(), _childEncoder.BuildArray(), nullBitmap.Build(), nullCount)); }