private void CompareArrays <T>(PrimitiveArray <T> actualArray) where T : struct, IEquatable <T> { Assert.IsAssignableFrom <PrimitiveArray <T> >(_expectedArray); PrimitiveArray <T> expectedArray = (PrimitiveArray <T>)_expectedArray; Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); Assert.Equal(expectedArray.Offset, actualArray.Offset); if (expectedArray.NullCount > 0) { Assert.True(expectedArray.NullBitmapBuffer.Span.SequenceEqual(actualArray.NullBitmapBuffer.Span)); } else { // expectedArray may have passed in a null bitmap. DataFrame might have populated it with Length set bits Assert.Equal(0, expectedArray.NullCount); Assert.Equal(0, actualArray.NullCount); for (int i = 0; i < actualArray.Length; i++) { Assert.True(actualArray.IsValid(i)); } } Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length))); }
private void BindPrimitiveArray(HbmPrimitiveArray arrayMapping, PrimitiveArray model, string prefix, string path, System.Type containingType, IDictionary <string, MetaAttribute> inheritedMetas) { BindCollection(arrayMapping, model, prefix, path, containingType, inheritedMetas); var element = arrayMapping.ElementRelationship as HbmElement; if (element != null) { string typeName; var typeAttribute = element.Type; if (typeAttribute != null) { typeName = typeAttribute.name; } else { throw new MappingException("type for <element> was not defined"); } IType type = TypeFactory.HeuristicType(typeName, null); if (type == null) { throw new MappingException("could not interpret type: " + typeName); } model.ElementClassName = type.ReturnedClass.AssemblyQualifiedName; } }
private void CompareArrays <T>(PrimitiveArray <T> actualArray) where T : struct, IEquatable <T> { Assert.IsAssignableFrom <PrimitiveArray <T> >(_expectedArray); PrimitiveArray <T> expectedArray = (PrimitiveArray <T>)_expectedArray; actualArray.Data.DataType.Accept(_arrayTypeComparer); Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); Assert.Equal(expectedArray.Offset, actualArray.Offset); CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, actualArray.NullBitmapBuffer); if (_strictCompare) { Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length))); } else { for (int i = 0; i < expectedArray.Length; i++) { Assert.Equal(expectedArray.GetValue(i), actualArray.GetValue(i)); } } }
private Mapping.Collection CreatePrimitiveArray(XmlNode node, string prefix, string path, PersistentClass owner, System.Type containingType, IDictionary <string, MetaAttribute> inheritedMetas) { PrimitiveArray array = new PrimitiveArray(owner); BindArray(node, array, prefix, path, containingType, inheritedMetas); return(array); }
private Mapping.Collection CreatePrimitiveArray(HbmPrimitiveArray primitiveArrayMapping, string prefix, string path, PersistentClass owner, System.Type containingType, IDictionary <string, MetaAttribute> inheritedMetas) { var array = new PrimitiveArray(owner); BindPrimitiveArray(primitiveArrayMapping, array, prefix, path, containingType, inheritedMetas); AddPrimitiveArraySecondPass(primitiveArrayMapping, array, inheritedMetas); return(array); }
public bool Equals(PrimitiveArray <ValueType> other) { if (other == null) { return(false); } return((m_ArrayElements == null) ? (other.m_ArrayElements == null) : m_ArrayElements.SequenceEqual(other.m_ArrayElements)); }
internal override void NewBatch(IArrowArray arrowArray) { if (arrowArray is PrimitiveArray <T> primitiveArray) { Array = primitiveArray; } else { throw new ArgumentException("Expected primitive array", nameof(arrowArray)); } }
private void CompareArrays <T>(PrimitiveArray <T> actualArray) where T : struct, IEquatable <T> { Assert.IsAssignableFrom <PrimitiveArray <T> >(_expectedArray); PrimitiveArray <T> expectedArray = (PrimitiveArray <T>)_expectedArray; Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); Assert.Equal(expectedArray.Offset, actualArray.Offset); Assert.True(expectedArray.NullBitmapBuffer.Span.SequenceEqual(actualArray.NullBitmapBuffer.Span)); Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length))); }
internal void Init() { this.PRparseTypeEnum = InternalParseTypeE.Empty; this.PRobjectTypeEnum = InternalObjectTypeE.Empty; this.PRarrayTypeEnum = InternalArrayTypeE.Empty; this.PRmemberTypeEnum = InternalMemberTypeE.Empty; this.PRmemberValueEnum = InternalMemberValueE.Empty; this.PRobjectPositionEnum = InternalObjectPositionE.Empty; this.PRname = (string)null; this.PRvalue = (string)null; this.PRkeyDt = (string)null; this.PRdtType = (Type)null; this.PRdtTypeCode = InternalPrimitiveTypeE.Invalid; this.PRisEnum = false; this.PRobjectId = 0L; this.PRidRef = 0L; this.PRarrayElementTypeString = (string)null; this.PRarrayElementType = (Type)null; this.PRisArrayVariant = false; this.PRarrayElementTypeCode = InternalPrimitiveTypeE.Invalid; this.PRrank = 0; this.PRlengthA = (int[])null; this.PRpositionA = (int[])null; this.PRlowerBoundA = (int[])null; this.PRupperBoundA = (int[])null; this.PRindexMap = (int[])null; this.PRmemberIndex = 0; this.PRlinearlength = 0; this.PRrectangularMap = (int[])null; this.PRisLowerBound = false; this.PRtopId = 0L; this.PRheaderId = 0L; this.PRisValueTypeFixup = false; this.PRnewObj = (object)null; this.PRobjectA = (object[])null; this.PRprimitiveArray = (PrimitiveArray)null; this.PRobjectInfo = (ReadObjectInfo)null; this.PRisRegistered = false; this.PRmemberData = (object[])null; this.PRsi = (SerializationInfo)null; this.PRnullCount = 0; }
private void CreateBuffers <T>(PrimitiveArray <T> array) where T : struct { _buffers.Add(CreateBuffer(array.NullBitmapBuffer)); _buffers.Add(CreateBuffer(array.ValueBuffer)); }
static bool TryMakeColumnSpec(string name, feather.fbs.Type effectiveType, ref PrimitiveArray arrayDetails, string[] categoryLevels, DateTimePrecisionType precision, out ColumnSpec columnSpec, out string errorMessage) { var arrayOffset = arrayDetails.Offset; var arrayLength = arrayDetails.Length; var arrayNulls = arrayDetails.NullCount; var arrayEncoding = arrayDetails.Encoding; // TODO (Dictionary Encoding) if (arrayEncoding != feather.fbs.Encoding.PLAIN) { throw new NotImplementedException(); } // END TODO ColumnType type; bool isNullable; if (!TryGetType(effectiveType, ref arrayDetails, precision, out type, out isNullable, out errorMessage)) { columnSpec = default(ColumnSpec); return(false); } long numNullBytes = 0; if (isNullable) { numNullBytes = arrayLength / 8; if (arrayLength % 8 != 0) { numNullBytes++; } } // a naive reading of the spec suggests that the null bitmask should be // aligned based on the _type_ but it appears to always be long // aligned. // this may be a bug in the spec int nullPadding = 0; if ((numNullBytes % FeatherMagic.NULL_BITMASK_ALIGNMENT) != 0) { nullPadding = FeatherMagic.NULL_BITMASK_ALIGNMENT - (int)(numNullBytes % FeatherMagic.NULL_BITMASK_ALIGNMENT); } var nullOffset = isNullable ? (arrayOffset) : -1; var dataOffset = !isNullable ? (arrayOffset) : (nullOffset + numNullBytes + nullPadding); columnSpec = new ColumnSpec { Name = name, NullBitmaskOffset = nullOffset, DataOffset = dataOffset, Length = arrayLength, Type = type, CategoryLevels = categoryLevels, // only spin up this map if we've got categories to potentially map to CategoryEnumMap = categoryLevels != null ? new Dictionary <System.Type, CategoryEnumMapType>() : null }; errorMessage = null; return(true); }
static bool TryGetType(feather.fbs.Type effectiveType, ref PrimitiveArray array, DateTimePrecisionType precision, out ColumnType type, out bool isNullable, out string errorMessage) { isNullable = array.NullCount != 0; if (!isNullable) { switch (effectiveType) { case feather.fbs.Type.BINARY: type = ColumnType.Binary; errorMessage = null; return(true); case feather.fbs.Type.BOOL: type = ColumnType.Bool; errorMessage = null; return(true); case feather.fbs.Type.CATEGORY: type = ColumnType.Category; errorMessage = null; return(true); case feather.fbs.Type.DATE: type = ColumnType.Date; errorMessage = null; return(true); case feather.fbs.Type.DOUBLE: type = ColumnType.Double; errorMessage = null; return(true); case feather.fbs.Type.FLOAT: type = ColumnType.Float; errorMessage = null; return(true); case feather.fbs.Type.INT16: type = ColumnType.Int16; errorMessage = null; return(true); case feather.fbs.Type.INT32: type = ColumnType.Int32; errorMessage = null; return(true); case feather.fbs.Type.INT64: type = ColumnType.Int64; errorMessage = null; return(true); case feather.fbs.Type.INT8: type = ColumnType.Int8; errorMessage = null; return(true); case feather.fbs.Type.TIMESTAMP: switch (precision) { case DateTimePrecisionType.Microsecond: type = ColumnType.Timestamp_Microsecond; errorMessage = null; return(true); case DateTimePrecisionType.Millisecond: type = ColumnType.Timestamp_Millisecond; errorMessage = null; return(true); case DateTimePrecisionType.Nanosecond: type = ColumnType.Timestamp_Nanosecond; errorMessage = null; return(true); case DateTimePrecisionType.Second: type = ColumnType.Timestamp_Second; errorMessage = null; return(true); default: errorMessage = $"Unknown precision {precision}"; type = ColumnType.NONE; return(false); } case feather.fbs.Type.TIME: switch (precision) { case DateTimePrecisionType.Microsecond: type = ColumnType.Time_Microsecond; errorMessage = null; return(true); case DateTimePrecisionType.Millisecond: type = ColumnType.Time_Millisecond; errorMessage = null; return(true); case DateTimePrecisionType.Nanosecond: type = ColumnType.Time_Nanosecond; errorMessage = null; return(true); case DateTimePrecisionType.Second: type = ColumnType.Time_Second; errorMessage = null; return(true); default: errorMessage = $"Unknown precision {precision}"; type = ColumnType.NONE; return(false); } case feather.fbs.Type.UINT16: type = ColumnType.Uint16; errorMessage = null; return(true); case feather.fbs.Type.UINT32: type = ColumnType.Uint32; errorMessage = null; return(true); case feather.fbs.Type.UINT64: type = ColumnType.Uint64; errorMessage = null; return(true); case feather.fbs.Type.UINT8: type = ColumnType.Uint8; errorMessage = null; return(true); case feather.fbs.Type.UTF8: type = ColumnType.String; errorMessage = null; return(true); default: errorMessage = $"Unknown column type {array.Type}"; type = ColumnType.NONE; return(false); } } else { switch (effectiveType) { case feather.fbs.Type.BINARY: type = ColumnType.NullableBinary; errorMessage = null; return(true); case feather.fbs.Type.BOOL: type = ColumnType.NullableBool; errorMessage = null; return(true); case feather.fbs.Type.CATEGORY: type = ColumnType.NullableCategory; errorMessage = null; return(true); case feather.fbs.Type.DATE: type = ColumnType.NullableDate; errorMessage = null; return(true); case feather.fbs.Type.DOUBLE: type = ColumnType.NullableDouble; errorMessage = null; return(true); case feather.fbs.Type.FLOAT: type = ColumnType.NullableFloat; errorMessage = null; return(true); case feather.fbs.Type.INT16: type = ColumnType.NullableInt16; errorMessage = null; return(true); case feather.fbs.Type.INT32: type = ColumnType.NullableInt32; errorMessage = null; return(true); case feather.fbs.Type.INT64: type = ColumnType.NullableInt64; errorMessage = null; return(true); case feather.fbs.Type.INT8: type = ColumnType.NullableInt8; errorMessage = null; return(true); case feather.fbs.Type.TIMESTAMP: switch (precision) { case DateTimePrecisionType.Microsecond: type = ColumnType.NullableTimestamp_Microsecond; errorMessage = null; return(true); case DateTimePrecisionType.Millisecond: type = ColumnType.NullableTimestamp_Millisecond; errorMessage = null; return(true); case DateTimePrecisionType.Nanosecond: type = ColumnType.NullableTimestamp_Nanosecond; errorMessage = null; return(true); case DateTimePrecisionType.Second: type = ColumnType.NullableTimestamp_Second; errorMessage = null; return(true); default: errorMessage = $"Unknown precision {precision}"; type = ColumnType.NONE; return(false); } case feather.fbs.Type.TIME: switch (precision) { case DateTimePrecisionType.Microsecond: type = ColumnType.NullableTime_Microsecond; errorMessage = null; return(true); case DateTimePrecisionType.Millisecond: type = ColumnType.NullableTime_Millisecond; errorMessage = null; return(true); case DateTimePrecisionType.Nanosecond: type = ColumnType.NullableTime_Nanosecond; errorMessage = null; return(true); case DateTimePrecisionType.Second: type = ColumnType.NullableTime_Second; errorMessage = null; return(true); default: errorMessage = $"Unknown precision {precision}"; type = ColumnType.NONE; return(false); } case feather.fbs.Type.UINT16: type = ColumnType.NullableUint16; errorMessage = null; return(true); case feather.fbs.Type.UINT32: type = ColumnType.NullableUint32; errorMessage = null; return(true); case feather.fbs.Type.UINT64: type = ColumnType.NullableUint64; errorMessage = null; return(true); case feather.fbs.Type.UINT8: type = ColumnType.NullableUint8; errorMessage = null; return(true); case feather.fbs.Type.UTF8: type = ColumnType.NullableString; errorMessage = null; return(true); default: errorMessage = $"Unknown column type {array.Type}"; type = ColumnType.NONE; return(false); } } }
/// <summary> /// Wraps a <see cref="DataFrame"/> around an Arrow <see cref="RecordBatch"/> without copying data /// </summary> /// <param name="recordBatch"></param> /// <returns><see cref="DataFrame"/></returns> public static DataFrame FromArrowRecordBatch(RecordBatch recordBatch) { DataFrame ret = new DataFrame(); Apache.Arrow.Schema arrowSchema = recordBatch.Schema; int fieldIndex = 0; IEnumerable <IArrowArray> arrowArrays = recordBatch.Arrays; foreach (IArrowArray arrowArray in arrowArrays) { Field field = arrowSchema.GetFieldByIndex(fieldIndex); IArrowType fieldType = field.DataType; DataFrameColumn dataFrameColumn = null; switch (fieldType.TypeId) { case ArrowTypeId.Boolean: BooleanArray arrowBooleanArray = (BooleanArray)arrowArray; ReadOnlyMemory <byte> valueBuffer = arrowBooleanArray.ValueBuffer.Memory; ReadOnlyMemory <byte> nullBitMapBuffer = arrowBooleanArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <bool>(field.Name, valueBuffer, nullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Double: PrimitiveArray <double> arrowDoubleArray = (PrimitiveArray <double>)arrowArray; ReadOnlyMemory <byte> doubleValueBuffer = arrowDoubleArray.ValueBuffer.Memory; ReadOnlyMemory <byte> doubleNullBitMapBuffer = arrowDoubleArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <double>(field.Name, doubleValueBuffer, doubleNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Float: PrimitiveArray <float> arrowFloatArray = (PrimitiveArray <float>)arrowArray; ReadOnlyMemory <byte> floatValueBuffer = arrowFloatArray.ValueBuffer.Memory; ReadOnlyMemory <byte> floatNullBitMapBuffer = arrowFloatArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <float>(field.Name, floatValueBuffer, floatNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int8: PrimitiveArray <sbyte> arrowsbyteArray = (PrimitiveArray <sbyte>)arrowArray; ReadOnlyMemory <byte> sbyteValueBuffer = arrowsbyteArray.ValueBuffer.Memory; ReadOnlyMemory <byte> sbyteNullBitMapBuffer = arrowsbyteArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <sbyte>(field.Name, sbyteValueBuffer, sbyteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int16: PrimitiveArray <short> arrowshortArray = (PrimitiveArray <short>)arrowArray; ReadOnlyMemory <byte> shortValueBuffer = arrowshortArray.ValueBuffer.Memory; ReadOnlyMemory <byte> shortNullBitMapBuffer = arrowshortArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <short>(field.Name, shortValueBuffer, shortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int32: PrimitiveArray <int> arrowIntArray = (PrimitiveArray <int>)arrowArray; ReadOnlyMemory <byte> intValueBuffer = arrowIntArray.ValueBuffer.Memory; ReadOnlyMemory <byte> intNullBitMapBuffer = arrowIntArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <int>(field.Name, intValueBuffer, intNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int64: PrimitiveArray <long> arrowLongArray = (PrimitiveArray <long>)arrowArray; ReadOnlyMemory <byte> longValueBuffer = arrowLongArray.ValueBuffer.Memory; ReadOnlyMemory <byte> longNullBitMapBuffer = arrowLongArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <long>(field.Name, longValueBuffer, longNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.String: StringArray stringArray = (StringArray)arrowArray; ReadOnlyMemory <byte> dataMemory = stringArray.ValueBuffer.Memory; ReadOnlyMemory <byte> offsetsMemory = stringArray.ValueOffsetsBuffer.Memory; ReadOnlyMemory <byte> nullMemory = stringArray.NullBitmapBuffer.Memory; dataFrameColumn = new ArrowStringDataFrameColumn(field.Name, dataMemory, offsetsMemory, nullMemory, stringArray.Length, stringArray.NullCount); break; case ArrowTypeId.UInt8: PrimitiveArray <byte> arrowbyteArray = (PrimitiveArray <byte>)arrowArray; ReadOnlyMemory <byte> byteValueBuffer = arrowbyteArray.ValueBuffer.Memory; ReadOnlyMemory <byte> byteNullBitMapBuffer = arrowbyteArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <byte>(field.Name, byteValueBuffer, byteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.UInt16: PrimitiveArray <ushort> arrowUshortArray = (PrimitiveArray <ushort>)arrowArray; ReadOnlyMemory <byte> ushortValueBuffer = arrowUshortArray.ValueBuffer.Memory; ReadOnlyMemory <byte> ushortNullBitMapBuffer = arrowUshortArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <ushort>(field.Name, ushortValueBuffer, ushortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.UInt32: PrimitiveArray <uint> arrowUintArray = (PrimitiveArray <uint>)arrowArray; ReadOnlyMemory <byte> uintValueBuffer = arrowUintArray.ValueBuffer.Memory; ReadOnlyMemory <byte> uintNullBitMapBuffer = arrowUintArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <uint>(field.Name, uintValueBuffer, uintNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.UInt64: PrimitiveArray <ulong> arrowUlongArray = (PrimitiveArray <ulong>)arrowArray; ReadOnlyMemory <byte> ulongValueBuffer = arrowUlongArray.ValueBuffer.Memory; ReadOnlyMemory <byte> ulongNullBitMapBuffer = arrowUlongArray.NullBitmapBuffer.Memory; dataFrameColumn = new PrimitiveDataFrameColumn <ulong>(field.Name, ulongValueBuffer, ulongNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Decimal: case ArrowTypeId.Binary: case ArrowTypeId.Date32: case ArrowTypeId.Date64: case ArrowTypeId.Dictionary: case ArrowTypeId.FixedSizedBinary: case ArrowTypeId.HalfFloat: case ArrowTypeId.Interval: case ArrowTypeId.List: case ArrowTypeId.Map: case ArrowTypeId.Null: case ArrowTypeId.Struct: case ArrowTypeId.Time32: case ArrowTypeId.Time64: default: throw new NotImplementedException(nameof(fieldType.Name)); } ret.Columns.Insert(ret.Columns.Count, dataFrameColumn); fieldIndex++; } return(ret); }
public IteratorAnonymousInnerClass(PrimitiveArray <A, T> outerInstance) { this.outerInstance = outerInstance; size = outerInstance.Length(); }
private static void AppendDataFrameColumnFromArrowArray(Field field, IArrowArray arrowArray, DataFrame ret, string fieldNamePrefix = "") { IArrowType fieldType = field.DataType; DataFrameColumn dataFrameColumn = null; string fieldName = fieldNamePrefix + field.Name; switch (fieldType.TypeId) { case ArrowTypeId.Boolean: BooleanArray arrowBooleanArray = (BooleanArray)arrowArray; ReadOnlyMemory <byte> valueBuffer = arrowBooleanArray.ValueBuffer.Memory; ReadOnlyMemory <byte> nullBitMapBuffer = arrowBooleanArray.NullBitmapBuffer.Memory; dataFrameColumn = new BooleanDataFrameColumn(fieldName, valueBuffer, nullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Double: PrimitiveArray <double> arrowDoubleArray = (PrimitiveArray <double>)arrowArray; ReadOnlyMemory <byte> doubleValueBuffer = arrowDoubleArray.ValueBuffer.Memory; ReadOnlyMemory <byte> doubleNullBitMapBuffer = arrowDoubleArray.NullBitmapBuffer.Memory; dataFrameColumn = new DoubleDataFrameColumn(fieldName, doubleValueBuffer, doubleNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Float: PrimitiveArray <float> arrowFloatArray = (PrimitiveArray <float>)arrowArray; ReadOnlyMemory <byte> floatValueBuffer = arrowFloatArray.ValueBuffer.Memory; ReadOnlyMemory <byte> floatNullBitMapBuffer = arrowFloatArray.NullBitmapBuffer.Memory; dataFrameColumn = new SingleDataFrameColumn(fieldName, floatValueBuffer, floatNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int8: PrimitiveArray <sbyte> arrowsbyteArray = (PrimitiveArray <sbyte>)arrowArray; ReadOnlyMemory <byte> sbyteValueBuffer = arrowsbyteArray.ValueBuffer.Memory; ReadOnlyMemory <byte> sbyteNullBitMapBuffer = arrowsbyteArray.NullBitmapBuffer.Memory; dataFrameColumn = new SByteDataFrameColumn(fieldName, sbyteValueBuffer, sbyteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int16: PrimitiveArray <short> arrowshortArray = (PrimitiveArray <short>)arrowArray; ReadOnlyMemory <byte> shortValueBuffer = arrowshortArray.ValueBuffer.Memory; ReadOnlyMemory <byte> shortNullBitMapBuffer = arrowshortArray.NullBitmapBuffer.Memory; dataFrameColumn = new Int16DataFrameColumn(fieldName, shortValueBuffer, shortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int32: PrimitiveArray <int> arrowIntArray = (PrimitiveArray <int>)arrowArray; ReadOnlyMemory <byte> intValueBuffer = arrowIntArray.ValueBuffer.Memory; ReadOnlyMemory <byte> intNullBitMapBuffer = arrowIntArray.NullBitmapBuffer.Memory; dataFrameColumn = new Int32DataFrameColumn(fieldName, intValueBuffer, intNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Int64: PrimitiveArray <long> arrowLongArray = (PrimitiveArray <long>)arrowArray; ReadOnlyMemory <byte> longValueBuffer = arrowLongArray.ValueBuffer.Memory; ReadOnlyMemory <byte> longNullBitMapBuffer = arrowLongArray.NullBitmapBuffer.Memory; dataFrameColumn = new Int64DataFrameColumn(fieldName, longValueBuffer, longNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.String: StringArray stringArray = (StringArray)arrowArray; ReadOnlyMemory <byte> dataMemory = stringArray.ValueBuffer.Memory; ReadOnlyMemory <byte> offsetsMemory = stringArray.ValueOffsetsBuffer.Memory; ReadOnlyMemory <byte> nullMemory = stringArray.NullBitmapBuffer.Memory; dataFrameColumn = new ArrowStringDataFrameColumn(fieldName, dataMemory, offsetsMemory, nullMemory, stringArray.Length, stringArray.NullCount); break; case ArrowTypeId.UInt8: PrimitiveArray <byte> arrowbyteArray = (PrimitiveArray <byte>)arrowArray; ReadOnlyMemory <byte> byteValueBuffer = arrowbyteArray.ValueBuffer.Memory; ReadOnlyMemory <byte> byteNullBitMapBuffer = arrowbyteArray.NullBitmapBuffer.Memory; dataFrameColumn = new ByteDataFrameColumn(fieldName, byteValueBuffer, byteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.UInt16: PrimitiveArray <ushort> arrowUshortArray = (PrimitiveArray <ushort>)arrowArray; ReadOnlyMemory <byte> ushortValueBuffer = arrowUshortArray.ValueBuffer.Memory; ReadOnlyMemory <byte> ushortNullBitMapBuffer = arrowUshortArray.NullBitmapBuffer.Memory; dataFrameColumn = new UInt16DataFrameColumn(fieldName, ushortValueBuffer, ushortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.UInt32: PrimitiveArray <uint> arrowUintArray = (PrimitiveArray <uint>)arrowArray; ReadOnlyMemory <byte> uintValueBuffer = arrowUintArray.ValueBuffer.Memory; ReadOnlyMemory <byte> uintNullBitMapBuffer = arrowUintArray.NullBitmapBuffer.Memory; dataFrameColumn = new UInt32DataFrameColumn(fieldName, uintValueBuffer, uintNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.UInt64: PrimitiveArray <ulong> arrowUlongArray = (PrimitiveArray <ulong>)arrowArray; ReadOnlyMemory <byte> ulongValueBuffer = arrowUlongArray.ValueBuffer.Memory; ReadOnlyMemory <byte> ulongNullBitMapBuffer = arrowUlongArray.NullBitmapBuffer.Memory; dataFrameColumn = new UInt64DataFrameColumn(fieldName, ulongValueBuffer, ulongNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount); break; case ArrowTypeId.Struct: StructArray structArray = (StructArray)arrowArray; StructType structType = (StructType)field.DataType; IEnumerator <Field> fieldsEnumerator = structType.Fields.GetEnumerator(); IEnumerator <IArrowArray> structArrayEnumerator = structArray.Fields.GetEnumerator(); while (fieldsEnumerator.MoveNext() && structArrayEnumerator.MoveNext()) { AppendDataFrameColumnFromArrowArray(fieldsEnumerator.Current, structArrayEnumerator.Current, ret, field.Name + "_"); } break; case ArrowTypeId.Decimal: case ArrowTypeId.Binary: case ArrowTypeId.Date32: case ArrowTypeId.Date64: case ArrowTypeId.Dictionary: case ArrowTypeId.FixedSizedBinary: case ArrowTypeId.HalfFloat: case ArrowTypeId.Interval: case ArrowTypeId.List: case ArrowTypeId.Map: case ArrowTypeId.Null: case ArrowTypeId.Time32: case ArrowTypeId.Time64: default: throw new NotImplementedException(nameof(fieldType.Name)); } if (dataFrameColumn != null) { ret.Columns.Insert(ret.Columns.Count, dataFrameColumn); } }