private void CompareArrays <T>(PrimitiveArray <T> actualArray)
where T : struct, IEquatable <T>
{
Assert.IsAssignableFrom <PrimitiveArray <T> >(_expectedArray);
PrimitiveArray <T> expectedArray = (PrimitiveArray <T>)_expectedArray;
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
Assert.Equal(expectedArray.Offset, actualArray.Offset);
if (expectedArray.NullCount > 0)
{
Assert.True(expectedArray.NullBitmapBuffer.Span.SequenceEqual(actualArray.NullBitmapBuffer.Span));
}
else
{
// expectedArray may have passed in a null bitmap. DataFrame might have populated it with Length set bits
Assert.Equal(0, expectedArray.NullCount);
Assert.Equal(0, actualArray.NullCount);
for (int i = 0; i < actualArray.Length; i++)
{
Assert.True(actualArray.IsValid(i));
}
}
Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length)));
}
private void BindPrimitiveArray(HbmPrimitiveArray arrayMapping, PrimitiveArray model, string prefix, string path, System.Type containingType, IDictionary <string, MetaAttribute> inheritedMetas)
{
BindCollection(arrayMapping, model, prefix, path, containingType, inheritedMetas);
var element = arrayMapping.ElementRelationship as HbmElement;
if (element != null)
{
string typeName;
var typeAttribute = element.Type;
if (typeAttribute != null)
{
typeName = typeAttribute.name;
}
else
{
throw new MappingException("type for <element> was not defined");
}
IType type = TypeFactory.HeuristicType(typeName, null);
if (type == null)
{
throw new MappingException("could not interpret type: " + typeName);
}
model.ElementClassName = type.ReturnedClass.AssemblyQualifiedName;
}
}
private void CompareArrays <T>(PrimitiveArray <T> actualArray)
where T : struct, IEquatable <T>
{
Assert.IsAssignableFrom <PrimitiveArray <T> >(_expectedArray);
PrimitiveArray <T> expectedArray = (PrimitiveArray <T>)_expectedArray;
actualArray.Data.DataType.Accept(_arrayTypeComparer);
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
Assert.Equal(expectedArray.Offset, actualArray.Offset);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, actualArray.NullBitmapBuffer);
if (_strictCompare)
{
Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length)));
}
else
{
for (int i = 0; i < expectedArray.Length; i++)
{
Assert.Equal(expectedArray.GetValue(i), actualArray.GetValue(i));
}
}
}
private Mapping.Collection CreatePrimitiveArray(XmlNode node, string prefix, string path,
PersistentClass owner, System.Type containingType, IDictionary <string, MetaAttribute> inheritedMetas)
{
PrimitiveArray array = new PrimitiveArray(owner);
BindArray(node, array, prefix, path, containingType, inheritedMetas);
return(array);
}
private Mapping.Collection CreatePrimitiveArray(HbmPrimitiveArray primitiveArrayMapping, string prefix, string path,
PersistentClass owner, System.Type containingType, IDictionary <string, MetaAttribute> inheritedMetas)
{
var array = new PrimitiveArray(owner);
BindPrimitiveArray(primitiveArrayMapping, array, prefix, path, containingType, inheritedMetas);
AddPrimitiveArraySecondPass(primitiveArrayMapping, array, inheritedMetas);
return(array);
}
public bool Equals(PrimitiveArray <ValueType> other)
{
if (other == null)
{
return(false);
}
return((m_ArrayElements == null) ? (other.m_ArrayElements == null) : m_ArrayElements.SequenceEqual(other.m_ArrayElements));
}
internal override void NewBatch(IArrowArray arrowArray)
{
if (arrowArray is PrimitiveArray <T> primitiveArray)
{
Array = primitiveArray;
}
else
{
throw new ArgumentException("Expected primitive array", nameof(arrowArray));
}
}
private void CompareArrays <T>(PrimitiveArray <T> actualArray)
where T : struct, IEquatable <T>
{
Assert.IsAssignableFrom <PrimitiveArray <T> >(_expectedArray);
PrimitiveArray <T> expectedArray = (PrimitiveArray <T>)_expectedArray;
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
Assert.Equal(expectedArray.Offset, actualArray.Offset);
Assert.True(expectedArray.NullBitmapBuffer.Span.SequenceEqual(actualArray.NullBitmapBuffer.Span));
Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length)));
}
internal void Init()
{
this.PRparseTypeEnum = InternalParseTypeE.Empty;
this.PRobjectTypeEnum = InternalObjectTypeE.Empty;
this.PRarrayTypeEnum = InternalArrayTypeE.Empty;
this.PRmemberTypeEnum = InternalMemberTypeE.Empty;
this.PRmemberValueEnum = InternalMemberValueE.Empty;
this.PRobjectPositionEnum = InternalObjectPositionE.Empty;
this.PRname = (string)null;
this.PRvalue = (string)null;
this.PRkeyDt = (string)null;
this.PRdtType = (Type)null;
this.PRdtTypeCode = InternalPrimitiveTypeE.Invalid;
this.PRisEnum = false;
this.PRobjectId = 0L;
this.PRidRef = 0L;
this.PRarrayElementTypeString = (string)null;
this.PRarrayElementType = (Type)null;
this.PRisArrayVariant = false;
this.PRarrayElementTypeCode = InternalPrimitiveTypeE.Invalid;
this.PRrank = 0;
this.PRlengthA = (int[])null;
this.PRpositionA = (int[])null;
this.PRlowerBoundA = (int[])null;
this.PRupperBoundA = (int[])null;
this.PRindexMap = (int[])null;
this.PRmemberIndex = 0;
this.PRlinearlength = 0;
this.PRrectangularMap = (int[])null;
this.PRisLowerBound = false;
this.PRtopId = 0L;
this.PRheaderId = 0L;
this.PRisValueTypeFixup = false;
this.PRnewObj = (object)null;
this.PRobjectA = (object[])null;
this.PRprimitiveArray = (PrimitiveArray)null;
this.PRobjectInfo = (ReadObjectInfo)null;
this.PRisRegistered = false;
this.PRmemberData = (object[])null;
this.PRsi = (SerializationInfo)null;
this.PRnullCount = 0;
}
private void CreateBuffers <T>(PrimitiveArray <T> array)
where T : struct
{
_buffers.Add(CreateBuffer(array.NullBitmapBuffer));
_buffers.Add(CreateBuffer(array.ValueBuffer));
}
static bool TryMakeColumnSpec(string name, feather.fbs.Type effectiveType, ref PrimitiveArray arrayDetails, string[] categoryLevels, DateTimePrecisionType precision, out ColumnSpec columnSpec, out string errorMessage)
{
var arrayOffset = arrayDetails.Offset;
var arrayLength = arrayDetails.Length;
var arrayNulls = arrayDetails.NullCount;
var arrayEncoding = arrayDetails.Encoding;
// TODO (Dictionary Encoding)
if (arrayEncoding != feather.fbs.Encoding.PLAIN)
{
throw new NotImplementedException();
}
// END TODO
ColumnType type;
bool isNullable;
if (!TryGetType(effectiveType, ref arrayDetails, precision, out type, out isNullable, out errorMessage))
{
columnSpec = default(ColumnSpec);
return(false);
}
long numNullBytes = 0;
if (isNullable)
{
numNullBytes = arrayLength / 8;
if (arrayLength % 8 != 0)
{
numNullBytes++;
}
}
// a naive reading of the spec suggests that the null bitmask should be
// aligned based on the _type_ but it appears to always be long
// aligned.
// this may be a bug in the spec
int nullPadding = 0;
if ((numNullBytes % FeatherMagic.NULL_BITMASK_ALIGNMENT) != 0)
{
nullPadding = FeatherMagic.NULL_BITMASK_ALIGNMENT - (int)(numNullBytes % FeatherMagic.NULL_BITMASK_ALIGNMENT);
}
var nullOffset = isNullable ? (arrayOffset) : -1;
var dataOffset = !isNullable ? (arrayOffset) : (nullOffset + numNullBytes + nullPadding);
columnSpec =
new ColumnSpec
{
Name = name,
NullBitmaskOffset = nullOffset,
DataOffset = dataOffset,
Length = arrayLength,
Type = type,
CategoryLevels = categoryLevels,
// only spin up this map if we've got categories to potentially map to
CategoryEnumMap = categoryLevels != null ? new Dictionary <System.Type, CategoryEnumMapType>() : null
};
errorMessage = null;
return(true);
}
static bool TryGetType(feather.fbs.Type effectiveType, ref PrimitiveArray array, DateTimePrecisionType precision, out ColumnType type, out bool isNullable, out string errorMessage)
{
isNullable = array.NullCount != 0;
if (!isNullable)
{
switch (effectiveType)
{
case feather.fbs.Type.BINARY:
type = ColumnType.Binary;
errorMessage = null;
return(true);
case feather.fbs.Type.BOOL:
type = ColumnType.Bool;
errorMessage = null;
return(true);
case feather.fbs.Type.CATEGORY:
type = ColumnType.Category;
errorMessage = null;
return(true);
case feather.fbs.Type.DATE:
type = ColumnType.Date;
errorMessage = null;
return(true);
case feather.fbs.Type.DOUBLE:
type = ColumnType.Double;
errorMessage = null;
return(true);
case feather.fbs.Type.FLOAT:
type = ColumnType.Float;
errorMessage = null;
return(true);
case feather.fbs.Type.INT16:
type = ColumnType.Int16;
errorMessage = null;
return(true);
case feather.fbs.Type.INT32:
type = ColumnType.Int32;
errorMessage = null;
return(true);
case feather.fbs.Type.INT64:
type = ColumnType.Int64;
errorMessage = null;
return(true);
case feather.fbs.Type.INT8:
type = ColumnType.Int8;
errorMessage = null;
return(true);
case feather.fbs.Type.TIMESTAMP:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.Timestamp_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.Timestamp_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.Timestamp_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.Timestamp_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.TIME:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.Time_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.Time_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.Time_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.Time_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.UINT16:
type = ColumnType.Uint16;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT32:
type = ColumnType.Uint32;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT64:
type = ColumnType.Uint64;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT8:
type = ColumnType.Uint8;
errorMessage = null;
return(true);
case feather.fbs.Type.UTF8:
type = ColumnType.String;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown column type {array.Type}";
type = ColumnType.NONE;
return(false);
}
}
else
{
switch (effectiveType)
{
case feather.fbs.Type.BINARY:
type = ColumnType.NullableBinary;
errorMessage = null;
return(true);
case feather.fbs.Type.BOOL:
type = ColumnType.NullableBool;
errorMessage = null;
return(true);
case feather.fbs.Type.CATEGORY:
type = ColumnType.NullableCategory;
errorMessage = null;
return(true);
case feather.fbs.Type.DATE:
type = ColumnType.NullableDate;
errorMessage = null;
return(true);
case feather.fbs.Type.DOUBLE:
type = ColumnType.NullableDouble;
errorMessage = null;
return(true);
case feather.fbs.Type.FLOAT:
type = ColumnType.NullableFloat;
errorMessage = null;
return(true);
case feather.fbs.Type.INT16:
type = ColumnType.NullableInt16;
errorMessage = null;
return(true);
case feather.fbs.Type.INT32:
type = ColumnType.NullableInt32;
errorMessage = null;
return(true);
case feather.fbs.Type.INT64:
type = ColumnType.NullableInt64;
errorMessage = null;
return(true);
case feather.fbs.Type.INT8:
type = ColumnType.NullableInt8;
errorMessage = null;
return(true);
case feather.fbs.Type.TIMESTAMP:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.NullableTimestamp_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.NullableTimestamp_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.NullableTimestamp_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.NullableTimestamp_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.TIME:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.NullableTime_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.NullableTime_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.NullableTime_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.NullableTime_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.UINT16:
type = ColumnType.NullableUint16;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT32:
type = ColumnType.NullableUint32;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT64:
type = ColumnType.NullableUint64;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT8:
type = ColumnType.NullableUint8;
errorMessage = null;
return(true);
case feather.fbs.Type.UTF8:
type = ColumnType.NullableString;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown column type {array.Type}";
type = ColumnType.NONE;
return(false);
}
}
}
/// <summary>
/// Wraps a <see cref="DataFrame"/> around an Arrow <see cref="RecordBatch"/> without copying data
/// </summary>
/// <param name="recordBatch"></param>
/// <returns><see cref="DataFrame"/></returns>
public static DataFrame FromArrowRecordBatch(RecordBatch recordBatch)
{
DataFrame ret = new DataFrame();
Apache.Arrow.Schema arrowSchema = recordBatch.Schema;
int fieldIndex = 0;
IEnumerable <IArrowArray> arrowArrays = recordBatch.Arrays;
foreach (IArrowArray arrowArray in arrowArrays)
{
Field field = arrowSchema.GetFieldByIndex(fieldIndex);
IArrowType fieldType = field.DataType;
DataFrameColumn dataFrameColumn = null;
switch (fieldType.TypeId)
{
case ArrowTypeId.Boolean:
BooleanArray arrowBooleanArray = (BooleanArray)arrowArray;
ReadOnlyMemory <byte> valueBuffer = arrowBooleanArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> nullBitMapBuffer = arrowBooleanArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <bool>(field.Name, valueBuffer, nullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Double:
PrimitiveArray <double> arrowDoubleArray = (PrimitiveArray <double>)arrowArray;
ReadOnlyMemory <byte> doubleValueBuffer = arrowDoubleArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> doubleNullBitMapBuffer = arrowDoubleArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <double>(field.Name, doubleValueBuffer, doubleNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Float:
PrimitiveArray <float> arrowFloatArray = (PrimitiveArray <float>)arrowArray;
ReadOnlyMemory <byte> floatValueBuffer = arrowFloatArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> floatNullBitMapBuffer = arrowFloatArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <float>(field.Name, floatValueBuffer, floatNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int8:
PrimitiveArray <sbyte> arrowsbyteArray = (PrimitiveArray <sbyte>)arrowArray;
ReadOnlyMemory <byte> sbyteValueBuffer = arrowsbyteArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> sbyteNullBitMapBuffer = arrowsbyteArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <sbyte>(field.Name, sbyteValueBuffer, sbyteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int16:
PrimitiveArray <short> arrowshortArray = (PrimitiveArray <short>)arrowArray;
ReadOnlyMemory <byte> shortValueBuffer = arrowshortArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> shortNullBitMapBuffer = arrowshortArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <short>(field.Name, shortValueBuffer, shortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int32:
PrimitiveArray <int> arrowIntArray = (PrimitiveArray <int>)arrowArray;
ReadOnlyMemory <byte> intValueBuffer = arrowIntArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> intNullBitMapBuffer = arrowIntArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <int>(field.Name, intValueBuffer, intNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int64:
PrimitiveArray <long> arrowLongArray = (PrimitiveArray <long>)arrowArray;
ReadOnlyMemory <byte> longValueBuffer = arrowLongArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> longNullBitMapBuffer = arrowLongArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <long>(field.Name, longValueBuffer, longNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.String:
StringArray stringArray = (StringArray)arrowArray;
ReadOnlyMemory <byte> dataMemory = stringArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> offsetsMemory = stringArray.ValueOffsetsBuffer.Memory;
ReadOnlyMemory <byte> nullMemory = stringArray.NullBitmapBuffer.Memory;
dataFrameColumn = new ArrowStringDataFrameColumn(field.Name, dataMemory, offsetsMemory, nullMemory, stringArray.Length, stringArray.NullCount);
break;
case ArrowTypeId.UInt8:
PrimitiveArray <byte> arrowbyteArray = (PrimitiveArray <byte>)arrowArray;
ReadOnlyMemory <byte> byteValueBuffer = arrowbyteArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> byteNullBitMapBuffer = arrowbyteArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <byte>(field.Name, byteValueBuffer, byteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.UInt16:
PrimitiveArray <ushort> arrowUshortArray = (PrimitiveArray <ushort>)arrowArray;
ReadOnlyMemory <byte> ushortValueBuffer = arrowUshortArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> ushortNullBitMapBuffer = arrowUshortArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <ushort>(field.Name, ushortValueBuffer, ushortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.UInt32:
PrimitiveArray <uint> arrowUintArray = (PrimitiveArray <uint>)arrowArray;
ReadOnlyMemory <byte> uintValueBuffer = arrowUintArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> uintNullBitMapBuffer = arrowUintArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <uint>(field.Name, uintValueBuffer, uintNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.UInt64:
PrimitiveArray <ulong> arrowUlongArray = (PrimitiveArray <ulong>)arrowArray;
ReadOnlyMemory <byte> ulongValueBuffer = arrowUlongArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> ulongNullBitMapBuffer = arrowUlongArray.NullBitmapBuffer.Memory;
dataFrameColumn = new PrimitiveDataFrameColumn <ulong>(field.Name, ulongValueBuffer, ulongNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Decimal:
case ArrowTypeId.Binary:
case ArrowTypeId.Date32:
case ArrowTypeId.Date64:
case ArrowTypeId.Dictionary:
case ArrowTypeId.FixedSizedBinary:
case ArrowTypeId.HalfFloat:
case ArrowTypeId.Interval:
case ArrowTypeId.List:
case ArrowTypeId.Map:
case ArrowTypeId.Null:
case ArrowTypeId.Struct:
case ArrowTypeId.Time32:
case ArrowTypeId.Time64:
default:
throw new NotImplementedException(nameof(fieldType.Name));
}
ret.Columns.Insert(ret.Columns.Count, dataFrameColumn);
fieldIndex++;
}
return(ret);
}
public IteratorAnonymousInnerClass(PrimitiveArray <A, T> outerInstance)
{
this.outerInstance = outerInstance;
size = outerInstance.Length();
}
private static void AppendDataFrameColumnFromArrowArray(Field field, IArrowArray arrowArray, DataFrame ret, string fieldNamePrefix = "")
{
IArrowType fieldType = field.DataType;
DataFrameColumn dataFrameColumn = null;
string fieldName = fieldNamePrefix + field.Name;
switch (fieldType.TypeId)
{
case ArrowTypeId.Boolean:
BooleanArray arrowBooleanArray = (BooleanArray)arrowArray;
ReadOnlyMemory <byte> valueBuffer = arrowBooleanArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> nullBitMapBuffer = arrowBooleanArray.NullBitmapBuffer.Memory;
dataFrameColumn = new BooleanDataFrameColumn(fieldName, valueBuffer, nullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Double:
PrimitiveArray <double> arrowDoubleArray = (PrimitiveArray <double>)arrowArray;
ReadOnlyMemory <byte> doubleValueBuffer = arrowDoubleArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> doubleNullBitMapBuffer = arrowDoubleArray.NullBitmapBuffer.Memory;
dataFrameColumn = new DoubleDataFrameColumn(fieldName, doubleValueBuffer, doubleNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Float:
PrimitiveArray <float> arrowFloatArray = (PrimitiveArray <float>)arrowArray;
ReadOnlyMemory <byte> floatValueBuffer = arrowFloatArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> floatNullBitMapBuffer = arrowFloatArray.NullBitmapBuffer.Memory;
dataFrameColumn = new SingleDataFrameColumn(fieldName, floatValueBuffer, floatNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int8:
PrimitiveArray <sbyte> arrowsbyteArray = (PrimitiveArray <sbyte>)arrowArray;
ReadOnlyMemory <byte> sbyteValueBuffer = arrowsbyteArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> sbyteNullBitMapBuffer = arrowsbyteArray.NullBitmapBuffer.Memory;
dataFrameColumn = new SByteDataFrameColumn(fieldName, sbyteValueBuffer, sbyteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int16:
PrimitiveArray <short> arrowshortArray = (PrimitiveArray <short>)arrowArray;
ReadOnlyMemory <byte> shortValueBuffer = arrowshortArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> shortNullBitMapBuffer = arrowshortArray.NullBitmapBuffer.Memory;
dataFrameColumn = new Int16DataFrameColumn(fieldName, shortValueBuffer, shortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int32:
PrimitiveArray <int> arrowIntArray = (PrimitiveArray <int>)arrowArray;
ReadOnlyMemory <byte> intValueBuffer = arrowIntArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> intNullBitMapBuffer = arrowIntArray.NullBitmapBuffer.Memory;
dataFrameColumn = new Int32DataFrameColumn(fieldName, intValueBuffer, intNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Int64:
PrimitiveArray <long> arrowLongArray = (PrimitiveArray <long>)arrowArray;
ReadOnlyMemory <byte> longValueBuffer = arrowLongArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> longNullBitMapBuffer = arrowLongArray.NullBitmapBuffer.Memory;
dataFrameColumn = new Int64DataFrameColumn(fieldName, longValueBuffer, longNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.String:
StringArray stringArray = (StringArray)arrowArray;
ReadOnlyMemory <byte> dataMemory = stringArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> offsetsMemory = stringArray.ValueOffsetsBuffer.Memory;
ReadOnlyMemory <byte> nullMemory = stringArray.NullBitmapBuffer.Memory;
dataFrameColumn = new ArrowStringDataFrameColumn(fieldName, dataMemory, offsetsMemory, nullMemory, stringArray.Length, stringArray.NullCount);
break;
case ArrowTypeId.UInt8:
PrimitiveArray <byte> arrowbyteArray = (PrimitiveArray <byte>)arrowArray;
ReadOnlyMemory <byte> byteValueBuffer = arrowbyteArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> byteNullBitMapBuffer = arrowbyteArray.NullBitmapBuffer.Memory;
dataFrameColumn = new ByteDataFrameColumn(fieldName, byteValueBuffer, byteNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.UInt16:
PrimitiveArray <ushort> arrowUshortArray = (PrimitiveArray <ushort>)arrowArray;
ReadOnlyMemory <byte> ushortValueBuffer = arrowUshortArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> ushortNullBitMapBuffer = arrowUshortArray.NullBitmapBuffer.Memory;
dataFrameColumn = new UInt16DataFrameColumn(fieldName, ushortValueBuffer, ushortNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.UInt32:
PrimitiveArray <uint> arrowUintArray = (PrimitiveArray <uint>)arrowArray;
ReadOnlyMemory <byte> uintValueBuffer = arrowUintArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> uintNullBitMapBuffer = arrowUintArray.NullBitmapBuffer.Memory;
dataFrameColumn = new UInt32DataFrameColumn(fieldName, uintValueBuffer, uintNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.UInt64:
PrimitiveArray <ulong> arrowUlongArray = (PrimitiveArray <ulong>)arrowArray;
ReadOnlyMemory <byte> ulongValueBuffer = arrowUlongArray.ValueBuffer.Memory;
ReadOnlyMemory <byte> ulongNullBitMapBuffer = arrowUlongArray.NullBitmapBuffer.Memory;
dataFrameColumn = new UInt64DataFrameColumn(fieldName, ulongValueBuffer, ulongNullBitMapBuffer, arrowArray.Length, arrowArray.NullCount);
break;
case ArrowTypeId.Struct:
StructArray structArray = (StructArray)arrowArray;
StructType structType = (StructType)field.DataType;
IEnumerator <Field> fieldsEnumerator = structType.Fields.GetEnumerator();
IEnumerator <IArrowArray> structArrayEnumerator = structArray.Fields.GetEnumerator();
while (fieldsEnumerator.MoveNext() && structArrayEnumerator.MoveNext())
{
AppendDataFrameColumnFromArrowArray(fieldsEnumerator.Current, structArrayEnumerator.Current, ret, field.Name + "_");
}
break;
case ArrowTypeId.Decimal:
case ArrowTypeId.Binary:
case ArrowTypeId.Date32:
case ArrowTypeId.Date64:
case ArrowTypeId.Dictionary:
case ArrowTypeId.FixedSizedBinary:
case ArrowTypeId.HalfFloat:
case ArrowTypeId.Interval:
case ArrowTypeId.List:
case ArrowTypeId.Map:
case ArrowTypeId.Null:
case ArrowTypeId.Time32:
case ArrowTypeId.Time64:
default:
throw new NotImplementedException(nameof(fieldType.Name));
}
if (dataFrameColumn != null)
{
ret.Columns.Insert(ret.Columns.Count, dataFrameColumn);
}
}
