public void Visit(ListType type)
{
CheckData(type, 2);
ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
ArrowBuffer offsetBuffer = ConcatenateOffsetBuffer();
ArrayData child = Concatenate(SelectChildren(0), _allocator);
Result = new ArrayData(type, _totalLength, _totalNullCount, 0, new ArrowBuffer[] { validityBuffer, offsetBuffer }, new[] { child });
}
private void ConcatenateVariableBinaryArrayData(IArrowType type)
{
CheckData(type, 3);
ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
ArrowBuffer offsetBuffer = ConcatenateOffsetBuffer();
ArrowBuffer valueBuffer = ConcatenateVariableBinaryValueBuffer();
Result = new ArrayData(type, _totalLength, _totalNullCount, 0, new ArrowBuffer[] { validityBuffer, offsetBuffer, valueBuffer });
}
public TArray Build(MemoryAllocator allocator = default)
{
ValueOffsets.Append(Offset);
var data = new ArrayData(DataType, ValueOffsets.Length - 1, 0, 0,
new[] { ArrowBuffer.Empty, ValueOffsets.Build(allocator), ValueBuffer.Build(allocator) });
return(Build(data));
}
public static void EnsureBufferCount(this ArrayData data, int count)
{
if (data.Buffers.Length != count)
{
// TODO: Use localizable string resource
throw new ArgumentException(
$"Buffer count <{data.Buffers.Length}> must be at least <{count}>",
nameof(data.Buffers.Length));
}
}
public static void EnsureDataType(this ArrayData data, ArrowTypeId id)
{
if (data.DataType.TypeId != id)
{
// TODO: Use localizable string resource
throw new ArgumentException(
$"Specified array type <{data.DataType.TypeId}> does not match expected type(s) <{id}>",
nameof(data.DataType.TypeId));
}
}
public void Visit(StructType type)
{
CheckData(type, 1);
List <ArrayData> children = new List <ArrayData>(type.Fields.Count);
for (int i = 0; i < type.Fields.Count; i++)
{
children.Add(Concatenate(SelectChildren(i), _allocator));
}
Result = new ArrayData(type, _arrayDataList[0].Length, _arrayDataList[0].NullCount, 0, _arrayDataList[0].Buffers, children);
}
public ArrayData(
IArrowType dataType,
int length, int nullCount = 0, int offset = 0,
ArrowBuffer[] buffers = null, ArrayData[] children = null, ArrayData dictionary = null)
{
DataType = dataType ?? NullType.Default;
Length = length;
NullCount = nullCount;
Offset = offset;
Buffers = buffers;
Children = children;
Dictionary = dictionary;
}
public ArrayData(
IArrowType dataType,
int length, int nullCount = 0, int offset = 0,
IEnumerable <ArrowBuffer> buffers = null, IEnumerable <ArrayData> children = null, ArrayData dictionary = null)
{
DataType = dataType ?? NullType.Default;
Length = length;
NullCount = nullCount;
Offset = offset;
Buffers = buffers?.ToArray();
Children = children?.ToArray();
Dictionary = dictionary;
}
public TArray Build(MemoryAllocator allocator = default)
{
ValueOffsets.Append(Offset);
ArrowBuffer validityBuffer = NullCount > 0
? ValidityBuffer.Build(allocator).ValueBuffer
: ArrowBuffer.Empty;
var data = new ArrayData(DataType, ValueOffsets.Length - 1, NullCount, 0,
new[] { validityBuffer, ValueOffsets.Build(allocator), ValueBuffer.Build(allocator) });
return(Build(data));
}
public Array Slice(int offset, int length)
{
if (offset > Length)
{
throw new ArgumentException($"Offset {offset} cannot be greater than Length {Length} for Array.Slice");
}
length = Math.Min(Data.Length - offset, length);
offset += Data.Offset;
ArrayData newData = Data.Slice(offset, length);
return(ArrowArrayFactory.BuildArray(newData) as Array);
}
/// <summary>
/// Build an Arrow array from the appended contents so far.
/// </summary>
/// <param name="allocator">Optional memory allocator.</param>
/// <returns>Returns an array of type <typeparamref name="TArray"/>.</returns>
public TArray Build(MemoryAllocator allocator = default)
{
var bufs = new[]
{
NullCount > 0 ? ValidityBuffer.Build(allocator) : ArrowBuffer.Empty,
ValueOffsets.Build(allocator),
ValueBuffer.Build(allocator),
};
var data = new ArrayData(
DataType,
length: Length,
NullCount,
offset: 0,
bufs);
return(Build(data));
}
public DictionaryArray(ArrayData data) : base(data)
{
data.EnsureBufferCount(2);
data.EnsureDataType(ArrowTypeId.Dictionary);
if (data.Dictionary == null)
{
throw new ArgumentException($"{nameof(data.Dictionary)} must not be null");
}
var dicType = (DictionaryType)data.DataType;
data.Dictionary.EnsureDataType(dicType.ValueType.TypeId);
var indicesData = new ArrayData(dicType.IndexType, data.Length, data.NullCount, data.Offset, data.Buffers, data.Children);
Indices = ArrowArrayFactory.BuildArray(indicesData);
Dictionary = ArrowArrayFactory.BuildArray(data.Dictionary);
}
public static void EnsureDataType(this ArrayData data, params ArrowTypeId[] ids)
{
var valid = true;
foreach (var id in ids)
{
if (data.DataType.TypeId != id)
{
valid = false;
}
}
if (!valid)
{
// TODO: Use localizable string resource
throw new ArgumentException(
$"Specified array type <{data.DataType.TypeId}> does not match expected type(s) <{string.Join(",", ids)}>",
nameof(data.DataType.TypeId));
}
}
public DoubleArray(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.Double);
}
protected abstract TArray Build(ArrayData data);
public BinaryArray(ArrowTypeId typeId, ArrayData data)
: base(data)
{
data.EnsureDataType(typeId);
data.EnsureBufferCount(3);
}
protected override BinaryArray Build(ArrayData data)
{
return(new BinaryArray(data));
}
public UInt16Array(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.UInt16);
}
public Int8Array(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.Int8);
}
public FloatArray(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.Float);
}
public StringArray(ArrayData data)
: base(ArrowTypeId.String, data)
{
}
private ListArray(ArrayData data, IArrowArray values) : base(data)
{
data.EnsureBufferCount(2);
data.EnsureDataType(ArrowTypeId.List);
Values = values;
}
public TimestampArray(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.Timestamp);
}
public Time32Array(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.Time32);
}
public static IArrowArray BuildArray(ArrayData data)
{
switch (data.DataType.TypeId)
{
case ArrowTypeId.Boolean:
return(new BooleanArray(data));
case ArrowTypeId.UInt8:
return(new UInt8Array(data));
case ArrowTypeId.Int8:
return(new Int8Array(data));
case ArrowTypeId.UInt16:
return(new UInt16Array(data));
case ArrowTypeId.Int16:
return(new Int16Array(data));
case ArrowTypeId.UInt32:
return(new UInt32Array(data));
case ArrowTypeId.Int32:
return(new Int32Array(data));
case ArrowTypeId.UInt64:
return(new UInt64Array(data));
case ArrowTypeId.Int64:
return(new Int64Array(data));
case ArrowTypeId.Float:
return(new FloatArray(data));
case ArrowTypeId.Double:
return(new DoubleArray(data));
case ArrowTypeId.String:
return(new StringArray(data));
case ArrowTypeId.FixedSizedBinary:
return(new FixedSizeBinaryArray(data));
case ArrowTypeId.Binary:
return(new BinaryArray(data));
case ArrowTypeId.Timestamp:
return(new TimestampArray(data));
case ArrowTypeId.List:
return(new ListArray(data));
case ArrowTypeId.Struct:
return(new StructArray(data));
case ArrowTypeId.Union:
return(new UnionArray(data));
case ArrowTypeId.Date64:
return(new Date64Array(data));
case ArrowTypeId.Date32:
return(new Date32Array(data));
case ArrowTypeId.Time32:
return(new Time32Array(data));
case ArrowTypeId.Time64:
return(new Time64Array(data));
case ArrowTypeId.Decimal128:
return(new Decimal128Array(data));
case ArrowTypeId.Decimal256:
return(new Decimal256Array(data));
case ArrowTypeId.Dictionary:
return(new DictionaryArray(data));
case ArrowTypeId.HalfFloat:
case ArrowTypeId.Interval:
case ArrowTypeId.Map:
default:
throw new NotSupportedException($"An ArrowArray cannot be built for type {data.DataType.TypeId}.");
}
}
public Date64Array(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.Date64);
}
protected Array(ArrayData data)
{
Data = data ?? throw new ArgumentNullException(nameof(data));
}
protected PrimitiveArray(ArrayData data)
: base(data)
{
data.EnsureBufferCount(2);
}
public BooleanArray(ArrayData data)
: base(data)
{
data.EnsureDataType(ArrowTypeId.Boolean);
}
public ListArray(ArrayData data)
: this(data, ArrowArrayFactory.BuildArray(data.Children[0]))
{
}
