public void IsValidValue()
{
const int totalValueCount = 8;
const byte nullBitmap = 0b_11110011;
var nullBitmapBuffer = new ArrowBuffer.Builder <byte>().Append(nullBitmap).Build();
var valueBuffer = new ArrowBuffer.Builder <long>().Append(0).Append(1).Append(4).Append(5).Append(6).Append(7).Append(8).Build();
//Check all offset and length
for (var offset = 0; offset < totalValueCount; offset++)
{
var nullCount = totalValueCount - offset - BitUtility.CountBits(nullBitmapBuffer.Span, offset);
for (var length = 1; length + offset < totalValueCount; length++)
{
TestIsValid(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
}
}
void TestIsValid(ArrowBuffer valueBuf, ArrowBuffer nullBitmapBuf, int length, int nullCount, int offset)
{
var array = new Int64Array(valueBuf, nullBitmapBuf, length, nullCount, offset);
for (var i = 0; i < length; i++)
{
if (BitUtility.GetBit(nullBitmap, i + offset))
{
Assert.True(array.IsValid(i));
}
else
{
Assert.False(array.IsValid(i));
}
}
}
}
private static double SumAllNumbers(RecordBatch recordBatch)
{
double sum = 0;
for (int k = 0; k < recordBatch.ColumnCount; k++)
{
var array = recordBatch.Arrays.ElementAt(k);
switch (recordBatch.Schema.GetFieldByIndex(k).DataType.TypeId)
{
case ArrowTypeId.Int64:
Int64Array int64Array = (Int64Array)array;
sum += Sum(int64Array);
break;
case ArrowTypeId.Double:
DoubleArray doubleArray = (DoubleArray)array;
sum += Sum(doubleArray);
break;
case ArrowTypeId.Decimal128:
Decimal128Array decimalArray = (Decimal128Array)array;
sum += Sum(decimalArray);
break;
}
}
return(sum);
}
private static long Sum(Int64Array int64Array)
{
long sum = 0;
ReadOnlySpan <long> values = int64Array.Values;
for (int valueIndex = 0; valueIndex < values.Length; valueIndex++)
{
sum += values[valueIndex];
}
return(sum);
}
public static Sample DecodeData(DataMessage dmsg)
{
var returnSample = new Sample(dmsg.DataType, null);
if (dmsg.DataType == DataMessage.Types.DataType.Image)
{
var msg = new ImageSamples();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.Audio)
{
var msg = new AudioSamples();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.String)
{
var msg = new StringSample();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.Float32Arr)
{
var msg = new Float32Array();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.Int64Arr)
{
var msg = new Int64Array();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else
{
Debug.Log("Unknown message");
return(null);
}
}
public async Task ArrowFlight()
{
var flightTicket = new FlightTicket(Sql);
var stream = flightClient.GetStream(flightTicket);
await foreach (var recordBatch in stream.ResponseStream)
{
for (int i = 0; i < recordBatch.ColumnCount; i++)
{
var array = recordBatch.Arrays.ElementAt(i);
switch (recordBatch.Schema.GetFieldByIndex(i).DataType.TypeId)
{
case ArrowTypeId.Int64:
Int64Array int64Array = (Int64Array)array;
Sum(int64Array);
break;
case ArrowTypeId.String:
StringArray stringArray = (StringArray)array;
for (int z = 0; z < stringArray.Length; z++)
{
stringArray.GetString(z);
}
break;
case ArrowTypeId.Timestamp:
TimestampArray timestampArray = (TimestampArray)array;
for (int z = 0; z < timestampArray.Length; z++)
{
timestampArray.GetTimestamp(z);
}
break;
}
}
//nop
}
}
public void Visit(Int64Array array) => CompareArrays(array);
public void Visit(Int64Array array) => CreateBuffers(array);
public static Func <int, T> GetGetter <T>(IArrowArray array)
{
if (array is null)
{
return(null);
}
// TODO: determine fastest way to read out a value from the array.
if (typeof(T) == typeof(bool))
{
var booleanArray = new BooleanArray(array.Data);
return((Func <int, T>)(object) new Func <int, bool>(
index => booleanArray.GetBoolean(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(bool?))
{
var booleanArray = new BooleanArray(array.Data);
return((Func <int, T>)(object) new Func <int, bool?>(booleanArray.GetBoolean));
}
if (typeof(T) == typeof(sbyte))
{
var int8Array = new Int8Array(array.Data);
return((Func <int, T>)(object) new Func <int, sbyte>(
index => int8Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(sbyte?))
{
var int8Array = new Int8Array(array.Data);
return((Func <int, T>)(object) new Func <int, sbyte?>(int8Array.GetValue));
}
if (typeof(T) == typeof(byte))
{
var uint8Array = new UInt8Array(array.Data);
return((Func <int, T>)(object) new Func <int, byte>(
index => uint8Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(byte?))
{
var uint8Array = new UInt8Array(array.Data);
return((Func <int, T>)(object) new Func <int, byte?>(uint8Array.GetValue));
}
if (typeof(T) == typeof(short))
{
var int16Array = new Int16Array(array.Data);
return((Func <int, T>)(object) new Func <int, short>(
index => int16Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(short?))
{
var int16Array = new Int16Array(array.Data);
return((Func <int, T>)(object) new Func <int, short?>(int16Array.GetValue));
}
if (typeof(T) == typeof(ushort))
{
var uint16Array = new UInt16Array(array.Data);
return((Func <int, T>)(object) new Func <int, ushort>(
index => uint16Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(ushort?))
{
var uint16Array = new UInt16Array(array.Data);
return((Func <int, T>)(object) new Func <int, ushort?>(uint16Array.GetValue));
}
if (typeof(T) == typeof(int))
{
var int32Array = new Int32Array(array.Data);
return((Func <int, T>)(object) new Func <int, int>(
index => int32Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(int?))
{
var int32Array = new Int32Array(array.Data);
return((Func <int, T>)(object) new Func <int, int?>(int32Array.GetValue));
}
if (typeof(T) == typeof(uint))
{
var uint32Array = new UInt32Array(array.Data);
return((Func <int, T>)(object) new Func <int, uint>(
index => uint32Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(uint?))
{
var uint32Array = new UInt32Array(array.Data);
return((Func <int, T>)(object) new Func <int, uint?>(uint32Array.GetValue));
}
if (typeof(T) == typeof(long))
{
var int64Array = new Int64Array(array.Data);
return((Func <int, T>)(object) new Func <int, long>(
index => int64Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(long?))
{
var int64Array = new Int64Array(array.Data);
return((Func <int, T>)(object) new Func <int, long?>(int64Array.GetValue));
}
if (typeof(T) == typeof(ulong))
{
var uint64Array = new UInt64Array(array.Data);
return((Func <int, T>)(object) new Func <int, ulong>(
index => uint64Array.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(ulong?))
{
var uint64Array = new UInt64Array(array.Data);
return((Func <int, T>)(object) new Func <int, ulong?>(uint64Array.GetValue));
}
if (typeof(T) == typeof(double))
{
var doubleArray = new DoubleArray(array.Data);
return((Func <int, T>)(object) new Func <int, double>(
index => doubleArray.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(double?))
{
var doubleArray = new DoubleArray(array.Data);
return((Func <int, T>)(object) new Func <int, double?>(doubleArray.GetValue));
}
if (typeof(T) == typeof(float))
{
var floatArray = new FloatArray(array.Data);
return((Func <int, T>)(object) new Func <int, float>(
index => floatArray.GetValue(index).GetValueOrDefault()));
}
if (typeof(T) == typeof(float?))
{
var floatArray = new FloatArray(array.Data);
return((Func <int, T>)(object) new Func <int, float?>(floatArray.GetValue));
}
if (typeof(T) == typeof(DateTime))
{
if (array.Data.DataType.TypeId == ArrowTypeId.Date32)
{
var date32Array = new Date32Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTime>(
index => date32Array.GetDate(index).GetValueOrDefault().DateTime));
}
else if (array.Data.DataType.TypeId == ArrowTypeId.Date64)
{
var date64Array = new Date64Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTime>(
index => date64Array.GetDate(index).GetValueOrDefault().DateTime));
}
}
if (typeof(T) == typeof(DateTime?))
{
if (array.Data.DataType.TypeId == ArrowTypeId.Date32)
{
var date32Array = new Date32Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTime?>(
index => date32Array.GetDate(index)?.DateTime));
}
else if (array.Data.DataType.TypeId == ArrowTypeId.Date64)
{
var date64Array = new Date64Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTime?>(
index => date64Array.GetDate(index)?.DateTime));
}
}
if (typeof(T) == typeof(DateTimeOffset))
{
if (array.Data.DataType.TypeId == ArrowTypeId.Date32)
{
var date32Array = new Date32Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTimeOffset>(
index => date32Array.GetDate(index).GetValueOrDefault()));
}
else if (array.Data.DataType.TypeId == ArrowTypeId.Date64)
{
var date64Array = new Date64Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTimeOffset>(
index => date64Array.GetDate(index).GetValueOrDefault()));
}
}
if (typeof(T) == typeof(DateTimeOffset?))
{
if (array.Data.DataType.TypeId == ArrowTypeId.Date32)
{
var date32Array = new Date32Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTimeOffset?>(
date32Array.GetDate));
}
else if (array.Data.DataType.TypeId == ArrowTypeId.Date64)
{
var date64Array = new Date64Array(array.Data);
return((Func <int, T>)(object) new Func <int, DateTimeOffset?>(
date64Array.GetDate));
}
}
if (typeof(T) == typeof(TimeSpan))
{
var timestampArray = new TimestampArray(array.Data);
return((Func <int, T>)(object) new Func <int, TimeSpan>(
index => timestampArray.GetTimestamp(index).GetValueOrDefault().TimeOfDay));
}
if (typeof(T) == typeof(TimeSpan?))
{
var timestampArray = new TimestampArray(array.Data);
return((Func <int, T>)(object) new Func <int, TimeSpan?>(
index => timestampArray.GetTimestamp(index)?.TimeOfDay));
}
if (typeof(T) == typeof(byte[]))
{
var binaryArray = new BinaryArray(array.Data);
return((Func <int, T>)(object) new Func <int, byte[]>(
// TODO: how to avoid this allocation/copy?
index => binaryArray.GetBytes(index).ToArray()));
}
if (typeof(T) == typeof(string))
{
var stringArray = new StringArray(array.Data);
return((Func <int, T>)(object) new Func <int, string>(
index => stringArray.GetString(index)));
}
// It's something else we don't yet support.
switch (array.Data.DataType.TypeId)
{
case ArrowTypeId.Decimal:
case ArrowTypeId.Dictionary:
case ArrowTypeId.FixedSizedBinary:
case ArrowTypeId.HalfFloat:
case ArrowTypeId.Interval:
case ArrowTypeId.List:
case ArrowTypeId.Map:
case ArrowTypeId.Null:
case ArrowTypeId.Struct:
case ArrowTypeId.Time32:
case ArrowTypeId.Time64:
case ArrowTypeId.Union:
default:
// TODO: support additional types?
throw new NotSupportedException(
$"Not supported array type: {array.Data.DataType.TypeId}");
}
}