private static ArrayData LoadVariableField(Field field,
Flatbuf.FieldNode fieldNode,
Flatbuf.RecordBatch recordBatch,
FlatBuffers.ByteBuffer bodyData,
ref int bufferIndex)
{
var nullBitmapBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault();
var offsetBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault();
var valueBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault();
ArrowBuffer nullArrowBuffer = BuildArrowBuffer(bodyData, nullBitmapBuffer);
ArrowBuffer offsetArrowBuffer = BuildArrowBuffer(bodyData, offsetBuffer);
ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, valueBuffer);
var fieldLength = (int)fieldNode.Length;
var fieldNullCount = (int)fieldNode.NullCount;
if (fieldLength < 0)
{
throw new InvalidDataException("Field length must be >= 0"); // TODO: Localize exception message
}
if (fieldNullCount < 0)
{
throw new InvalidDataException("Null count length must be >= 0"); //TODO: Localize exception message
}
var arrowBuff = new[] { nullArrowBuffer, offsetArrowBuffer, valueArrowBuffer };
return(new ArrayData(field.DataType, fieldLength, fieldNullCount, 0, arrowBuff));
}
private List <IArrowArray> BuildArrays(
Schema schema,
ByteBuffer messageBuffer,
Flatbuf.RecordBatch recordBatchMessage)
{
var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength);
if (recordBatchMessage.NodesLength == 0)
{
return(arrays);
}
var recordBatchEnumerator = new RecordBatchEnumerator(in recordBatchMessage);
int schemaFieldIndex = 0;
do
{
Field field = schema.GetFieldByIndex(schemaFieldIndex++);
Flatbuf.FieldNode fieldNode = recordBatchEnumerator.CurrentNode;
ArrayData arrayData = field.DataType.IsFixedPrimitive()
? LoadPrimitiveField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer)
: LoadVariableField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer);
arrays.Add(ArrowArrayFactory.BuildArray(arrayData));
} while (recordBatchEnumerator.MoveNextNode());
return(arrays);
}
protected RecordBatch CreateArrowObjectFromMessage(
Flatbuf.Message message, ByteBuffer bodyByteBuffer, IMemoryOwner <byte> memoryOwner)
{
switch (message.HeaderType)
{
case Flatbuf.MessageHeader.Schema:
// TODO: Read schema and verify equality?
break;
case Flatbuf.MessageHeader.DictionaryBatch:
// TODO: not supported currently
Debug.WriteLine("Dictionaries are not yet supported.");
break;
case Flatbuf.MessageHeader.RecordBatch:
Flatbuf.RecordBatch rb = message.Header <Flatbuf.RecordBatch>().Value;
List <IArrowArray> arrays = BuildArrays(Schema, bodyByteBuffer, rb);
return(new RecordBatch(Schema, memoryOwner, arrays, (int)rb.Length));
default:
// NOTE: Skip unsupported message type
Debug.WriteLine($"Skipping unsupported message type '{message.HeaderType}'");
break;
}
return(null);
}
private List <IArrowArray> BuildArrays(
Schema schema,
ByteBuffer messageBuffer,
Flatbuf.RecordBatch recordBatchMessage)
{
var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength);
int bufferIndex = 0;
for (var n = 0; n < recordBatchMessage.NodesLength; n++)
{
Field field = schema.GetFieldByIndex(n);
Flatbuf.FieldNode fieldNode = recordBatchMessage.Nodes(n).GetValueOrDefault();
ArrayData arrayData = field.DataType.IsFixedPrimitive() ?
LoadPrimitiveField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex) :
LoadVariableField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex);
arrays.Add(ArrowArrayFactory.BuildArray(arrayData));
}
return(arrays);
}
protected static IEnumerable <IArrowArray> BuildArrays(Schema schema,
FlatBuffers.ByteBuffer messageBuffer,
Flatbuf.RecordBatch recordBatchMessage)
{
var arrays = new List <ArrayData>();
var bufferIndex = 0;
for (var n = 0; n < recordBatchMessage.NodesLength; n++)
{
var field = schema.GetFieldByIndex(n);
var fieldNode = recordBatchMessage.Nodes(n).GetValueOrDefault();
if (field.DataType.IsFixedPrimitive())
{
arrays.Add(LoadPrimitiveField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex));
}
else
{
arrays.Add(LoadVariableField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex));
}
}
return(arrays.Select(ArrowArrayFactory.BuildArray));
}
