private static ArrayData LoadVariableField(Field field, Flatbuf.FieldNode fieldNode, Flatbuf.RecordBatch recordBatch, FlatBuffers.ByteBuffer bodyData, ref int bufferIndex) { var nullBitmapBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault(); var offsetBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault(); var valueBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault(); ArrowBuffer nullArrowBuffer = BuildArrowBuffer(bodyData, nullBitmapBuffer); ArrowBuffer offsetArrowBuffer = BuildArrowBuffer(bodyData, offsetBuffer); ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, valueBuffer); var fieldLength = (int)fieldNode.Length; var fieldNullCount = (int)fieldNode.NullCount; if (fieldLength < 0) { throw new InvalidDataException("Field length must be >= 0"); // TODO: Localize exception message } if (fieldNullCount < 0) { throw new InvalidDataException("Null count length must be >= 0"); //TODO: Localize exception message } var arrowBuff = new[] { nullArrowBuffer, offsetArrowBuffer, valueArrowBuffer }; return(new ArrayData(field.DataType, fieldLength, fieldNullCount, 0, arrowBuff)); }
private List <IArrowArray> BuildArrays( Schema schema, ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength); if (recordBatchMessage.NodesLength == 0) { return(arrays); } var recordBatchEnumerator = new RecordBatchEnumerator(in recordBatchMessage); int schemaFieldIndex = 0; do { Field field = schema.GetFieldByIndex(schemaFieldIndex++); Flatbuf.FieldNode fieldNode = recordBatchEnumerator.CurrentNode; ArrayData arrayData = field.DataType.IsFixedPrimitive() ? LoadPrimitiveField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer) : LoadVariableField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer); arrays.Add(ArrowArrayFactory.BuildArray(arrayData)); } while (recordBatchEnumerator.MoveNextNode()); return(arrays); }
protected RecordBatch CreateArrowObjectFromMessage( Flatbuf.Message message, ByteBuffer bodyByteBuffer, IMemoryOwner <byte> memoryOwner) { switch (message.HeaderType) { case Flatbuf.MessageHeader.Schema: // TODO: Read schema and verify equality? break; case Flatbuf.MessageHeader.DictionaryBatch: // TODO: not supported currently Debug.WriteLine("Dictionaries are not yet supported."); break; case Flatbuf.MessageHeader.RecordBatch: Flatbuf.RecordBatch rb = message.Header <Flatbuf.RecordBatch>().Value; List <IArrowArray> arrays = BuildArrays(Schema, bodyByteBuffer, rb); return(new RecordBatch(Schema, memoryOwner, arrays, (int)rb.Length)); default: // NOTE: Skip unsupported message type Debug.WriteLine($"Skipping unsupported message type '{message.HeaderType}'"); break; } return(null); }
private List <IArrowArray> BuildArrays( Schema schema, ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength); int bufferIndex = 0; for (var n = 0; n < recordBatchMessage.NodesLength; n++) { Field field = schema.GetFieldByIndex(n); Flatbuf.FieldNode fieldNode = recordBatchMessage.Nodes(n).GetValueOrDefault(); ArrayData arrayData = field.DataType.IsFixedPrimitive() ? LoadPrimitiveField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex) : LoadVariableField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex); arrays.Add(ArrowArrayFactory.BuildArray(arrayData)); } return(arrays); }
protected static IEnumerable <IArrowArray> BuildArrays(Schema schema, FlatBuffers.ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { var arrays = new List <ArrayData>(); var bufferIndex = 0; for (var n = 0; n < recordBatchMessage.NodesLength; n++) { var field = schema.GetFieldByIndex(n); var fieldNode = recordBatchMessage.Nodes(n).GetValueOrDefault(); if (field.DataType.IsFixedPrimitive()) { arrays.Add(LoadPrimitiveField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex)); } else { arrays.Add(LoadVariableField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex)); } } return(arrays.Select(ArrowArrayFactory.BuildArray)); }