private List <IArrowArray> BuildArrays( Schema schema, ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength); if (recordBatchMessage.NodesLength == 0) { return(arrays); } var recordBatchEnumerator = new RecordBatchEnumerator(in recordBatchMessage); int schemaFieldIndex = 0; do { Field field = schema.GetFieldByIndex(schemaFieldIndex++); Flatbuf.FieldNode fieldNode = recordBatchEnumerator.CurrentNode; ArrayData arrayData = field.DataType.IsFixedPrimitive() ? LoadPrimitiveField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer) : LoadVariableField(ref recordBatchEnumerator, field, in fieldNode, messageBuffer); arrays.Add(ArrowArrayFactory.BuildArray(arrayData)); } while (recordBatchEnumerator.MoveNextNode()); return(arrays); }
private static ArrayData LoadVariableField(Field field, Flatbuf.FieldNode fieldNode, Flatbuf.RecordBatch recordBatch, FlatBuffers.ByteBuffer bodyData, ref int bufferIndex) { var nullBitmapBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault(); var offsetBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault(); var valueBuffer = recordBatch.Buffers(bufferIndex++).GetValueOrDefault(); ArrowBuffer nullArrowBuffer = BuildArrowBuffer(bodyData, nullBitmapBuffer); ArrowBuffer offsetArrowBuffer = BuildArrowBuffer(bodyData, offsetBuffer); ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, valueBuffer); var fieldLength = (int)fieldNode.Length; var fieldNullCount = (int)fieldNode.NullCount; if (fieldLength < 0) { throw new InvalidDataException("Field length must be >= 0"); // TODO: Localize exception message } if (fieldNullCount < 0) { throw new InvalidDataException("Null count length must be >= 0"); //TODO: Localize exception message } var arrowBuff = new[] { nullArrowBuffer, offsetArrowBuffer, valueArrowBuffer }; return(new ArrayData(field.DataType, fieldLength, fieldNullCount, 0, arrowBuff)); }
private List <IArrowArray> BuildArrays( Schema schema, ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { var arrays = new List <IArrowArray>(recordBatchMessage.NodesLength); int bufferIndex = 0; for (var n = 0; n < recordBatchMessage.NodesLength; n++) { Field field = schema.GetFieldByIndex(n); Flatbuf.FieldNode fieldNode = recordBatchMessage.Nodes(n).GetValueOrDefault(); ArrayData arrayData = field.DataType.IsFixedPrimitive() ? LoadPrimitiveField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex) : LoadVariableField(field, fieldNode, recordBatchMessage, messageBuffer, ref bufferIndex); arrays.Add(ArrowArrayFactory.BuildArray(arrayData)); } return(arrays); }