private void ReadDictionaryBatch(Flatbuf.DictionaryBatch dictionaryBatch, ByteBuffer bodyByteBuffer, IMemoryOwner <byte> memoryOwner)
{
long id = dictionaryBatch.Id;
IArrowType valueType = DictionaryMemo.GetDictionaryType(id);
Flatbuf.RecordBatch?recordBatch = dictionaryBatch.Data;
if (!recordBatch.HasValue)
{
throw new InvalidDataException("Dictionary must contain RecordBatch");
}
Field valueField = new Field("dummy", valueType, true);
var schema = new Schema(new[] { valueField }, default);
IList <IArrowArray> arrays = BuildArrays(schema, bodyByteBuffer, recordBatch.Value);
if (arrays.Count != 1)
{
throw new InvalidDataException("Dictionary record batch must contain only one field");
}
if (dictionaryBatch.IsDelta)
{
throw new NotImplementedException("Dictionary delta is not supported yet");
}
else
{
DictionaryMemo.AddOrReplaceDictionary(id, arrays[0]);
}
}
/// <summary>
/// Create a record batch or dictionary batch from Flatbuf.Message.
/// </summary>
/// <remarks>
/// This method adds data to _dictionaryMemo and returns null when the message type is DictionaryBatch.
/// </remarks>>
/// <returns>
/// The record batch when the message type is RecordBatch.
/// Null when the message type is not RecordBatch.
/// </returns>
protected RecordBatch CreateArrowObjectFromMessage(
Flatbuf.Message message, ByteBuffer bodyByteBuffer, IMemoryOwner <byte> memoryOwner)
{
switch (message.HeaderType)
{
case Flatbuf.MessageHeader.Schema:
// TODO: Read schema and verify equality?
break;
case Flatbuf.MessageHeader.DictionaryBatch:
Flatbuf.DictionaryBatch dictionaryBatch = message.Header <Flatbuf.DictionaryBatch>().Value;
ReadDictionaryBatch(dictionaryBatch, bodyByteBuffer, memoryOwner);
break;
case Flatbuf.MessageHeader.RecordBatch:
Flatbuf.RecordBatch rb = message.Header <Flatbuf.RecordBatch>().Value;
List <IArrowArray> arrays = BuildArrays(Schema, bodyByteBuffer, rb);
return(new RecordBatch(Schema, memoryOwner, arrays, (int)rb.Length));
default:
// NOTE: Skip unsupported message type
Debug.WriteLine($"Skipping unsupported message type '{message.HeaderType}'");
break;
}
return(null);
}
