private void ReadSchema()
{
if (HasReadSchema)
{
return;
}
// Figure out length of schema
int schemaMessageLength = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Span.Slice(_bufferPosition));
_bufferPosition += sizeof(int);
ByteBuffer schemaBuffer = CreateByteBuffer(_buffer.Slice(_bufferPosition));
Schema = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemaBuffer));
_bufferPosition += schemaMessageLength;
}
protected virtual void ReadSchema()
{
if (HasReadSchema)
{
return;
}
// Figure out length of schema
int schemaMessageLength = ReadMessageLength(throwOnFullRead: true);
ArrayPool <byte> .Shared.RentReturn(schemaMessageLength, buff =>
{
int bytesRead = BaseStream.ReadFullBuffer(buff);
EnsureFullRead(buff, bytesRead);
FlatBuffers.ByteBuffer schemabb = CreateByteBuffer(buff);
Schema = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemabb), ref _dictionaryMemo);
});
}
protected virtual async ValueTask ReadSchemaAsync()
{
if (HasReadSchema)
{
return;
}
// Figure out length of schema
int schemaMessageLength = await ReadMessageLengthAsync(throwOnFullRead : true)
.ConfigureAwait(false);
await ArrayPool <byte> .Shared.RentReturnAsync(schemaMessageLength, async (buff) =>
{
// Read in schema
int bytesRead = await BaseStream.ReadFullBufferAsync(buff).ConfigureAwait(false);
EnsureFullRead(buff, bytesRead);
FlatBuffers.ByteBuffer schemabb = CreateByteBuffer(buff);
Schema = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemabb), ref _dictionaryMemo);
}).ConfigureAwait(false);
}
protected virtual async Task <Schema> ReadSchemaAsync()
{
if (HasReadSchema)
{
return(Schema);
}
byte[] buff = null;
try
{
// Figure out length of schema
buff = Buffers.Rent(4);
await BaseStream.ReadAsync(buff, 0, 4);
var schemaMessageLength = BitConverter.ToInt32(buff, 0);
Buffers.Return(buff);
// Allocate byte array for schema flat buffer
buff = Buffers.Rent(schemaMessageLength);
var schemabb = new FlatBuffers.ByteBuffer(buff);
// Read in schema
await BaseStream.ReadAsync(buff, 0, schemaMessageLength);
Schema = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemabb));
return(Schema);
}
finally
{
if (buff != null)
{
Buffers.Return(buff);
}
}
}
private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field)
{
switch (field.TypeType)
{
case Flatbuf.Type.Int:
var intMetaData = field.Type <Flatbuf.Int>().Value;
return(MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned));
case Flatbuf.Type.FloatingPoint:
var floatingPointTypeMetadta = field.Type <Flatbuf.FloatingPoint>().Value;
switch (floatingPointTypeMetadta.Precision)
{
case Flatbuf.Precision.SINGLE:
return(Types.FloatType.Default);
case Flatbuf.Precision.DOUBLE:
return(Types.DoubleType.Default);
case Flatbuf.Precision.HALF:
return(Types.HalfFloatType.Default);
default:
throw new InvalidDataException("Unsupported floating point precision");
}
case Flatbuf.Type.Bool:
return(new Types.BooleanType());
case Flatbuf.Type.Decimal:
var decMeta = field.Type <Flatbuf.Decimal>().Value;
return(new Types.DecimalType(decMeta.Precision, decMeta.Scale));
case Flatbuf.Type.Date:
var dateMeta = field.Type <Flatbuf.Date>().Value;
switch (dateMeta.Unit)
{
case Flatbuf.DateUnit.DAY:
return(Types.Date32Type.Default);
case Flatbuf.DateUnit.MILLISECOND:
return(Types.Date64Type.Default);
default:
throw new InvalidDataException("Unsupported date unit");
}
case Flatbuf.Type.Time:
var timeMeta = field.Type <Flatbuf.Time>().Value;
switch (timeMeta.BitWidth)
{
case 32:
return(new Types.Time32Type(timeMeta.Unit.ToArrow()));
case 64:
return(new Types.Time64Type(timeMeta.Unit.ToArrow()));
default:
throw new InvalidDataException("Unsupported time bit width");
}
case Flatbuf.Type.Timestamp:
var timestampTypeMetadata = field.Type <Flatbuf.Timestamp>().Value;
var unit = timestampTypeMetadata.Unit.ToArrow();
var timezone = timestampTypeMetadata.Timezone;
return(new Types.TimestampType(unit, timezone));
case Flatbuf.Type.Interval:
var intervalMetadata = field.Type <Flatbuf.Interval>().Value;
return(new Types.IntervalType(intervalMetadata.Unit.ToArrow()));
case Flatbuf.Type.Utf8:
return(new Types.StringType());
case Flatbuf.Type.Binary:
return(Types.BinaryType.Default);
default:
throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
}
}
private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field, Field[] childFields = null)
{
switch (field.TypeType)
{
case Flatbuf.Type.Int:
Flatbuf.Int intMetaData = field.Type <Flatbuf.Int>().Value;
return(MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned));
case Flatbuf.Type.FloatingPoint:
Flatbuf.FloatingPoint floatingPointTypeMetadata = field.Type <Flatbuf.FloatingPoint>().Value;
switch (floatingPointTypeMetadata.Precision)
{
case Flatbuf.Precision.SINGLE:
return(Types.FloatType.Default);
case Flatbuf.Precision.DOUBLE:
return(Types.DoubleType.Default);
case Flatbuf.Precision.HALF:
return(Types.HalfFloatType.Default);
default:
throw new InvalidDataException("Unsupported floating point precision");
}
case Flatbuf.Type.Bool:
return(new Types.BooleanType());
case Flatbuf.Type.Decimal:
Flatbuf.Decimal decMeta = field.Type <Flatbuf.Decimal>().Value;
return(new Types.DecimalType(decMeta.Precision, decMeta.Scale));
case Flatbuf.Type.Date:
Flatbuf.Date dateMeta = field.Type <Flatbuf.Date>().Value;
switch (dateMeta.Unit)
{
case Flatbuf.DateUnit.DAY:
return(Types.Date32Type.Default);
case Flatbuf.DateUnit.MILLISECOND:
return(Types.Date64Type.Default);
default:
throw new InvalidDataException("Unsupported date unit");
}
case Flatbuf.Type.Time:
Flatbuf.Time timeMeta = field.Type <Flatbuf.Time>().Value;
switch (timeMeta.BitWidth)
{
case 32:
return(new Types.Time32Type(timeMeta.Unit.ToArrow()));
case 64:
return(new Types.Time64Type(timeMeta.Unit.ToArrow()));
default:
throw new InvalidDataException("Unsupported time bit width");
}
case Flatbuf.Type.Timestamp:
Flatbuf.Timestamp timestampTypeMetadata = field.Type <Flatbuf.Timestamp>().Value;
Types.TimeUnit unit = timestampTypeMetadata.Unit.ToArrow();
string timezone = timestampTypeMetadata.Timezone;
return(new Types.TimestampType(unit, timezone));
case Flatbuf.Type.Interval:
Flatbuf.Interval intervalMetadata = field.Type <Flatbuf.Interval>().Value;
return(new Types.IntervalType(intervalMetadata.Unit.ToArrow()));
case Flatbuf.Type.Utf8:
return(new Types.StringType());
case Flatbuf.Type.Binary:
return(Types.BinaryType.Default);
case Flatbuf.Type.List:
if (childFields == null || childFields.Length != 1)
{
throw new InvalidDataException($"List type must have exactly one child.");
}
return(new Types.ListType(childFields[0]));
case Flatbuf.Type.Struct_:
Debug.Assert(childFields != null);
return(new Types.StructType(childFields));
default:
throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
}
}