internal static Schema GetSchema(Flatbuf.Schema schema)
{
var schemaBuilder = new Schema.Builder();
for (int i = 0; i < schema.FieldsLength; i++)
{
Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
schemaBuilder.Field(FieldFromFlatbuffer(field));
}
return(schemaBuilder.Build());
}
private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField)
{
Field[] childFields = null;
if (flatbufField.ChildrenLength > 0)
{
childFields = new Field[flatbufField.ChildrenLength];
for (int i = 0; i < flatbufField.ChildrenLength; i++)
{
Flatbuf.Field?childFlatbufField = flatbufField.Children(i);
childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value);
}
}
return(new Field(flatbufField.Name, GetFieldArrowType(flatbufField, childFields), flatbufField.Nullable));
}
internal static Schema GetSchema(Flatbuf.Schema schema)
{
var schemaBuilder = new Schema.Builder();
for (int i = 0; i < schema.FieldsLength; i++)
{
Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
schemaBuilder.Field(
new Field(field.Name, GetFieldArrowType(field), field.Nullable));
}
return(schemaBuilder.Build());
}
private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField)
{
Field[] childFields = flatbufField.ChildrenLength > 0 ? new Field[flatbufField.ChildrenLength] : null;
for (int i = 0; i < flatbufField.ChildrenLength; i++)
{
Flatbuf.Field?childFlatbufField = flatbufField.Children(i);
childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value);
}
Dictionary <string, string> metadata = flatbufField.CustomMetadataLength > 0 ? new Dictionary <string, string>() : null;
for (int i = 0; i < flatbufField.CustomMetadataLength; i++)
{
Flatbuf.KeyValue keyValue = flatbufField.CustomMetadata(i).GetValueOrDefault();
metadata[keyValue.Key] = keyValue.Value;
}
return(new Field(flatbufField.Name, GetFieldArrowType(flatbufField, childFields), flatbufField.Nullable, metadata, copyCollections: false));
}
private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField, ref DictionaryMemo dictionaryMemo)
{
Field[] childFields = flatbufField.ChildrenLength > 0 ? new Field[flatbufField.ChildrenLength] : null;
for (int i = 0; i < flatbufField.ChildrenLength; i++)
{
Flatbuf.Field?childFlatbufField = flatbufField.Children(i);
childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value, ref dictionaryMemo);
}
Flatbuf.DictionaryEncoding?dictionaryEncoding = flatbufField.Dictionary;
IArrowType type = GetFieldArrowType(flatbufField, childFields);
if (dictionaryEncoding.HasValue)
{
Flatbuf.Int?indexTypeAsInt = dictionaryEncoding.Value.IndexType;
IArrowType indexType = indexTypeAsInt.HasValue ?
GetNumberType(indexTypeAsInt.Value.BitWidth, indexTypeAsInt.Value.IsSigned) :
GetNumberType(Int32Type.Default.BitWidth, Int32Type.Default.IsSigned);
type = new DictionaryType(indexType, type, dictionaryEncoding.Value.IsOrdered);
}
Dictionary <string, string> metadata = flatbufField.CustomMetadataLength > 0 ? new Dictionary <string, string>() : null;
for (int i = 0; i < flatbufField.CustomMetadataLength; i++)
{
Flatbuf.KeyValue keyValue = flatbufField.CustomMetadata(i).GetValueOrDefault();
metadata[keyValue.Key] = keyValue.Value;
}
var arrowField = new Field(flatbufField.Name, type, flatbufField.Nullable, metadata, copyCollections: false);
if (dictionaryEncoding.HasValue)
{
dictionaryMemo ??= new DictionaryMemo();
dictionaryMemo.AddField(dictionaryEncoding.Value.Id, arrowField);
}
return(arrowField);
}
internal static Schema GetSchema(Flatbuf.Schema schema, ref DictionaryMemo dictionaryMemo)
{
List <Field> fields = new List <Field>();
for (int i = 0; i < schema.FieldsLength; i++)
{
Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
fields.Add(FieldFromFlatbuffer(field, ref dictionaryMemo));
}
Dictionary <string, string> metadata = schema.CustomMetadataLength > 0 ? new Dictionary <string, string>() : null;
for (int i = 0; i < schema.CustomMetadataLength; i++)
{
Flatbuf.KeyValue keyValue = schema.CustomMetadata(i).GetValueOrDefault();
metadata[keyValue.Key] = keyValue.Value;
}
return(new Schema(fields, metadata, copyCollections: false));
}
private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field)
{
switch (field.TypeType)
{
case Flatbuf.Type.Int:
var intMetaData = field.Type <Flatbuf.Int>().Value;
return(MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned));
case Flatbuf.Type.FloatingPoint:
var floatingPointTypeMetadta = field.Type <Flatbuf.FloatingPoint>().Value;
switch (floatingPointTypeMetadta.Precision)
{
case Flatbuf.Precision.SINGLE:
return(Types.FloatType.Default);
case Flatbuf.Precision.DOUBLE:
return(Types.DoubleType.Default);
case Flatbuf.Precision.HALF:
return(Types.HalfFloatType.Default);
default:
throw new InvalidDataException("Unsupported floating point precision");
}
case Flatbuf.Type.Bool:
return(new Types.BooleanType());
case Flatbuf.Type.Decimal:
var decMeta = field.Type <Flatbuf.Decimal>().Value;
return(new Types.DecimalType(decMeta.Precision, decMeta.Scale));
case Flatbuf.Type.Date:
var dateMeta = field.Type <Flatbuf.Date>().Value;
switch (dateMeta.Unit)
{
case Flatbuf.DateUnit.DAY:
return(Types.Date32Type.Default);
case Flatbuf.DateUnit.MILLISECOND:
return(Types.Date64Type.Default);
default:
throw new InvalidDataException("Unsupported date unit");
}
case Flatbuf.Type.Time:
var timeMeta = field.Type <Flatbuf.Time>().Value;
switch (timeMeta.BitWidth)
{
case 32:
return(new Types.Time32Type(timeMeta.Unit.ToArrow()));
case 64:
return(new Types.Time64Type(timeMeta.Unit.ToArrow()));
default:
throw new InvalidDataException("Unsupported time bit width");
}
case Flatbuf.Type.Timestamp:
var timestampTypeMetadata = field.Type <Flatbuf.Timestamp>().Value;
var unit = timestampTypeMetadata.Unit.ToArrow();
var timezone = timestampTypeMetadata.Timezone;
return(new Types.TimestampType(unit, timezone));
case Flatbuf.Type.Interval:
var intervalMetadata = field.Type <Flatbuf.Interval>().Value;
return(new Types.IntervalType(intervalMetadata.Unit.ToArrow()));
case Flatbuf.Type.Utf8:
return(new Types.StringType());
case Flatbuf.Type.Binary:
return(Types.BinaryType.Default);
default:
throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
}
}
private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field, Field[] childFields = null)
{
switch (field.TypeType)
{
case Flatbuf.Type.Int:
Flatbuf.Int intMetaData = field.Type <Flatbuf.Int>().Value;
return(MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned));
case Flatbuf.Type.FloatingPoint:
Flatbuf.FloatingPoint floatingPointTypeMetadata = field.Type <Flatbuf.FloatingPoint>().Value;
switch (floatingPointTypeMetadata.Precision)
{
case Flatbuf.Precision.SINGLE:
return(Types.FloatType.Default);
case Flatbuf.Precision.DOUBLE:
return(Types.DoubleType.Default);
case Flatbuf.Precision.HALF:
return(Types.HalfFloatType.Default);
default:
throw new InvalidDataException("Unsupported floating point precision");
}
case Flatbuf.Type.Bool:
return(new Types.BooleanType());
case Flatbuf.Type.Decimal:
Flatbuf.Decimal decMeta = field.Type <Flatbuf.Decimal>().Value;
return(new Types.DecimalType(decMeta.Precision, decMeta.Scale));
case Flatbuf.Type.Date:
Flatbuf.Date dateMeta = field.Type <Flatbuf.Date>().Value;
switch (dateMeta.Unit)
{
case Flatbuf.DateUnit.DAY:
return(Types.Date32Type.Default);
case Flatbuf.DateUnit.MILLISECOND:
return(Types.Date64Type.Default);
default:
throw new InvalidDataException("Unsupported date unit");
}
case Flatbuf.Type.Time:
Flatbuf.Time timeMeta = field.Type <Flatbuf.Time>().Value;
switch (timeMeta.BitWidth)
{
case 32:
return(new Types.Time32Type(timeMeta.Unit.ToArrow()));
case 64:
return(new Types.Time64Type(timeMeta.Unit.ToArrow()));
default:
throw new InvalidDataException("Unsupported time bit width");
}
case Flatbuf.Type.Timestamp:
Flatbuf.Timestamp timestampTypeMetadata = field.Type <Flatbuf.Timestamp>().Value;
Types.TimeUnit unit = timestampTypeMetadata.Unit.ToArrow();
string timezone = timestampTypeMetadata.Timezone;
return(new Types.TimestampType(unit, timezone));
case Flatbuf.Type.Interval:
Flatbuf.Interval intervalMetadata = field.Type <Flatbuf.Interval>().Value;
return(new Types.IntervalType(intervalMetadata.Unit.ToArrow()));
case Flatbuf.Type.Utf8:
return(new Types.StringType());
case Flatbuf.Type.Binary:
return(Types.BinaryType.Default);
case Flatbuf.Type.List:
if (childFields == null || childFields.Length != 1)
{
throw new InvalidDataException($"List type must have exactly one child.");
}
return(new Types.ListType(childFields[0]));
case Flatbuf.Type.Struct_:
Debug.Assert(childFields != null);
return(new Types.StructType(childFields));
default:
throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
}
}
