private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField) { Field[] childFields = null; if (flatbufField.ChildrenLength > 0) { childFields = new Field[flatbufField.ChildrenLength]; for (int i = 0; i < flatbufField.ChildrenLength; i++) { Flatbuf.Field?childFlatbufField = flatbufField.Children(i); childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value); } } return(new Field(flatbufField.Name, GetFieldArrowType(flatbufField, childFields), flatbufField.Nullable)); }
private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField) { Field[] childFields = flatbufField.ChildrenLength > 0 ? new Field[flatbufField.ChildrenLength] : null; for (int i = 0; i < flatbufField.ChildrenLength; i++) { Flatbuf.Field?childFlatbufField = flatbufField.Children(i); childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value); } Dictionary <string, string> metadata = flatbufField.CustomMetadataLength > 0 ? new Dictionary <string, string>() : null; for (int i = 0; i < flatbufField.CustomMetadataLength; i++) { Flatbuf.KeyValue keyValue = flatbufField.CustomMetadata(i).GetValueOrDefault(); metadata[keyValue.Key] = keyValue.Value; } return(new Field(flatbufField.Name, GetFieldArrowType(flatbufField, childFields), flatbufField.Nullable, metadata, copyCollections: false)); }
private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField, ref DictionaryMemo dictionaryMemo) { Field[] childFields = flatbufField.ChildrenLength > 0 ? new Field[flatbufField.ChildrenLength] : null; for (int i = 0; i < flatbufField.ChildrenLength; i++) { Flatbuf.Field?childFlatbufField = flatbufField.Children(i); childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value, ref dictionaryMemo); } Flatbuf.DictionaryEncoding?dictionaryEncoding = flatbufField.Dictionary; IArrowType type = GetFieldArrowType(flatbufField, childFields); if (dictionaryEncoding.HasValue) { Flatbuf.Int?indexTypeAsInt = dictionaryEncoding.Value.IndexType; IArrowType indexType = indexTypeAsInt.HasValue ? GetNumberType(indexTypeAsInt.Value.BitWidth, indexTypeAsInt.Value.IsSigned) : GetNumberType(Int32Type.Default.BitWidth, Int32Type.Default.IsSigned); type = new DictionaryType(indexType, type, dictionaryEncoding.Value.IsOrdered); } Dictionary <string, string> metadata = flatbufField.CustomMetadataLength > 0 ? new Dictionary <string, string>() : null; for (int i = 0; i < flatbufField.CustomMetadataLength; i++) { Flatbuf.KeyValue keyValue = flatbufField.CustomMetadata(i).GetValueOrDefault(); metadata[keyValue.Key] = keyValue.Value; } var arrowField = new Field(flatbufField.Name, type, flatbufField.Nullable, metadata, copyCollections: false); if (dictionaryEncoding.HasValue) { dictionaryMemo ??= new DictionaryMemo(); dictionaryMemo.AddField(dictionaryEncoding.Value.Id, arrowField); } return(arrowField); }
private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field, Field[] childFields = null) { switch (field.TypeType) { case Flatbuf.Type.Int: Flatbuf.Int intMetaData = field.Type <Flatbuf.Int>().Value; return(MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned)); case Flatbuf.Type.FloatingPoint: Flatbuf.FloatingPoint floatingPointTypeMetadata = field.Type <Flatbuf.FloatingPoint>().Value; switch (floatingPointTypeMetadata.Precision) { case Flatbuf.Precision.SINGLE: return(Types.FloatType.Default); case Flatbuf.Precision.DOUBLE: return(Types.DoubleType.Default); case Flatbuf.Precision.HALF: return(Types.HalfFloatType.Default); default: throw new InvalidDataException("Unsupported floating point precision"); } case Flatbuf.Type.Bool: return(new Types.BooleanType()); case Flatbuf.Type.Decimal: Flatbuf.Decimal decMeta = field.Type <Flatbuf.Decimal>().Value; return(new Types.DecimalType(decMeta.Precision, decMeta.Scale)); case Flatbuf.Type.Date: Flatbuf.Date dateMeta = field.Type <Flatbuf.Date>().Value; switch (dateMeta.Unit) { case Flatbuf.DateUnit.DAY: return(Types.Date32Type.Default); case Flatbuf.DateUnit.MILLISECOND: return(Types.Date64Type.Default); default: throw new InvalidDataException("Unsupported date unit"); } case Flatbuf.Type.Time: Flatbuf.Time timeMeta = field.Type <Flatbuf.Time>().Value; switch (timeMeta.BitWidth) { case 32: return(new Types.Time32Type(timeMeta.Unit.ToArrow())); case 64: return(new Types.Time64Type(timeMeta.Unit.ToArrow())); default: throw new InvalidDataException("Unsupported time bit width"); } case Flatbuf.Type.Timestamp: Flatbuf.Timestamp timestampTypeMetadata = field.Type <Flatbuf.Timestamp>().Value; Types.TimeUnit unit = timestampTypeMetadata.Unit.ToArrow(); string timezone = timestampTypeMetadata.Timezone; return(new Types.TimestampType(unit, timezone)); case Flatbuf.Type.Interval: Flatbuf.Interval intervalMetadata = field.Type <Flatbuf.Interval>().Value; return(new Types.IntervalType(intervalMetadata.Unit.ToArrow())); case Flatbuf.Type.Utf8: return(new Types.StringType()); case Flatbuf.Type.Binary: return(Types.BinaryType.Default); case Flatbuf.Type.List: if (field.ChildrenLength != 1) { throw new InvalidDataException($"List type must have only one child."); } return(new Types.ListType(GetFieldArrowType(field.Children(0).GetValueOrDefault()))); case Flatbuf.Type.Struct_: Debug.Assert(childFields != null); return(new Types.StructType(childFields)); default: throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported."); } }