private SchemaElement BuildSchemaElement(Thrift.SchemaElement tse, SchemaElement parent, ParquetOptions formatOptions, Type elementType, string name = null)
{
Type columnType;
if (elementType != null)
{
columnType = elementType;
if (elementType == typeof(IEnumerable))
{
Type itemType = TypePrimitive.GetSystemTypeBySchema(tse, formatOptions);
Type ienumType = typeof(IEnumerable <>);
Type ienumGenericType = ienumType.MakeGenericType(itemType);
elementType = itemType;
columnType = ienumGenericType;
}
}
else
{
elementType = TypePrimitive.GetSystemTypeBySchema(tse, formatOptions);
columnType = elementType;
}
var se = new SchemaElement(tse, name, elementType, columnType, null);
se.Parent = parent;
return(se);
}
private SchemaElement BuildMapSchema(ref Thrift.SchemaElement tse, ref int i, bool isRoot, SchemaElement node, ParquetOptions formatOptions)
{
//tse is followed by map container (REPEATED) and another two elements - key and value
Thrift.SchemaElement tseContainer = _fileMeta.Schema[++i];
Thrift.SchemaElement tseKey = _fileMeta.Schema[++i];
Thrift.SchemaElement tseValue = _fileMeta.Schema[++i];
Type keyType = TypePrimitive.GetSystemTypeBySchema(tseKey, formatOptions);
Type valueType = TypePrimitive.GetSystemTypeBySchema(tseValue, formatOptions);
Type gt = typeof(Dictionary <,>);
Type masterType = gt.MakeGenericType(keyType, valueType);
//master schema
var se = new SchemaElement(tseContainer, tse.Name, masterType, masterType,
string.Join(Schema.PathSeparator, tse.Name, tseContainer.Name));
if (!isRoot)
{
se.Path = node.Parent + Schema.PathSeparator + se.Path;
}
se.Parent = node;
se.IsMap = true;
AddFlags(se, tse, tseContainer);
//extra schamas
var kse = new SchemaElement(tseKey, null, keyType, keyType, null)
{
Parent = se
};
var vse = new SchemaElement(tseValue, null, valueType, valueType, null)
{
Parent = se
};
se.Extra.Add(kse);
se.Extra.Add(vse);
AddFlags(kse, tseKey);
AddFlags(vse, tseValue);
tse = tseValue;
return(se);
}
public static void AdjustSchema(Thrift.SchemaElement schema, Type systemType)
{
if (IsPrimitiveNullable(systemType))
{
throw new ArgumentException($"Type '{systemType}' in column '{schema.Name}' is a nullable type. Please pass either a class or a primitive type.", nameof(systemType));
}
TypePrimitive tp = TypePrimitive.Find(systemType);
schema.Type = tp.ThriftType;
if (tp.ThriftAnnotation != null)
{
schema.Converted_type = tp.ThriftAnnotation.Value;
}
//todo: not the best place for it, but it's a special case at the moment
if (systemType == typeof(decimal))
{
schema.Precision = 38;
schema.Scale = 18;
}
}
public TSchemaElement CreateSchemaElement(string name, Type systemType,
out Type elementType,
out string path)
{
var th = new TSchemaElement(name);
if (TypeFactory.TryExtractEnumerableType(systemType, out Type baseType))
{
elementType = baseType;
path = BuildRepeatablePath(name);
th.Repetition_type = Thrift.FieldRepetitionType.REPEATED;
}
else if (typeof(Row) == systemType)
{
//this is a hierarchy, therefore can be a fake type
elementType = typeof(int);
path = null;
th.Repetition_type = Thrift.FieldRepetitionType.OPTIONAL;
}
else
{
if (TryGetNonNullableType(systemType, out Type nonNullableSystemType))
{
elementType = nonNullableSystemType;
th.Repetition_type = Thrift.FieldRepetitionType.OPTIONAL;
}
else if (systemType.GetTypeInfo().IsClass)
{
//reference types are always nullable
elementType = systemType;
th.Repetition_type = Thrift.FieldRepetitionType.OPTIONAL;
}
else
{
elementType = systemType;
//this might be changed later or if column has nulls (on write)
th.Repetition_type = Thrift.FieldRepetitionType.REQUIRED;
}
path = null;
}
//adjust schema
TypePrimitive tp = TypePrimitive.Find(elementType);
th.Type = tp.ThriftType;
if (tp.ThriftAnnotation != null)
{
th.Converted_type = tp.ThriftAnnotation.Value;
}
//todo: not the best place for it, but it's a special case at the moment
if (systemType == typeof(decimal))
{
th.Precision = 38;
th.Scale = 18;
th.Type_length = 16; //set to default type length to be used when no elements are added
}
return(th);
}
public static IList Create(SchemaElement schema, ParquetOptions options, bool nullable = false)
{
Type t = TypePrimitive.GetSystemTypeBySchema(schema, options);
return(Create(t, nullable));
}
