private SchemaElement BuildMapSchema(ref Thrift.SchemaElement tse, ref int i, bool isRoot, SchemaElement node, ParquetOptions formatOptions)
{
//tse is followed by map container (REPEATED) and another two elements - key and value
Thrift.SchemaElement tseContainer = _fileMeta.Schema[++i];
Thrift.SchemaElement tseKey = _fileMeta.Schema[++i];
Thrift.SchemaElement tseValue = _fileMeta.Schema[++i];
Type keyType = TypePrimitive.GetSystemTypeBySchema(tseKey, formatOptions);
Type valueType = TypePrimitive.GetSystemTypeBySchema(tseValue, formatOptions);
Type gt = typeof(Dictionary <,>);
Type masterType = gt.MakeGenericType(keyType, valueType);
//master schema
var se = new SchemaElement(tseContainer, tse.Name, masterType, masterType,
string.Join(Schema.PathSeparator, tse.Name, tseContainer.Name));
if (!isRoot)
{
se.Path = node.Parent + Schema.PathSeparator + se.Path;
}
se.Parent = node;
se.IsMap = true;
AddFlags(se, tse, tseContainer);
//extra schamas
var kse = new SchemaElement(tseKey, null, keyType, keyType, null)
{
Parent = se
};
var vse = new SchemaElement(tseValue, null, valueType, valueType, null)
{
Parent = se
};
se.Extra.Add(kse);
se.Extra.Add(vse);
AddFlags(kse, tseKey);
AddFlags(vse, tseValue);
tse = tseValue;
return(se);
}
private void CreateModelSchema(string path, IList <Field> container, int childCount, ref int si, ParquetOptions formatOptions)
{
for (int i = 0; i < childCount && si < _fileMeta.Schema.Count; i++)
{
Thrift.SchemaElement tse = _fileMeta.Schema[si];
IDataTypeHandler dth = DataTypeFactory.Match(tse, formatOptions);
if (dth == null)
{
throw new InvalidOperationException($"cannot find data type handler to create model schema for {tse.Describe()}");
}
Field se = dth.CreateSchemaElement(_fileMeta.Schema, ref si, out int ownedChildCount);
se.Path = string.Join(Schema.PathSeparator, new[] { path, se.Path ?? se.Name }.Where(p => p != null));
if (ownedChildCount > 0)
{
var childContainer = new List <Field>();
CreateModelSchema(se.Path, childContainer, ownedChildCount, ref si, formatOptions);
foreach (Field cse in childContainer)
{
se.Assign(cse);
}
}
container.Add(se);
}
}
public bool IsMatch(Thrift.SchemaElement tse, ParquetOptions formatOptions)
{
return
(tse.__isset.converted_type &&
(tse.Converted_type == Thrift.ConvertedType.MAP || tse.Converted_type == Thrift.ConvertedType.MAP_KEY_VALUE));
}
public override int Read(BinaryReader reader, Thrift.SchemaElement tse, Array dest, int offset, ParquetOptions formatOptions)
{
string[] tdest = (string[])dest;
int totalLength = (int)reader.BaseStream.Length;
int idx = offset;
Stream s = reader.BaseStream;
while (s.Position < totalLength)
{
string element = ReadOne(reader);
tdest[idx++] = element;
}
return(idx - offset);
}
