private SchemaElement BuildMapSchema(ref Thrift.SchemaElement tse, ref int i, bool isRoot, SchemaElement node, ParquetOptions formatOptions) { //tse is followed by map container (REPEATED) and another two elements - key and value Thrift.SchemaElement tseContainer = _fileMeta.Schema[++i]; Thrift.SchemaElement tseKey = _fileMeta.Schema[++i]; Thrift.SchemaElement tseValue = _fileMeta.Schema[++i]; Type keyType = TypePrimitive.GetSystemTypeBySchema(tseKey, formatOptions); Type valueType = TypePrimitive.GetSystemTypeBySchema(tseValue, formatOptions); Type gt = typeof(Dictionary <,>); Type masterType = gt.MakeGenericType(keyType, valueType); //master schema var se = new SchemaElement(tseContainer, tse.Name, masterType, masterType, string.Join(Schema.PathSeparator, tse.Name, tseContainer.Name)); if (!isRoot) { se.Path = node.Parent + Schema.PathSeparator + se.Path; } se.Parent = node; se.IsMap = true; AddFlags(se, tse, tseContainer); //extra schamas var kse = new SchemaElement(tseKey, null, keyType, keyType, null) { Parent = se }; var vse = new SchemaElement(tseValue, null, valueType, valueType, null) { Parent = se }; se.Extra.Add(kse); se.Extra.Add(vse); AddFlags(kse, tseKey); AddFlags(vse, tseValue); tse = tseValue; return(se); }
private void CreateModelSchema(string path, IList <Field> container, int childCount, ref int si, ParquetOptions formatOptions) { for (int i = 0; i < childCount && si < _fileMeta.Schema.Count; i++) { Thrift.SchemaElement tse = _fileMeta.Schema[si]; IDataTypeHandler dth = DataTypeFactory.Match(tse, formatOptions); if (dth == null) { throw new InvalidOperationException($"cannot find data type handler to create model schema for {tse.Describe()}"); } Field se = dth.CreateSchemaElement(_fileMeta.Schema, ref si, out int ownedChildCount); se.Path = string.Join(Schema.PathSeparator, new[] { path, se.Path ?? se.Name }.Where(p => p != null)); if (ownedChildCount > 0) { var childContainer = new List <Field>(); CreateModelSchema(se.Path, childContainer, ownedChildCount, ref si, formatOptions); foreach (Field cse in childContainer) { se.Assign(cse); } } container.Add(se); } }
public bool IsMatch(Thrift.SchemaElement tse, ParquetOptions formatOptions) { return (tse.__isset.converted_type && (tse.Converted_type == Thrift.ConvertedType.MAP || tse.Converted_type == Thrift.ConvertedType.MAP_KEY_VALUE)); }
public override int Read(BinaryReader reader, Thrift.SchemaElement tse, Array dest, int offset, ParquetOptions formatOptions) { string[] tdest = (string[])dest; int totalLength = (int)reader.BaseStream.Length; int idx = offset; Stream s = reader.BaseStream; while (s.Position < totalLength) { string element = ReadOne(reader); tdest[idx++] = element; } return(idx - offset); }