public static bool IsAnnotatedWithAny(this Thrift.SchemaElement schemaElement, Thrift.ConvertedType[] convertedTypes) { if (convertedTypes == null || convertedTypes.Length == 0) { return(false); } return (schemaElement.__isset.converted_type && convertedTypes.Any(ct => ct == schemaElement.Converted_type)); }
/// <summary> /// Decodes raw bytes from <see cref="Thrift.Statistics"/> into a CLR value /// </summary> public static object DecodeSingleStatsValue(this Thrift.FileMetaData fileMeta, Thrift.ColumnChunk columnChunk, byte[] rawBytes) { if (rawBytes == null || rawBytes.Length == 0) { return(null); } var footer = new ThriftFooter(fileMeta); Thrift.SchemaElement schema = footer.GetSchemaElement(columnChunk); IDataTypeHandler handler = DataTypeFactory.Match(schema, new ParquetOptions { TreatByteArrayAsString = true }); using (var ms = new MemoryStream(rawBytes)) using (var reader = new BinaryReader(ms)) { object value = handler.Read(reader, schema, rawBytes.Length); return(value); } }
/// <summary> /// /// </summary> /// <param name="column"></param> public void WriteColumn(DataColumn column) { if (column == null) { throw new ArgumentNullException(nameof(column)); } Thrift.SchemaElement tse = _thschema[_colIdx]; if (!column.Field.Equals(tse)) { throw new ArgumentException($"cannot write this column, expected '{tse.Name}', passed: '{column.Field.Name}'", nameof(column)); } IDataTypeHandler dataTypeHandler = DataTypeFactory.Match(tse, _formatOptions); _colIdx += 1; List <string> path = _footer.GetPath(tse); var writer = new DataColumnWriter(_stream, _thriftStream, _footer, tse, _compressionMethod, _rowCount); Thrift.ColumnChunk chunk = writer.Write(path, column, dataTypeHandler); _thriftRowGroup.Columns.Add(chunk); }
public static bool IsNullable(this Thrift.SchemaElement schemaElement) { return(schemaElement.Repetition_type != Thrift.FieldRepetitionType.REQUIRED); }
public async Task ReadAsync(TProtocol iprot, CancellationToken cancellationToken) { iprot.IncrementRecursionDepth(); try { bool isset_version = false; bool isset_schema = false; bool isset_num_rows = false; bool isset_row_groups = false; TField field; await iprot.ReadStructBeginAsync(cancellationToken); while (true) { field = await iprot.ReadFieldBeginAsync(cancellationToken); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.I32) { Version = await iprot.ReadI32Async(cancellationToken); isset_version = true; } else { await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); } break; case 2: if (field.Type == TType.List) { { Schema = new List <SchemaElement>(); TList _list44 = await iprot.ReadListBeginAsync(cancellationToken); for (int _i45 = 0; _i45 < _list44.Count; ++_i45) { SchemaElement _elem46; _elem46 = new SchemaElement(); await _elem46.ReadAsync(iprot, cancellationToken); Schema.Add(_elem46); } await iprot.ReadListEndAsync(cancellationToken); } isset_schema = true; } else { await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); } break; case 3: if (field.Type == TType.I64) { Num_rows = await iprot.ReadI64Async(cancellationToken); isset_num_rows = true; } else { await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); } break; case 4: if (field.Type == TType.List) { { Row_groups = new List <RowGroup>(); TList _list47 = await iprot.ReadListBeginAsync(cancellationToken); for (int _i48 = 0; _i48 < _list47.Count; ++_i48) { RowGroup _elem49; _elem49 = new RowGroup(); await _elem49.ReadAsync(iprot, cancellationToken); Row_groups.Add(_elem49); } await iprot.ReadListEndAsync(cancellationToken); } isset_row_groups = true; } else { await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); } break; case 5: if (field.Type == TType.List) { { Key_value_metadata = new List <KeyValue>(); TList _list50 = await iprot.ReadListBeginAsync(cancellationToken); for (int _i51 = 0; _i51 < _list50.Count; ++_i51) { KeyValue _elem52; _elem52 = new KeyValue(); await _elem52.ReadAsync(iprot, cancellationToken); Key_value_metadata.Add(_elem52); } await iprot.ReadListEndAsync(cancellationToken); } } else { await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); } break; case 6: if (field.Type == TType.String) { Created_by = await iprot.ReadStringAsync(cancellationToken); } else { await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); } break; case 7: if (field.Type == TType.List) { { Column_orders = new List <ColumnOrder>(); TList _list53 = await iprot.ReadListBeginAsync(cancellationToken); for (int _i54 = 0; _i54 < _list53.Count; ++_i54) { ColumnOrder _elem55; _elem55 = new ColumnOrder(); await _elem55.ReadAsync(iprot, cancellationToken); Column_orders.Add(_elem55); } await iprot.ReadListEndAsync(cancellationToken); } } else { await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); } break; default: await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken); break; } await iprot.ReadFieldEndAsync(cancellationToken); } await iprot.ReadStructEndAsync(cancellationToken); if (!isset_version) { throw new TProtocolException(TProtocolException.INVALID_DATA); } if (!isset_schema) { throw new TProtocolException(TProtocolException.INVALID_DATA); } if (!isset_num_rows) { throw new TProtocolException(TProtocolException.INVALID_DATA); } if (!isset_row_groups) { throw new TProtocolException(TProtocolException.INVALID_DATA); } } finally { iprot.DecrementRecursionDepth(); } }