public static bool IsAnnotatedWithAny(this Thrift.SchemaElement schemaElement, Thrift.ConvertedType[] convertedTypes)
{
if (convertedTypes == null || convertedTypes.Length == 0)
{
return(false);
}
return
(schemaElement.__isset.converted_type &&
convertedTypes.Any(ct => ct == schemaElement.Converted_type));
}
/// <summary>
/// Decodes raw bytes from <see cref="Thrift.Statistics"/> into a CLR value
/// </summary>
public static object DecodeSingleStatsValue(this Thrift.FileMetaData fileMeta, Thrift.ColumnChunk columnChunk, byte[] rawBytes)
{
if (rawBytes == null || rawBytes.Length == 0)
{
return(null);
}
var footer = new ThriftFooter(fileMeta);
Thrift.SchemaElement schema = footer.GetSchemaElement(columnChunk);
IDataTypeHandler handler = DataTypeFactory.Match(schema, new ParquetOptions {
TreatByteArrayAsString = true
});
using (var ms = new MemoryStream(rawBytes))
using (var reader = new BinaryReader(ms))
{
object value = handler.Read(reader, schema, rawBytes.Length);
return(value);
}
}
/// <summary>
///
/// </summary>
/// <param name="column"></param>
public void WriteColumn(DataColumn column)
{
if (column == null)
{
throw new ArgumentNullException(nameof(column));
}
Thrift.SchemaElement tse = _thschema[_colIdx];
if (!column.Field.Equals(tse))
{
throw new ArgumentException($"cannot write this column, expected '{tse.Name}', passed: '{column.Field.Name}'", nameof(column));
}
IDataTypeHandler dataTypeHandler = DataTypeFactory.Match(tse, _formatOptions);
_colIdx += 1;
List <string> path = _footer.GetPath(tse);
var writer = new DataColumnWriter(_stream, _thriftStream, _footer, tse, _compressionMethod, _rowCount);
Thrift.ColumnChunk chunk = writer.Write(path, column, dataTypeHandler);
_thriftRowGroup.Columns.Add(chunk);
}
public static bool IsNullable(this Thrift.SchemaElement schemaElement)
{
return(schemaElement.Repetition_type != Thrift.FieldRepetitionType.REQUIRED);
}
public async Task ReadAsync(TProtocol iprot, CancellationToken cancellationToken)
{
iprot.IncrementRecursionDepth();
try
{
bool isset_version = false;
bool isset_schema = false;
bool isset_num_rows = false;
bool isset_row_groups = false;
TField field;
await iprot.ReadStructBeginAsync(cancellationToken);
while (true)
{
field = await iprot.ReadFieldBeginAsync(cancellationToken);
if (field.Type == TType.Stop)
{
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.I32)
{
Version = await iprot.ReadI32Async(cancellationToken);
isset_version = true;
}
else
{
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
}
break;
case 2:
if (field.Type == TType.List)
{
{
Schema = new List <SchemaElement>();
TList _list44 = await iprot.ReadListBeginAsync(cancellationToken);
for (int _i45 = 0; _i45 < _list44.Count; ++_i45)
{
SchemaElement _elem46;
_elem46 = new SchemaElement();
await _elem46.ReadAsync(iprot, cancellationToken);
Schema.Add(_elem46);
}
await iprot.ReadListEndAsync(cancellationToken);
}
isset_schema = true;
}
else
{
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
}
break;
case 3:
if (field.Type == TType.I64)
{
Num_rows = await iprot.ReadI64Async(cancellationToken);
isset_num_rows = true;
}
else
{
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
}
break;
case 4:
if (field.Type == TType.List)
{
{
Row_groups = new List <RowGroup>();
TList _list47 = await iprot.ReadListBeginAsync(cancellationToken);
for (int _i48 = 0; _i48 < _list47.Count; ++_i48)
{
RowGroup _elem49;
_elem49 = new RowGroup();
await _elem49.ReadAsync(iprot, cancellationToken);
Row_groups.Add(_elem49);
}
await iprot.ReadListEndAsync(cancellationToken);
}
isset_row_groups = true;
}
else
{
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
}
break;
case 5:
if (field.Type == TType.List)
{
{
Key_value_metadata = new List <KeyValue>();
TList _list50 = await iprot.ReadListBeginAsync(cancellationToken);
for (int _i51 = 0; _i51 < _list50.Count; ++_i51)
{
KeyValue _elem52;
_elem52 = new KeyValue();
await _elem52.ReadAsync(iprot, cancellationToken);
Key_value_metadata.Add(_elem52);
}
await iprot.ReadListEndAsync(cancellationToken);
}
}
else
{
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
}
break;
case 6:
if (field.Type == TType.String)
{
Created_by = await iprot.ReadStringAsync(cancellationToken);
}
else
{
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
}
break;
case 7:
if (field.Type == TType.List)
{
{
Column_orders = new List <ColumnOrder>();
TList _list53 = await iprot.ReadListBeginAsync(cancellationToken);
for (int _i54 = 0; _i54 < _list53.Count; ++_i54)
{
ColumnOrder _elem55;
_elem55 = new ColumnOrder();
await _elem55.ReadAsync(iprot, cancellationToken);
Column_orders.Add(_elem55);
}
await iprot.ReadListEndAsync(cancellationToken);
}
}
else
{
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
}
break;
default:
await TProtocolUtil.SkipAsync(iprot, field.Type, cancellationToken);
break;
}
await iprot.ReadFieldEndAsync(cancellationToken);
}
await iprot.ReadStructEndAsync(cancellationToken);
if (!isset_version)
{
throw new TProtocolException(TProtocolException.INVALID_DATA);
}
if (!isset_schema)
{
throw new TProtocolException(TProtocolException.INVALID_DATA);
}
if (!isset_num_rows)
{
throw new TProtocolException(TProtocolException.INVALID_DATA);
}
if (!isset_row_groups)
{
throw new TProtocolException(TProtocolException.INVALID_DATA);
}
}
finally
{
iprot.DecrementRecursionDepth();
}
}