internal Schema(Thrift.FileMetaData fm)
{
_elements = fm.Schema.Skip(1).Select(se => new SchemaElement(se)).ToList();
foreach (Thrift.SchemaElement se in fm.Schema)
{
_pathToElement[se.Name] = new SchemaElement(se);
}
}
public FileMetadataBuilder()
{
_meta = new Thrift.FileMetaData();
_meta.Created_by = CreatedBy;
_meta.Version = 1;
_meta.Row_groups = new List <Thrift.RowGroup>();
}
public FileMetadataBuilder(WriterOptions writerOptions)
{
_meta = new Thrift.FileMetaData();
this._writerOptions = writerOptions;
_meta.Created_by = CreatedBy;
_meta.Version = 1;
_meta.Row_groups = new List <Thrift.RowGroup>();
}
public ThriftFooter(Schema schema, long totalRowCount)
{
if (schema == null)
{
throw new ArgumentNullException(nameof(schema));
}
_fileMeta = CreateThriftSchema(schema);
_fileMeta.Num_rows = totalRowCount;
_fileMeta.Created_by = $"Parquet.Net version %Version% (build %Git.LongCommitHash%)";
}
public Thrift.FileMetaData CreateThriftSchema(Schema schema)
{
var meta = new Thrift.FileMetaData();
meta.Version = 1;
meta.Schema = new List <Thrift.SchemaElement>();
Thrift.SchemaElement root = AddRoot(meta.Schema);
CreateThriftSchema(schema.Fields, root, meta.Schema);
return(meta);
}
public ThriftFooter(Schema schema, long totalRowCount)
{
if (schema == null)
{
throw new ArgumentNullException(nameof(schema));
}
_fileMeta = CreateThriftSchema(schema);
_fileMeta.Num_rows = totalRowCount;
Version fileVersion = FileVersion(typeof(ThriftFooter));
_fileMeta.Created_by = $"Parquet.Net version {fileVersion}";
}
private void PrintStatistics(Thrift.FileMetaData fileMeta, Thrift.ColumnChunk column, Thrift.Statistics stats)
{
WriteLine(" Statistics", T.HeadingTextColor);
if (stats == null || !(stats.__isset.null_count || stats.__isset.distinct_count || stats.__isset.min || stats.__isset.max))
{
WriteLine(" none defined", T.ErrorTextColor);
return;
}
const string undefined = "undefined";
var t = new Table("name", "value");
t.AddRow("Null Count", stats.__isset.null_count ? stats.Null_count.ToString() : undefined);
t.AddRow("Distinct Count", stats.__isset.distinct_count ? stats.Distinct_count.ToString() : undefined);
t.AddRow("Min", stats.__isset.min ? fileMeta.DecodeSingleStatsValue(column, stats.Min) : undefined);
t.AddRow("Max", stats.__isset.max ? fileMeta.DecodeSingleStatsValue(column, stats.Max) : undefined);
t.Render(false, 6, T.HeadingTextColor, T.NormalTextColor);
}
public void Execute()
{
Thrift.FileMetaData fileMeta = ReadInternalMetadata();
//root metadata
WriteLine("File Metadata", T.HeadingTextColor);
var t = new Table("name", "value");
t.AddRow("Created By", fileMeta.Created_by);
t.AddRow("Total Rows", fileMeta.Num_rows);
t.AddRow("Version", fileMeta.Version);
t.Render(false, 0, T.HeadingTextColor, T.NormalTextColor);
WriteLine();
//custom key-values
if (fileMeta.Key_value_metadata == null || fileMeta.Key_value_metadata.Count == 0)
{
WriteLine("no custom key-value metadata", ConsoleColor.Gray);
}
else
{
WriteLine("Key-Value Metadata", T.HeadingTextColor);
t = new Table("key", "vlaue");
foreach (Thrift.KeyValue kv in fileMeta.Key_value_metadata)
{
t.AddRow(kv.Key, kv.Value);
}
t.Render(false, 0, T.HeadingTextColor, T.NormalTextColor);
}
WriteLine();
//row groups
WriteLine("Row Groups", T.HeadingTextColor);
int i = 0;
foreach (Thrift.RowGroup rg in fileMeta.Row_groups)
{
Print(fileMeta, rg, i++);
}
}
private void Print(Thrift.FileMetaData fileMeta, Thrift.RowGroup rg, int index)
{
WriteLine();
PoshWriteLine($" Row Group #{{{index}}}", ConsoleColor.Red);
var t = new Table("name", "value");
t.AddRow("Total Rows", rg.Num_rows);
t.AddRow("Total Byte Size", GetSizeString(rg.Total_byte_size));
t.Render(false, 2, T.HeadingTextColor, T.NormalTextColor);
WriteLine();
//columns
int i = 0;
foreach (Thrift.ColumnChunk column in rg.Columns)
{
t = new Table("name", "value");
t.AddRow("File Offset", column.File_offset);
t.AddRow("File Path", column.File_path ?? string.Empty);
t.AddRow("Codec", column.Meta_data.Codec);
t.AddRow("Data Page Offset", column.Meta_data.Data_page_offset);
t.AddRow("Dictionary Page Offset", column.Meta_data.Dictionary_page_offset);
t.AddRow("Index Page Offset", column.Meta_data.Index_page_offset);
t.AddRow("Encodings", string.Join(", ", column.Meta_data.Encodings));
//t.AddRow("", column.Meta_data.Encoding_stats[0].)
t.AddRow("Total Values", column.Meta_data.Num_values);
t.AddRow("Path in Schema", string.Join(".", column.Meta_data.Path_in_schema));
t.AddRow("Compressed Size", GetSizeString(column.Meta_data.Total_compressed_size));
t.AddRow("Uncompressed Size", GetSizeString(column.Meta_data.Total_uncompressed_size));
t.AddRow("Type", column.Meta_data.Type);
PoshWriteLine($" Column #{{{i++}}}", ConsoleColor.Red);
t.Render(false, 4, T.HeadingTextColor, T.NormalTextColor);
PrintStatistics(fileMeta, column, column.Meta_data.Statistics);
WriteLine();
}
}
public void SetMeta(Thrift.FileMetaData meta)
{
_meta = meta;
}
public ThriftFooter(Thrift.FileMetaData fileMeta)
{
_fileMeta = fileMeta ?? throw new ArgumentNullException(nameof(fileMeta));
}
public ThriftFooter(Thrift.FileMetaData fileMeta)
{
_fileMeta = fileMeta ?? throw new ArgumentNullException(nameof(fileMeta));
_tree = new ThriftSchemaTree(_fileMeta.Schema);
}
public ThriftFooter(Thrift.FileMetaData fileMeta)
{
_fileMeta = fileMeta;
}
