public void TestParquetSchema()
{
var fields = new Dictionary <string, DataType>()
{
{ "ID", DataType.String },
{ "DELETED", DataType.Boolean },
{ "CREATED_BY", DataType.String },
{ "DATE_ENTERED", DataType.DateTime },
{ "MODIFIED_USER_ID", DataType.String },
{ "DATE_MODIFIED", DataType.DateTime },
{ "DATE_MODIFIED_UTC", DataType.DateTime },
{ "ASSIGNED_USER_ID", DataType.String },
{ "TEAM_ID", DataType.String },
{ "NAME", DataType.String },
{ "DATE_START", DataType.DateTime },
{ "TIME_START", DataType.DateTime },
{ "PARENT_TYPE", DataType.String },
{ "PARENT_ID", DataType.String },
{ "DESCRIPTION", DataType.String },
{ "DESCRIPTION_HTML", DataType.String },
{ "FROM_ADDR", DataType.String },
{ "FROM_NAME", DataType.String },
};
var container = new DataContainer();
var collection = new DataCollection(container, "emails-utf8.parquet");
var processor = new ParquetFileProcessor();
List <DataEntity> entities;
long rowCount = 0;
using (var stream = File.Open("../../../tests/emails-utf8.parquet", FileMode.Open))
{
entities = processor.ParseFileSchema(container, collection, stream, out rowCount);
}
Assert.Equal(200, rowCount);
foreach (var field in fields)
{
output.WriteLine($" check field {field.Key}");
var entity = entities.Find(x => x.Name.Equals(field.Key));
Assert.NotNull(entity);
Assert.Equal(field.Value, entity.DataType);
}
}
public void TestParquetCollect()
{
var container = new DataContainer();
var collection = new DataCollection(container, "emails-utf8.parquet");
var processor = new ParquetFileProcessor();
long rowCount;
List <DataEntity> entities;
using (var stream = File.Open("../../../tests/emails-utf8.parquet", FileMode.Open))
{
entities = processor.ParseFileSchema(container, collection, stream, out rowCount);
}
var entity = entities.FirstOrDefault(x => x.Name.Equals("FROM_ADDR"));
int index = entities.IndexOf(entity);
using (var stream = File.Open("../../../tests/emails-utf8.parquet", FileMode.Open)) {
var samples = processor.CollectSamples(container, collection, entity, index, stream, 5, 1);
Assert.Contains("*****@*****.**", samples);
}
}