static void Main(string[] args)
{
L.Config
.WriteTo.PoshConsole();
using (var time = new TimeMeasure())
{
var ds = new DataSet(
new SchemaElement <int>("id"),
new SchemaElement <string>("name"),
new SchemaElement <double>("lat"),
new SchemaElement <double>("lon"));
log.Trace(ds.Schema.ToString());
for (int i = 0; i < 10; i++)
{
ds.Add(
i,
NameGenerator.GeneratePersonFullName(),
Generator.RandomDouble,
Generator.RandomDouble);
}
ParquetWriter.WriteFile(ds, "c:\\tmp\\perf.parquet");
log.Trace("written in {0}", time.Elapsed);
}
}
//[Fact]
public void TempTest()
{
var dir = new DirectoryInfo(@"C:\Users\ivang\Downloads\Fullfeed-20170330004044");
FileInfo[] files = dir.GetFiles();
JObject[] jos = files
.Select(fi => JObject.Parse(System.IO.File.ReadAllText(fi.FullName)))
.Take(1000)
.ToArray();
var inferrer = new JsonSchemaInferring();
Schema schema = inferrer.InferSchema(jos);
var extractor = new JsonDataExtractor(schema);
var ds = new DataSet(schema);
for (int i = 0; i < jos.Length; i++)
{
extractor.AddRow(ds, jos[i]);
}
ParquetWriter.WriteFile(ds, "c:\\tmp\\com.parquet");
}