static void EmptyFileTest()
{
using (var w = new ChoParquetWriter("EmptyFile.parquet"))
{
//w.Write((dynamic)null);
}
}
static void SerializeValue()
{
using (var w = new ChoParquetWriter("SerializeValue.parquet"))
{
w.Write(4);
}
}
static void SerializeArray()
{
using (var w = new ChoParquetWriter("SerializeArray.parquet"))
{
w.Write(new int[] { 1, 2 });
}
}
static void Json2Parquet()
{
string json = @"
[{
""Health"": {
""Id"": 99,
""Status"": false
},
""Safety"": {
""Id"": 3,
""Fire"": 1
},
""Climate"": [{
""Id"": 0,
""State"": 2
}]
}]";
using (var r = ChoJSONReader <MyData> .LoadText(json)
.UseJsonSerialization())
{
//using (var w = new ChoParquetWriter("MyData.parquet")
// //.UseNestedKeyFormat()
// )
//{
// w.Write(r.Select(rec1 => rec1.ToDictionary().Flatten().ToDictionary()));
//}
var x = ChoParquetWriter.SerializeAll(r); //.Select(rec1 => rec1.ToDictionary().Flatten().ToDictionary()));
File.WriteAllBytes("MyData1.parquet", x);
}
}
static void JsonToParquet52()
{
using (var r = new ChoJSONReader("sample52.json")
.WithJSONPath("$..data")
)
{
using (var w = new ChoParquetWriter("myData52.parquet"))
w.Write(r);
}
}
static void SerializeValue()
{
//byte[] x = ChoParquetWriter.Serialize(4);
//File.WriteAllBytes("SerializeValue.parquet", ChoParquetWriter.Serialize(4));
//return;
using (var w = new ChoParquetWriter("SerializeValue.parquet"))
{
w.Write(4);
}
}
static void CreateParquetFile(string parquetFilePath, string csv)
{
using (var r = ChoCSVReader.LoadText(csv)
.WithFirstLineHeader()
)
{
using (var w = new ChoParquetWriter(parquetFilePath))
w.Write(r);
}
}
static void SerializeDictionary()
{
using (var w = new ChoParquetWriter("SerializeDictionary.parquet"))
{
w.Write(new Dictionary <int, string>()
{
[1] = "Tom",
[2] = "Mark"
});
}
}
static void ByteArrayTest()
{
using (var w = new ChoParquetWriter("ByteArrayTest.parquet"))
{
w.Write(new Dictionary <int, byte[]>()
{
[1] = Encoding.Default.GetBytes("Tom"),
[2] = Encoding.Default.GetBytes("Mark")
});
}
}
static void SerializeDateTime()
{
IList <DateTime> dateList = new List <DateTime>
{
new DateTime(2009, 12, 7), //, 23, 10, 0, DateTimeKind.Utc),
new DateTime(2010, 1, 1, 9, 0, 0, DateTimeKind.Utc),
new DateTime(2010, 2, 10, 10, 0, 0, DateTimeKind.Utc)
};
using (var w = new ChoParquetWriter <DateTime>("DateTimeTest.parquet"))
w.Write(dateList);
}
static void JSON2Parquet1()
{
string json = @"{
""facilities"": [
{
""id"": 39205,
""name"": ""Sample1"",
""uuid"": ""ac2f3464-c425-4063-86ad-163521b1d610"",
""createdAt"": ""2019-03-06T14:25:32Z"",
""updatedAt"": ""2019-03-06T14:29:31Z"",
""active"": true
},
{
""id"": 35907,
""name"": ""Sample2"",
""uuid"": ""d371debb-f030-4c1e-b198-5eb562ceac0f"",
""createdAt"": ""2019-02-21T09:33:25Z"",
""updatedAt"": ""2019-02-21T09:33:25Z"",
""active"": true
}
]
}
";
using (var r = ChoJSONReader.LoadText(json)
.WithJSONPath("$..facilities[*]", true)
.WithField("id")
.WithField("createdAt", fieldType: typeof(DateTimeOffset), valueConverter: o => DateTimeOffset.Now)
.ErrorMode(ChoErrorMode.IgnoreAndContinue)
)
{
using (var w = new ChoParquetWriter("JSON2Parquet1.parquet")
.ErrorMode(ChoErrorMode.IgnoreAndContinue)
)
{
w.Write(r);
}
return;
foreach (var rec in r)
{
Console.WriteLine(rec.Dump());
}
return;
//var x = r.Select(r1 => { r1.Location = new Point(100); return r1; }).ToArray();
//using (var w = new ChoParquetWriter<Facility>("JSON2Parquet1.parquet")
// .IgnoreField("Location.IsEmpty")
// )
//{
// w.Write(x);
//}
}
}
static void CSVArrayToParquet()
{
string csv = @"id,name,friends/0,friends/1
1,Tom,Dick,Harry";
using (var r = ChoCSVReader.LoadText(csv).WithFirstLineHeader()
//.Configure(c => c.AutoArrayDiscovery = true)
//.Configure(c => c.ArrayIndexSeparator = '/')
)
{
using (var w = new ChoParquetWriter("CSVArrayToParquet.parquet"))
{
w.Write(r);
}
}
}
static void QuickTest()
{
string csv = @"Id, Name
1, Tom
2, Mark";
using (var r = ChoCSVReader.LoadText(csv)
.WithFirstLineHeader()
.WithMaxScanRows(2)
.QuoteAllFields()
)
{
using (var w = new ChoParquetWriter("quicktest.parquet"))
{
w.Write(r);
}
}
}
static void CSV2ParquetTest()
{
using (var r = new ChoCSVReader(@"..\..\..\..\..\..\data\XBTUSD.csv")
.Configure(c => c.LiteParsing = true)
.NotifyAfter(100000)
.OnRowsLoaded((o, e) => $"Rows Loaded: {e.RowsLoaded} <-- {DateTime.Now}".Print())
.ThrowAndStopOnMissingField(false)
)
{
//r.Loop();
//return;
using (var w = new ChoParquetWriter(@"..\..\..\..\..\..\data\XBTUSD.parquet")
.Configure(c => c.RowGroupSize = 100000)
.Configure(c => c.LiteParsing = true)
)
w.Write(r);
}
}
static void Test1()
{
string csv = @"Cust_ID,CustName,CustOrder,Salary,Guid
TCF4338,INDEXABLE CUTTING TOOL,4/11/2016,""$100,000"",56531508-89c0-4ecf-afaf-cdf5aec56b19
CGO9650,Comercial Tecnipak Ltda,7/11/2016,""$80,000"",56531508-89c0-4ecf-afaf-cdf5aec56b19";
ChoTypeConverterFormatSpec.Instance.TreatCurrencyAsDecimal = false;
using (var r = ChoCSVReader.LoadText(csv)
.WithFirstLineHeader()
.WithMaxScanRows(2)
.QuoteAllFields()
)
{
using (var w = new ChoParquetWriter("test1.parquet"))
{
w.Write(r);
}
}
}
static void DB2ParquetTest()
{
using (var conn = new SqlConnection(@"Data Source=(LocalDB)\MSSQLLocalDB;AttachDbFilename=C:\Projects\GitHub\ChoETL\src\Test\ChoETL.SqlServer.Core.Test\bin\Debug\net5.0\localdb.mdf;Integrated Security=True;Connect Timeout=30"))
{
conn.Open();
var cmd = new SqlCommand("SELECT * FROM Trade", conn);
var dr = cmd.ExecuteReader();
using (var w = new ChoParquetWriter(@"..\..\..\..\..\..\data\Trade.parquet")
.Configure(c => c.LiteParsing = true)
.Configure(c => c.RowGroupSize = 5000)
.NotifyAfter(100000)
.OnRowsWritten((o, e) => $"Rows Loaded: {e.RowsWritten} <-- {DateTime.Now}".Print())
.ThrowAndStopOnMissingField(false)
)
{
w.Write(dr);
}
}
}
static void EnumTest()
{
string csv = @"Id, Name, EmpType
1, Tom, Full Time Employee
2, Mark, Contract Employee";
ChoTypeConverterFormatSpec.Instance.EnumFormat = ChoEnumFormatSpec.Description;
using (var r = ChoCSVReader.LoadText(csv)
.WithFirstLineHeader()
.WithField("Id", fieldType: typeof(int))
.WithField("Name")
.WithField("EmpType", fieldType: typeof(EmployeeType))
)
{
using (var w = new ChoParquetWriter("EnumTest.parquet"))
{
w.Write(r);
}
}
}
static void CSVArrayToParquet()
{
string csv = @"id,name,friends/0,friends/1
1,Tom,Dick,Harry";
using (var r = ChoCSVReader.LoadText(csv).WithFirstLineHeader()
.Configure(c => c.AutoArrayDiscovery = true)
.Configure(c => c.ArrayIndexSeparator = '/')
)
{
using (var w = new ChoParquetWriter("CSVArrayToParquet.parquet")
.Configure(c => c.UseNestedKeyFormat = false)
.WithField("id")
.WithField("name")
.WithField("friends", fieldType: typeof(byte[]), valueConverter: o => o.Serialize())
)
{
w.Write(r);
}
}
}
static void DataTableTest()
{
string csv = @"Id, Name
1, Tom
2, Mark";
using (var r = ChoCSVReader.LoadText(csv)
.WithFirstLineHeader()
)
{
var dt = r.AsDataTable("Emp");
using (var w = new ChoParquetWriter("datatable.parquet"))
{
w.Write(dt);
w.Close();
var s = w.Configuration.Schema;
s.Print();
}
}
}
static void Issue202()
{
using (var r = new ChoJSONReader("Issue202.json")
//.UseJsonSerialization()
)
{
using (var w = new ChoParquetWriter("Issue202.parquet")
.WithMaxScanRows(3)
.ThrowAndStopOnMissingField(false)
)
{
w.Write(r);
//w.Write(r.Select(rec1 => rec1.FlattenToDictionary()));
}
//using (var w = new ChoParquetWriter<CompleteFile>("CompleteFile.parquet")
// .WithField(f => f.Samples, valueConverter: o => "x", fieldType: typeof(string))
// )
//{
// w.Write(r); //.Select(rec1 => rec1.Flatten().ToDictionary()));
//}
}
}
static void ReadNWrite()
{
List <Employee> objs = new List <Employee>();
objs.Add(new Employee()
{
Id = 1, Name = "Tom"
});
objs.Add(new Employee()
{
Id = 2, Name = "Mark"
});
using (var parser = new ChoParquetWriter("Emp.parquet"))
{
parser.Write(objs);
}
foreach (var e in new ChoParquetReader("Emp.parquet"))
{
Console.WriteLine("Id: " + e.Id + " Name: " + e.Name);
}
}
static void JSON2Parquet1()
{
string json = @"{
""facilities"": [
{
""id"": 39205,
""name"": ""Sample1"",
""uuid"": ""ac2f3464-c425-4063-86ad-163521b1d610"",
""createdAt"": ""2019-03-06T14:25:32Z"",
""updatedAt"": ""2019-03-06T14:29:31Z"",
""active"": true
},
{
""id"": 35907,
""name"": ""Sample2"",
""uuid"": ""d371debb-f030-4c1e-b198-5eb562ceac0f"",
""createdAt"": ""2019-02-21T09:33:25Z"",
""updatedAt"": ""2019-02-21T09:33:25Z"",
""active"": true
}
]
}
";
using (var r = ChoJSONReader <Facility> .LoadText(json)
.WithJSONPath("$..facilities", false)
)
{
var x = r.Select(r1 => { r1.Location = new Point(100); return(r1); }).ToArray();
using (var w = new ChoParquetWriter <Facility>("JSON2Parquet1.parquet")
.IgnoreField("Location.IsEmpty")
)
{
w.Write(x);
}
}
}
static void Issue167()
{
var completeFile = new CompleteFile
{
DataSource = "DataSource",
DataType = 1,
Samples = new Samples
{
Samples1 = new List <Data>
{
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
},
Samples2 = new List <Data>
{
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
},
Samples3 = new List <Data>
{
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
},
Sample4 = new Data
{
Prop00 = 0,
Prop01 = "Prop01",
Properties = new Properties
{
Prop10 = 1,
Prop11 = 2,
}
},
}
};
StringBuilder json = new StringBuilder();
using (var w = new ChoJSONWriter <CompleteFile>(json)
)
{
w.Write(completeFile);
}
json.Print();
using (var r = ChoJSONReader <CompleteFile> .LoadText(json.ToString())
.UseJsonSerialization()
)
{
using (var w = new ChoParquetWriter("CompleteFile.parquet")
)
{
w.Write(r.Select(rec1 => rec1.FlattenToDictionary()));
}
//using (var w = new ChoParquetWriter<CompleteFile>("CompleteFile.parquet")
// .WithField(f => f.Samples, valueConverter: o => "x", fieldType: typeof(string))
// )
//{
// w.Write(r); //.Select(rec1 => rec1.Flatten().ToDictionary()));
//}
}
}
