static void StageLargeFile() { int c = 0; var parser = new ChoCSVLiteReader(); Stopwatch w2 = Stopwatch.StartNew(); using (var r = new StreamReader(@"..\..\..\..\..\..\data\XBTUSD.csv")) { foreach (var rec in parser.Read(r, ',').Take(1000000)) { //rec.Print(); c++; if (c % 100000 == 0) { $"Rows loaded: {c}".Print(); } } } w2.Stop(); $"StreamReader: {w2.Elapsed}".Print(); //ChoTypeDescriptor.DoNotUseTypeConverterForTypes = true; for (int i = 2; i < 3; i++) { var watch = Stopwatch.StartNew(); List <Trade> trades = null; using (var r = new ChoCSVReader(@"..\..\..\..\..\..\data\XBTUSD.csv") .Configure(c => c.NotifyAfter = 100000) .Setup(s => s.RowsLoaded += (o, e) => { $"Rows Loaded: {e.RowsLoaded} <-- {DateTime.Now}".Print(); }) .Configure(c => c.LiteParsing = true) ) { //r.Take(1).Print(); //return; var trades1 = r.Take(1000000).ToList(); //.Count().Print(); //return; //r.Take(1000000).StageOnSQLite(new ChoETLSqliteSettings() // .Configure(c => c.ConnectionString = "DataSource=local.db;Version=3;Synchronous=OFF;Journal Mode=OFF") // .Configure(c => c.NotifyAfter = 500000) // .Configure(c => c.BatchSize = 500000) // .Configure(c => c.RowsUploaded += (o, e) => // { // Console.WriteLine($"Rows uploaded: {e.RowsUploaded}"); // })); //trades = r.Take(1000000).Select(r => new Trade { Id = r.Column1, Price = r.Column2, Quantity = r.Column3 }).ToList(); } watch.Stop(); watch.Elapsed.Print(); return; watch = Stopwatch.StartNew(); trades.StageOnSQLite(new ChoETLSqliteSettings() .Configure(c => c.ConnectionString = "DataSource=local.db;Version=3;Synchronous=OFF;Journal Mode=OFF") .Configure(c => c.NotifyAfter = 500000) .Configure(c => c.BatchSize = 500000) .Configure(c => c.RowsUploaded += (o, e) => { Console.WriteLine($"Rows uploaded: {e.RowsUploaded}"); })); watch.Stop(); watch.Elapsed.Print(); } }
static void StageLargeFile() { //ChoTypeDescriptor.DoNotUseTypeConverterForTypes = true; for (int i = 0; i < 5; i++) { var watch = Stopwatch.StartNew(); List <Trade> trades = null; //using (MemoryMappedFile memoryMappedFile = MemoryMappedFile.CreateFromFile(@"..\..\..\..\..\..\data\XBTUSD.csv")) //using (MemoryMappedViewStream memoryMappedViewStream = memoryMappedFile.CreateViewStream(0, 0, MemoryMappedFileAccess.Read)) //{ // using (var r = new ChoCSVReader<Trade>(memoryMappedViewStream) // .Configure(c => c.NotifyAfter = 100000) // .Setup(s => s.RowsLoaded += (o, e) => // { // $"Rows Loaded: {e.RowsLoaded} <-- {DateTime.Now}".Print(); // }) // .Configure(c => c.LiteParsing = true) // ) // { // trades = r.Take(100000).ToList(); //.Count().Print(); // } //} using (var r = new ChoCSVReader <Trade>(@"..\..\..\..\..\..\data\XBTUSD.csv") .Configure(c => c.NotifyAfter = 100000) .Setup(s => s.RowsLoaded += (o, e) => { $"Rows Loaded: {e.RowsLoaded} <-- {DateTime.Now}".Print(); }) .Configure(c => c.LiteParsing = true) //.Configure(c => c.BufferSize = 1024 * 1024) //.Configure(c => c.TurnOffMemoryMappedFile = true) ) { //r.Take(1).Print(); //return; trades = r.Take(1000000).ToList(); //.Count().Print(); //return; //r.Take(1000000).StageOnSQLite(new ChoETLSqliteSettings() // .Configure(c => c.ConnectionString = "DataSource=local.db;Version=3;Synchronous=OFF;Journal Mode=OFF") // .Configure(c => c.NotifyAfter = 500000) // .Configure(c => c.BatchSize = 500000) // .Configure(c => c.RowsUploaded += (o, e) => // { // Console.WriteLine($"Rows uploaded: {e.RowsUploaded}"); // })); //trades = r.Take(1000000).Select(r => new Trade { Id = r.Column1, Price = r.Column2, Quantity = r.Column3 }).ToList(); } watch.Stop(); watch.Elapsed.Print(); //watch = Stopwatch.StartNew(); //trades.StageOnSqlServer(new ChoETLSqlServerSettings() // //.Configure(c => c.ConnectionString = "DataSource=local.db;Version=3;Synchronous=OFF;Journal Mode=OFF") // .Configure(c => c.NotifyAfter = 500000) // //.Configure(c => c.BatchSize = 500000) // .Configure(c => c.RowsUploaded += (o, e) => // { // Console.WriteLine($"Rows uploaded: {e.RowsUploaded}"); // })); //watch.Stop(); //watch.Elapsed.Print(); } }