public static async Task <IReadOnlyList <T> > ReadFileAsync <T>(string path, TCreateEntry <T> creator) { var items = new ConcurrentBag <T>(); using (var sr = new StreamReader(path)) { await ReadStream(sr.BaseStream, creator, items); } return(items.ToArray()); }
private static Task CreateBufferReadTask <T>(CSVBuffer buffer, TCreateEntry <T> creator, ConcurrentBag <T> items) { return(Task.Factory.StartNew(() => { buffer.PrepareBuffer(); // read until buffer is empty while (buffer.HasMoreData) { items.Add(creator(buffer)); } })); }
public static async Task <IReadOnlyList <T> > DownloadFileAsync <T>(Uri uri, TCreateEntry <T> creator) { var items = new ConcurrentBag <T>(); using (var client = new HttpClient()) { var stream = await client.GetStreamAsync(uri); await ReadStream(stream, creator, items); } return(items.ToArray()); }
private static async Task ReadStream <T>(Stream stream, TCreateEntry <T> creator, ConcurrentBag <T> items) { var readerTasks = new List <Task>(256); byte [] lastBuffer = null; for (; ;) { var dataBuffer = new byte [_bufferSize]; // read data from source int len = await stream.ReadAsync(dataBuffer, 0, _bufferSize); // break if we got no more data if (len == 0) { break; } // create csv buffer var buffer = new CSVBuffer(dataBuffer, lastBuffer, len); var task = CreateBufferReadTask(buffer, creator, items); readerTasks.Add(task); //CreateBufferReadTask( buffer, creator, items ).Wait( ); //readerTasks.Add( Task.FromResult( 0 ) ); // store last buffer so we can align the rows lastBuffer = dataBuffer; } // wait till all worker tasks are done await Task.WhenAll(readerTasks); }