public static async Task <IReadOnlyList <T> > ReadFileAsync <T>(string path, TCreateEntry <T> creator)
{
var items = new ConcurrentBag <T>();
using (var sr = new StreamReader(path))
{
await ReadStream(sr.BaseStream, creator, items);
}
return(items.ToArray());
}
private static Task CreateBufferReadTask <T>(CSVBuffer buffer, TCreateEntry <T> creator, ConcurrentBag <T> items)
{
return(Task.Factory.StartNew(() =>
{
buffer.PrepareBuffer();
// read until buffer is empty
while (buffer.HasMoreData)
{
items.Add(creator(buffer));
}
}));
}
public static async Task <IReadOnlyList <T> > DownloadFileAsync <T>(Uri uri, TCreateEntry <T> creator)
{
var items = new ConcurrentBag <T>();
using (var client = new HttpClient())
{
var stream = await client.GetStreamAsync(uri);
await ReadStream(stream, creator, items);
}
return(items.ToArray());
}
private static async Task ReadStream <T>(Stream stream, TCreateEntry <T> creator, ConcurrentBag <T> items)
{
var readerTasks = new List <Task>(256);
byte [] lastBuffer = null;
for (; ;)
{
var dataBuffer = new byte [_bufferSize];
// read data from source
int len = await stream.ReadAsync(dataBuffer, 0, _bufferSize);
// break if we got no more data
if (len == 0)
{
break;
}
// create csv buffer
var buffer = new CSVBuffer(dataBuffer, lastBuffer, len);
var task = CreateBufferReadTask(buffer, creator, items);
readerTasks.Add(task);
//CreateBufferReadTask( buffer, creator, items ).Wait( );
//readerTasks.Add( Task.FromResult( 0 ) );
// store last buffer so we can align the rows
lastBuffer = dataBuffer;
}
// wait till all worker tasks are done
await Task.WhenAll(readerTasks);
}
