private async Task <IEnumerable <IChunkReference <T> > > PushChunksToStackAndMerge(ChannelReader <List <T> > channelReader)
{
try
{
var stopwatch = Stopwatch.StartNew();
var chunkStack = _chunkStackFactory.CreateChunkStack();
var tempChunStack = _chunkStackFactory.CreateChunkStack();
var appender = new ChunkStackAppender(chunkStack, tempChunStack);
while (await channelReader.WaitToReadAsync().ConfigureAwait(false))
{
if (channelReader.TryRead(out var chunk))
{
_logger.Debug($"Starting to merge chunk with {chunk.Count} lines");
appender.PushToStackRecursively(chunk);
_logger.Debug($"Chunk was merged in {stopwatch.Elapsed}");
stopwatch.Restart();
}
}
if (_onlyMemoryMerge)
{
return(chunkStack.ToArray().Concat(tempChunStack.ToArray()));
}
return(new[] { appender.ExecuteFinalMerge() });
}
catch (Exception ex)
{
_logger.ErrorException("Error occured while merging: ", ex);
throw;
}
}
protected MergeSortBase(ChunkStack <T> chunkStack, ChunkStack <T> tempChunkStack, bool onlyMemoryMerge)
{
_appender = new ChunkStackAppender(chunkStack, tempChunkStack, onlyMemoryMerge);
_tempChunkStack = tempChunkStack;
_chunkStack = chunkStack;
}
