async Task MergeChunks(int chunksCount)
{
var unit = GetSortOutputSource(chunksCount);
long outputSize = 0L;
var outputBuffer = new List <string>(_outputBufferInitSize);
while (!unit.IsEmpty)
{
if (!unit.TryReadLine(out ProblemString line))
{
break;
}
outputSize += Consts.GetLineSize(line.RawString);
outputBuffer.Add(line.RawString);
if (outputSize > _mergeOutputBufferMaxSize)
{
await WriteResultPart(outputBuffer);
outputSize = 0L;
outputBuffer = new List <string>(_outputBufferInitSize);
}
}
if (outputBuffer.Count > 0)
{
await WriteResultPart(outputBuffer);
}
_writeResultPartBlock.Complete();
await _writeResultPartBlock.Completion;
}
public ChunkReader(int id, long bufferSize)
{
ID = id;
BufferSize = bufferSize;
_chunk = new StreamReader(Consts.GetChunkFileName(id));
_buffer = new Queue <ProblemString>();
Load();
}
public void Write()
{
Console.WriteLine($"Start write chunk {ID}, {_data.Count} lines");
var stopwatch = Stopwatch.StartNew();
File.WriteAllLines(Consts.GetChunkFileName(ID), _data.Select(x => x.RawString));
stopwatch.Stop();
Console.WriteLine($"Write chunk {ID} DONE in {stopwatch.Elapsed}, {_data.Count} lines");
}
async Task <int> CreateChunks()
{
int nextChunkID = 0;
using (var file = File.OpenText(_options.InputFileName))
{
long currentSize = 0L;
var lines = new List <ProblemString>();
while (true)
{
string line = file.ReadLine();
if (line != null)
{
lines.Add(new ProblemString(line));
currentSize += Consts.GetLineSize(line);
}
if ((line == null || currentSize >= _chunkSize) && lines.Count > 0)
{
while (IsChunkChainFull())
{
await Task.Delay(_defaultDelay);
}
var chunk = new ChunkWriter(nextChunkID++, lines);
_sortChunkBlock.Post(chunk);
lines = new List <ProblemString>();
currentSize = 0L;
}
if (line == null)
{
break;
}
}
}
_sortChunkBlock.Complete();
await _sortChunkBlock.Completion;
_writeChunkBlock.Complete();
await _writeChunkBlock.Completion;
return(nextChunkID);
}
/// <summary>
/// Чтение строк из файла;
/// Наверное стоило не допускать многопоточного чтения, но я не увидел особой разницы в сравнении с однопоточным чтением;
/// </summary>
void Load()
{
if (EOF)
{
return;
}
long readed = 0L;
while (readed < BufferSize)
{
var lineRaw = _chunk.ReadLine();
if (lineRaw == null)
{
_chunk.Dispose();
_chunk = null;
break;
}
var line = new ProblemString(lineRaw);
readed += Consts.GetLineSize(line.RawString);
_buffer.Enqueue(line);
}
}