private int SortBlocksSequential(RecordsReader recordsReader, int field, out RecordsBuffer firstBlock)
{
int blockIndex = 0;
firstBlock = null;
IEnumerable <RecordsBuffer> blocks = recordsReader.ReadBlocks();
foreach (RecordsBuffer block in blocks)
{
block.Sort(field);
if (blockIndex == 0)
{
if (recordsReader.IsLastBlock)
{
firstBlock = block;
return(1);
}
}
WriteBlock(block, blockIndex);
Console.WriteLine($"Block {blockIndex} sorted");
blockIndex++;
}
return(blockIndex);
}
private int SortBlocksParallel(RecordsReader recordsReader, int field, int workersCount,
out RecordsBuffer firstBlock)
{
firstBlock = null;
IEnumerator <RecordsBuffer> blocks = recordsReader.ReadBlocks().GetEnumerator();
if (!blocks.MoveNext())
{
return(0);
}
if (recordsReader.IsLastBlock)
{
firstBlock = blocks.Current;
firstBlock.Sort(field);
return(1);
}
int blockIndex = 0;
using (var blockCollection = new BlockingCollection <Tuple <RecordsBuffer, int> >(workersCount))
using (var sortCompletionCollection = new BlockingCollection <bool>(workersCount))
{
Task blocksReadingTask = Task.Factory.StartNew(() =>
{
do
{
blockCollection.Add(new Tuple <RecordsBuffer, int>(blocks.Current, blockIndex++));
sortCompletionCollection.Add(true);
} while (blocks.MoveNext());
blockCollection.CompleteAdding();
});
Task blocksSortingTask = Task.Factory.StartNew(() =>
{
List <Task> sortTasks = new List <Task>();
try
{
while (true)
{
Tuple <RecordsBuffer, int> blockAndIndex = blockCollection.Take();
Task t = StartBlockSortingTask(blockAndIndex.Item1, blockAndIndex.Item2,
field, sortCompletionCollection);
sortTasks.Add(t);
}
}
catch (InvalidOperationException)
{
// An InvalidOperationException means that Take() was called on a completed collection
}
Task.WaitAll(sortTasks.ToArray());
});
Task.WaitAll(blocksReadingTask, blocksSortingTask);
}
return(blockIndex);
}
/// <summary>
/// Reads records by blocks of specific size returning one after another.
/// </summary>
public IEnumerable <RecordsBuffer> ReadBlocks()
{
IsLastBlock = false;
RecordsBuffer recordsBuffer = new RecordsBuffer(_bufferSizeInBytes);
Record? recordToCompare = null;
IRecordFieldComparer comparer = _readWhileSameFieldValue != -1
? _recordComparer.CreateRecordFieldComparer(_readWhileSameFieldValue)
: null;
if (_hasRecordAhead)
{
recordToCompare = _recordAhead;
recordsBuffer.AddRecord(_recordAhead);
_hasRecordAhead = false;
}
string line = null;
while ((line = _streamReader.ReadLine()) != null)
{
Record record = _recordParser.Parse(line);
if (_readWhileSameFieldValue >= 0)
{
if (recordToCompare == null)
{
recordToCompare = record;
}
else if (comparer.Compare(record, recordToCompare.Value) != 0)
{
_recordAhead = record;
_hasRecordAhead = true;
break;
}
}
bool added = recordsBuffer.AddRecord(record);
if (!added)
{
if (recordsBuffer.Records.Count == 0)
{
throw new InvalidOperationException("Buffer is too small. Can't add even one record.");
}
yield return(recordsBuffer);
recordsBuffer = new RecordsBuffer(_bufferSizeInBytes);
recordsBuffer.AddRecord(record);
}
}
if (recordsBuffer.Records.Count > 0)
{
IsLastBlock = true;
yield return(recordsBuffer);
}
}
private void WriteBlock(RecordsBuffer block, int blockIndex)
{
using (Stream blockStream = _tempStreams.CreateBlockStream(blockIndex))
using (StreamWriter sw = new StreamWriter(blockStream))
{
RecordsWriter recordsWriter = new RecordsWriter(sw);
recordsWriter.WriteRecords(block);
}
}
/// <summary>
/// Writes all records from a records buffer.
/// </summary>
public void WriteRecords(RecordsBuffer buffer)
{
for (int i = 0; i < buffer.Records.Count; i++)
{
Record record = buffer.Records[i];
string recordString = _recordParser.ToString(record);
_textWriter.Write(recordString);
_textWriter.Write("\r\n");
}
}
private Task StartBlockSortingTask(RecordsBuffer block, int blockIndex, int field,
BlockingCollection <bool> sortCompletionCollection)
{
return(Task.Factory.StartNew(() =>
{
block.Sort(field);
WriteBlock(block, blockIndex);
sortCompletionCollection.Take();
Console.WriteLine($"Block {blockIndex} sorted");
}));
}
/// <summary>
/// Writes records adding them into buffer first and then when buffer is filled
/// flushed it to a stream.
/// </summary>
public void WriteRecord(Record record)
{
bool added = _recordsBuffer.AddRecord(record);
if (!added)
{
if (_recordsBuffer.Records.Count == 0)
{
throw new InvalidOperationException("Buffer is too small. Can't add even one record.");
}
Flush();
_recordsBuffer = new RecordsBuffer(_bufferSizeInBytes);
_recordsBuffer.AddRecord(record);
}
}
public BufferedRecordsWriter(TextWriter textWriter, long bufferSizeInBytes)
{
_recordsWriter = new RecordsWriter(textWriter);
_bufferSizeInBytes = bufferSizeInBytes;
_recordsBuffer = new RecordsBuffer(bufferSizeInBytes);
}