private int SortBlocksSequential(RecordsReader recordsReader, int field, out RecordsBuffer firstBlock) { int blockIndex = 0; firstBlock = null; IEnumerable <RecordsBuffer> blocks = recordsReader.ReadBlocks(); foreach (RecordsBuffer block in blocks) { block.Sort(field); if (blockIndex == 0) { if (recordsReader.IsLastBlock) { firstBlock = block; return(1); } } WriteBlock(block, blockIndex); Console.WriteLine($"Block {blockIndex} sorted"); blockIndex++; } return(blockIndex); }
private int SortBlocksParallel(RecordsReader recordsReader, int field, int workersCount, out RecordsBuffer firstBlock) { firstBlock = null; IEnumerator <RecordsBuffer> blocks = recordsReader.ReadBlocks().GetEnumerator(); if (!blocks.MoveNext()) { return(0); } if (recordsReader.IsLastBlock) { firstBlock = blocks.Current; firstBlock.Sort(field); return(1); } int blockIndex = 0; using (var blockCollection = new BlockingCollection <Tuple <RecordsBuffer, int> >(workersCount)) using (var sortCompletionCollection = new BlockingCollection <bool>(workersCount)) { Task blocksReadingTask = Task.Factory.StartNew(() => { do { blockCollection.Add(new Tuple <RecordsBuffer, int>(blocks.Current, blockIndex++)); sortCompletionCollection.Add(true); } while (blocks.MoveNext()); blockCollection.CompleteAdding(); }); Task blocksSortingTask = Task.Factory.StartNew(() => { List <Task> sortTasks = new List <Task>(); try { while (true) { Tuple <RecordsBuffer, int> blockAndIndex = blockCollection.Take(); Task t = StartBlockSortingTask(blockAndIndex.Item1, blockAndIndex.Item2, field, sortCompletionCollection); sortTasks.Add(t); } } catch (InvalidOperationException) { // An InvalidOperationException means that Take() was called on a completed collection } Task.WaitAll(sortTasks.ToArray()); }); Task.WaitAll(blocksReadingTask, blocksSortingTask); } return(blockIndex); }
/// <summary> /// Reads records by blocks of specific size returning one after another. /// </summary> public IEnumerable <RecordsBuffer> ReadBlocks() { IsLastBlock = false; RecordsBuffer recordsBuffer = new RecordsBuffer(_bufferSizeInBytes); Record? recordToCompare = null; IRecordFieldComparer comparer = _readWhileSameFieldValue != -1 ? _recordComparer.CreateRecordFieldComparer(_readWhileSameFieldValue) : null; if (_hasRecordAhead) { recordToCompare = _recordAhead; recordsBuffer.AddRecord(_recordAhead); _hasRecordAhead = false; } string line = null; while ((line = _streamReader.ReadLine()) != null) { Record record = _recordParser.Parse(line); if (_readWhileSameFieldValue >= 0) { if (recordToCompare == null) { recordToCompare = record; } else if (comparer.Compare(record, recordToCompare.Value) != 0) { _recordAhead = record; _hasRecordAhead = true; break; } } bool added = recordsBuffer.AddRecord(record); if (!added) { if (recordsBuffer.Records.Count == 0) { throw new InvalidOperationException("Buffer is too small. Can't add even one record."); } yield return(recordsBuffer); recordsBuffer = new RecordsBuffer(_bufferSizeInBytes); recordsBuffer.AddRecord(record); } } if (recordsBuffer.Records.Count > 0) { IsLastBlock = true; yield return(recordsBuffer); } }
private void WriteBlock(RecordsBuffer block, int blockIndex) { using (Stream blockStream = _tempStreams.CreateBlockStream(blockIndex)) using (StreamWriter sw = new StreamWriter(blockStream)) { RecordsWriter recordsWriter = new RecordsWriter(sw); recordsWriter.WriteRecords(block); } }
/// <summary> /// Writes all records from a records buffer. /// </summary> public void WriteRecords(RecordsBuffer buffer) { for (int i = 0; i < buffer.Records.Count; i++) { Record record = buffer.Records[i]; string recordString = _recordParser.ToString(record); _textWriter.Write(recordString); _textWriter.Write("\r\n"); } }
private Task StartBlockSortingTask(RecordsBuffer block, int blockIndex, int field, BlockingCollection <bool> sortCompletionCollection) { return(Task.Factory.StartNew(() => { block.Sort(field); WriteBlock(block, blockIndex); sortCompletionCollection.Take(); Console.WriteLine($"Block {blockIndex} sorted"); })); }
/// <summary> /// Writes records adding them into buffer first and then when buffer is filled /// flushed it to a stream. /// </summary> public void WriteRecord(Record record) { bool added = _recordsBuffer.AddRecord(record); if (!added) { if (_recordsBuffer.Records.Count == 0) { throw new InvalidOperationException("Buffer is too small. Can't add even one record."); } Flush(); _recordsBuffer = new RecordsBuffer(_bufferSizeInBytes); _recordsBuffer.AddRecord(record); } }
public BufferedRecordsWriter(TextWriter textWriter, long bufferSizeInBytes) { _recordsWriter = new RecordsWriter(textWriter); _bufferSizeInBytes = bufferSizeInBytes; _recordsBuffer = new RecordsBuffer(bufferSizeInBytes); }