private void MergeBlocks(List <IEnumerator <Record> > blockRecordsEnumerators, BufferedRecordsWriter recordsWriter, int field) { IRecordFieldComparer comparer = _recordComparer.CreateRecordFieldComparer(field); var currentRecordComparer = new SelectComparer <int, Record>( i => blockRecordsEnumerators[i].Current, comparer); var heap = new MinHeap <int>(blockRecordsEnumerators.Count, currentRecordComparer); for (int i = 0; i < blockRecordsEnumerators.Count; i++) { blockRecordsEnumerators[i].MoveNext(); heap.Add(i); } while (heap.Size > 0) { recordsWriter.WriteRecord(blockRecordsEnumerators[heap.Root].Current); bool empty = !blockRecordsEnumerators[heap.Root].MoveNext(); if (empty) { heap.Extract(); } else { heap.DownHeap(); } } }
/// <summary> /// Reads records by blocks of specific size returning one after another. /// </summary> public IEnumerable <RecordsBuffer> ReadBlocks() { IsLastBlock = false; RecordsBuffer recordsBuffer = new RecordsBuffer(_bufferSizeInBytes); Record? recordToCompare = null; IRecordFieldComparer comparer = _readWhileSameFieldValue != -1 ? _recordComparer.CreateRecordFieldComparer(_readWhileSameFieldValue) : null; if (_hasRecordAhead) { recordToCompare = _recordAhead; recordsBuffer.AddRecord(_recordAhead); _hasRecordAhead = false; } string line = null; while ((line = _streamReader.ReadLine()) != null) { Record record = _recordParser.Parse(line); if (_readWhileSameFieldValue >= 0) { if (recordToCompare == null) { recordToCompare = record; } else if (comparer.Compare(record, recordToCompare.Value) != 0) { _recordAhead = record; _hasRecordAhead = true; break; } } bool added = recordsBuffer.AddRecord(record); if (!added) { if (recordsBuffer.Records.Count == 0) { throw new InvalidOperationException("Buffer is too small. Can't add even one record."); } yield return(recordsBuffer); recordsBuffer = new RecordsBuffer(_bufferSizeInBytes); recordsBuffer.AddRecord(record); } } if (recordsBuffer.Records.Count > 0) { IsLastBlock = true; yield return(recordsBuffer); } }
/// <summary> /// Sorts records by specified field. /// </summary> /// <param name="field">Field index by which sorting will happen.</param> public void Sort(int field) { IRecordFieldComparer comparer = _recordComparer.CreateRecordFieldComparer(field); Records = Records.ParallelSort(comparer); }