public void AddRecord(DataRecord record)
{
if (_recordsToSort == null) {
_recordsToSort = new SortableRecordMemory(_internalSortAscending, _sortableRecordMaxMemorySize);
}
// attempt to add the new record
bool success = _recordsToSort.AddRecord(record);
// see if we need to reduce by setting the _filter or leaving it null
if (!_reductionDetermined)
{
if (!(record is IReducableRecord))
{
_internalReductionEnabled = false;
}
_reductionDetermined = true;
}
//if (inputInstance == null) inputInstance = input.CurrentRecord;
// rats! not enough memory. Sort and write temp file, then add record to new sorter
if (!success)
{
if (_doThreading) _SortAndWriteThreaded();
else _SortAndWrite();
// try again
AddRecord(record);
}
}
private void _SortAndWrite()
{
_recordsToSort.Sort();
_recordsToWrite = _recordsToSort;
_WriteSortedRecords();
}
private void _SortAndWriteThreaded()
{
// threading model looks like this:
// MainThread: | Ri | Rj | | Rk | | Rl
// SortThread: | Si | Sj | Sk
// WriteThread: | Wi | | Wj
// wait on all previous threads, sorts and writes
foreach (Thread t in _threads) {
t.Join();
}
_threads.Clear();
SortableRecordMemory writtenRecords = _recordsToWrite;
_recordsToWrite = _sortedRecords;
_sortedRecords = _recordsToSort;
ThreadStart sortJob = new ThreadStart(_sortedRecords.Sort);
Thread tt = new Thread(sortJob);
tt.Start();
_threads.Add(tt);
// write sorted records
if (_recordsToWrite != null) {
ThreadStart writeJob = new ThreadStart(_WriteSortedRecords);
tt = new Thread(writeJob);
tt.Start();
_threads.Add(tt);
}
_recordsToSort = writtenRecords;
if (_recordsToSort != null) _recordsToSort.Clear();
}
private void _Finish()
{
if (_doThreading) {
_SortAndWriteThreaded();
// wait on all previous threads, sorts and writes
foreach (Thread t in _threads) {
t.Join();
}
_threads.Clear();
}
else {
// don't write the last one to disk
_recordsToSort.Sort();
_sortedRecords = _recordsToSort;
}
int numToMerge = _numTempFiles;
if (_sortedRecords != null && _sortedRecords.NumRecords != 0) {
numToMerge++;
}
if (numToMerge > 1) {
_merger = new SortedRecordMerger();
if (_sortedRecords != null && _sortedRecords.NumRecords != 0) {
_merger.AddInput(_sortedRecords);
}
for (int i = 0; i < _numTempFiles; i++) {
InternalRecordSource source = new InternalRecordFileReader(GetTempFilename(i));
_merger.AddInput(source);
}
_output = _merger;
}
else {
_output = _sortedRecords;
}
// set up reduction filter for reduction across merged sources
if (_internalReductionEnabled) {
ReduceFilter reducer = new ReduceFilter();
RecordFilterDriver filterDriver = new RecordFilterDriver(reducer);
filterDriver.AddInput(_output);
_output = filterDriver;
}
// this is kind of a hack till i figure out how these should be set
TotalRecordBytesEstimate = _output.TotalRecordBytesEstimate;
TotalRecordsEstimate = _output.TotalRecordsEstimate;
}
