private void _Initialize()
{
// assume theyre either all sorted or all not. Also, some of these might
// be empty assuming there were empty buckets so handle that.
InternalRecordSource mergeDevice = null;
if (Sorting.IsSorted) {
mergeDevice = new SortedRecordMerger();
}
else {
// is it right to hard code sortAscending!?!?
mergeDevice = new RecordSorterReducer(_tempDir, true, _reductionEnabled);
}
// add the sources
foreach (InternalRecordSource source in Inputs) {
mergeDevice.AddInput(source);
}
_output = mergeDevice;
}
/// <summary>
/// Method for adding records to a TStore. (Push model).
/// </summary>
/// <param name="record">Record to be added.</param>
public void AddRecord(DataRecord record)
{
if (_sorterReducer == null) {
if (_input != null) {
throw new Exception("cannot use AddRecord method together with InternalRecordSource");
}
_sorterReducer = new RecordSorterReducer(TempDir, true, reductionEnabled);
if (maxMemorySize != -1)
_sorterReducer.MaxMemorySize = maxMemorySize;
}
_sorterReducer.AddRecord(record);
}
private void _SortReduce(bool sortAscending, bool reductionEnabled)
{
// lets change sort needs to a number for easy comparison.
// no sort = 0, sort ascending = 1, sort descending = 2
int askSortNum = 1; // we are asking for sorting on our output
if (!sortAscending) askSortNum = 2;
bool inputIsSorted = InternalSource.Sorting.IsSorted;
bool inputIsSortedAscending = InternalSource.Sorting.IsSortedAscending;
int inputSortNum = 0;
if (inputIsSorted)
inputSortNum = 1;
if (inputIsSorted && !inputIsSortedAscending)
inputSortNum = 2;
// (aside: one might ask, why not just have a separate sorter
// and reducer. Reduction is combined in the sorter so that
// reduction can happen in memory before temp files are written
// to disk.
if (askSortNum == inputSortNum && !reductionEnabled) return; // don't insert a sorter or reducer
if (askSortNum != inputSortNum) {
RecordSorterReducer sr = new RecordSorterReducer(Processor.TempDir, sortAscending, reductionEnabled);
sr.MaxMemorySize = Processor.MaxMemorySize;
sr.DeleteTempFiles = Processor.DeleteTempFiles;
InternalRecordSource temp = InternalSource; // grab our input
sr.AddInput(temp); // pipe it into the sorterReducer
this.InternalSource = sr; // make the sorterReducer the output of this source
}
else {
IRecordFilter reducer = new ReduceFilter();
RecordFilterDriver driver = new RecordFilterDriver(reducer);
driver.AddInput(InternalSource);
InternalSource = driver;
}
}
