private void _Initialize() { // assume theyre either all sorted or all not. Also, some of these might // be empty assuming there were empty buckets so handle that. InternalRecordSource mergeDevice = null; if (Sorting.IsSorted) { mergeDevice = new SortedRecordMerger(); } else { // is it right to hard code sortAscending!?!? mergeDevice = new RecordSorterReducer(_tempDir, true, _reductionEnabled); } // add the sources foreach (InternalRecordSource source in Inputs) { mergeDevice.AddInput(source); } _output = mergeDevice; }
/// <summary> /// Method for adding records to a TStore. (Push model). /// </summary> /// <param name="record">Record to be added.</param> public void AddRecord(DataRecord record) { if (_sorterReducer == null) { if (_input != null) { throw new Exception("cannot use AddRecord method together with InternalRecordSource"); } _sorterReducer = new RecordSorterReducer(TempDir, true, reductionEnabled); if (maxMemorySize != -1) _sorterReducer.MaxMemorySize = maxMemorySize; } _sorterReducer.AddRecord(record); }
private void _SortReduce(bool sortAscending, bool reductionEnabled) { // lets change sort needs to a number for easy comparison. // no sort = 0, sort ascending = 1, sort descending = 2 int askSortNum = 1; // we are asking for sorting on our output if (!sortAscending) askSortNum = 2; bool inputIsSorted = InternalSource.Sorting.IsSorted; bool inputIsSortedAscending = InternalSource.Sorting.IsSortedAscending; int inputSortNum = 0; if (inputIsSorted) inputSortNum = 1; if (inputIsSorted && !inputIsSortedAscending) inputSortNum = 2; // (aside: one might ask, why not just have a separate sorter // and reducer. Reduction is combined in the sorter so that // reduction can happen in memory before temp files are written // to disk. if (askSortNum == inputSortNum && !reductionEnabled) return; // don't insert a sorter or reducer if (askSortNum != inputSortNum) { RecordSorterReducer sr = new RecordSorterReducer(Processor.TempDir, sortAscending, reductionEnabled); sr.MaxMemorySize = Processor.MaxMemorySize; sr.DeleteTempFiles = Processor.DeleteTempFiles; InternalRecordSource temp = InternalSource; // grab our input sr.AddInput(temp); // pipe it into the sorterReducer this.InternalSource = sr; // make the sorterReducer the output of this source } else { IRecordFilter reducer = new ReduceFilter(); RecordFilterDriver driver = new RecordFilterDriver(reducer); driver.AddInput(InternalSource); InternalSource = driver; } }