Ejemplo n.º 1
0
        private void _Initialize()
        {
            // assume theyre either all sorted or all not.  Also, some of these might
            // be empty assuming there were empty buckets so handle that.

            InternalRecordSource mergeDevice = null;

            if (Sorting.IsSorted) {
                mergeDevice = new SortedRecordMerger();
            }

            else {
                // is it right to hard code sortAscending!?!?
                mergeDevice = new RecordSorterReducer(_tempDir, true, _reductionEnabled);
            }

            // add the sources
            foreach (InternalRecordSource source in Inputs) {
                mergeDevice.AddInput(source);
            }

            _output = mergeDevice;
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Method for adding records to a TStore.  (Push model).
        /// </summary>
        /// <param name="record">Record to be added.</param>
        public void AddRecord(DataRecord record)
        {
            if (_sorterReducer == null) {
                if (_input != null) {
                    throw new Exception("cannot use AddRecord method together with InternalRecordSource");
                }
                _sorterReducer = new RecordSorterReducer(TempDir, true, reductionEnabled);
                if (maxMemorySize != -1)
                    _sorterReducer.MaxMemorySize = maxMemorySize;
            }

            _sorterReducer.AddRecord(record);
        }
Ejemplo n.º 3
0
        private void _SortReduce(bool sortAscending, bool reductionEnabled)
        {
            // lets change sort needs to a number for easy comparison.
            // no sort = 0, sort ascending = 1, sort descending = 2

            int askSortNum = 1; // we are asking for sorting on our output
            if (!sortAscending) askSortNum = 2;

            bool inputIsSorted = InternalSource.Sorting.IsSorted;
            bool inputIsSortedAscending = InternalSource.Sorting.IsSortedAscending;

            int inputSortNum = 0;
            if (inputIsSorted)
                inputSortNum = 1;
            if (inputIsSorted && !inputIsSortedAscending)
                inputSortNum = 2;

            // (aside: one might ask, why not just have a separate sorter
            // and reducer.  Reduction is combined in the sorter so that
            // reduction can happen in memory before temp files are written
            // to disk.

            if (askSortNum == inputSortNum && !reductionEnabled) return; // don't insert a sorter or reducer

            if (askSortNum != inputSortNum) {
                RecordSorterReducer sr = new RecordSorterReducer(Processor.TempDir, sortAscending, reductionEnabled);
                sr.MaxMemorySize = Processor.MaxMemorySize;
                sr.DeleteTempFiles = Processor.DeleteTempFiles;

                InternalRecordSource temp = InternalSource; // grab our input
                sr.AddInput(temp); // pipe it into the sorterReducer
                this.InternalSource = sr; // make the sorterReducer the output of this source
            }

            else {
                IRecordFilter reducer = new ReduceFilter();
                RecordFilterDriver driver = new RecordFilterDriver(reducer);
                driver.AddInput(InternalSource);
                InternalSource = driver;
            }
        }