/// <exception cref="System.IO.IOException"/>
private void CombineAndSpill(RawKeyValueIterator kvIter, Counters.Counter inCounter
)
{
JobConf job = jobConf;
Reducer combiner = ReflectionUtils.NewInstance(combinerClass, job);
Type keyClass = (Type)job.GetMapOutputKeyClass();
Type valClass = (Type)job.GetMapOutputValueClass();
RawComparator <K> comparator = (RawComparator <K>)job.GetCombinerKeyGroupingComparator
();
try
{
Task.CombineValuesIterator values = new Task.CombineValuesIterator(kvIter, comparator
, keyClass, valClass, job, Reporter.Null, inCounter);
while (values.More())
{
combiner.Reduce(values.GetKey(), values, combineCollector, Reporter.Null);
values.NextKey();
}
}
finally
{
combiner.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void RunOldReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol
umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator
<INKEY> comparator)
{
System.Type keyClass = typeof(INKEY);
System.Type valueClass = typeof(INVALUE);
Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = ReflectionUtils.NewInstance(job
.GetReducerClass(), job);
// make output collector
string finalName = GetOutputName(GetPartition());
RecordWriter <OUTKEY, OUTVALUE> @out = new ReduceTask.OldTrackingRecordWriter <OUTKEY
, OUTVALUE>(this, job, reporter, finalName);
RecordWriter <OUTKEY, OUTVALUE> finalOut = @out;
OutputCollector <OUTKEY, OUTVALUE> collector = new _OutputCollector_419(finalOut,
reporter);
// indicate that progress update needs to be sent
// apply reduce function
try
{
//increment processed counter only if skipping feature is enabled
bool incrProcCount = SkipBadRecords.GetReducerMaxSkipGroups(job) > 0 && SkipBadRecords
.GetAutoIncrReducerProcCount(job);
ReduceTask.ReduceValuesIterator <INKEY, INVALUE> values = IsSkipping() ? new ReduceTask.SkippingReduceValuesIterator
<INKEY, INVALUE>(this, rIter, comparator, keyClass, valueClass, job, reporter, umbilical
) : new ReduceTask.ReduceValuesIterator <INKEY, INVALUE>(this, rIter, job.GetOutputValueGroupingComparator
(), keyClass, valueClass, job, reporter);
values.InformReduceProgress();
while (values.More())
{
reduceInputKeyCounter.Increment(1);
reducer.Reduce(values.GetKey(), values, collector, reporter);
if (incrProcCount)
{
reporter.IncrCounter(SkipBadRecords.CounterGroup, SkipBadRecords.CounterReduceProcessedGroups
, 1);
}
values.NextKey();
values.InformReduceProgress();
}
reducer.Close();
reducer = null;
@out.Close(reporter);
@out = null;
}
finally
{
IOUtils.Cleanup(Log, reducer);
CloseQuietly(@out, reporter);
}
}