public override void Configure(JobConf job)
{
SetKeep(job.GetFloat(GenericMRLoadGenerator.ReducePreservePercent, (float)100.0)
/ (float)100.0);
}
public MergeManagerImpl(TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS
, LocalDirAllocator localDirAllocator, Reporter reporter, CompressionCodec codec
, Type combinerClass, Task.CombineOutputCollector <K, V> combineCollector, Counters.Counter
spilledRecordsCounter, Counters.Counter reduceCombineInputCounter, Counters.Counter
mergedMapOutputsCounter, ExceptionReporter exceptionReporter, Progress mergePhase
, MapOutputFile mapOutputFile)
{
/* Maximum percentage of the in-memory limit that a single shuffle can
* consume*/
this.reduceId = reduceId;
this.jobConf = jobConf;
this.localDirAllocator = localDirAllocator;
this.exceptionReporter = exceptionReporter;
this.reporter = reporter;
this.codec = codec;
this.combinerClass = combinerClass;
this.combineCollector = combineCollector;
this.reduceCombineInputCounter = reduceCombineInputCounter;
this.spilledRecordsCounter = spilledRecordsCounter;
this.mergedMapOutputsCounter = mergedMapOutputsCounter;
this.mapOutputFile = mapOutputFile;
this.mapOutputFile.SetConf(jobConf);
this.localFS = localFS;
this.rfs = ((LocalFileSystem)localFS).GetRaw();
float maxInMemCopyUse = jobConf.GetFloat(MRJobConfig.ShuffleInputBufferPercent, MRJobConfig
.DefaultShuffleInputBufferPercent);
if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0)
{
throw new ArgumentException("Invalid value for " + MRJobConfig.ShuffleInputBufferPercent
+ ": " + maxInMemCopyUse);
}
// Allow unit tests to fix Runtime memory
this.memoryLimit = (long)(jobConf.GetLong(MRJobConfig.ReduceMemoryTotalBytes, Runtime
.GetRuntime().MaxMemory()) * maxInMemCopyUse);
this.ioSortFactor = jobConf.GetInt(MRJobConfig.IoSortFactor, 100);
float singleShuffleMemoryLimitPercent = jobConf.GetFloat(MRJobConfig.ShuffleMemoryLimitPercent
, DefaultShuffleMemoryLimitPercent);
if (singleShuffleMemoryLimitPercent <= 0.0f || singleShuffleMemoryLimitPercent >
1.0f)
{
throw new ArgumentException("Invalid value for " + MRJobConfig.ShuffleMemoryLimitPercent
+ ": " + singleShuffleMemoryLimitPercent);
}
usedMemory = 0L;
commitMemory = 0L;
this.maxSingleShuffleLimit = (long)(memoryLimit * singleShuffleMemoryLimitPercent
);
this.memToMemMergeOutputsThreshold = jobConf.GetInt(MRJobConfig.ReduceMemtomemThreshold
, ioSortFactor);
this.mergeThreshold = (long)(this.memoryLimit * jobConf.GetFloat(MRJobConfig.ShuffleMergePercent
, 0.90f));
Log.Info("MergerManager: memoryLimit=" + memoryLimit + ", " + "maxSingleShuffleLimit="
+ maxSingleShuffleLimit + ", " + "mergeThreshold=" + mergeThreshold + ", " + "ioSortFactor="
+ ioSortFactor + ", " + "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold
);
if (this.maxSingleShuffleLimit >= this.mergeThreshold)
{
throw new RuntimeException("Invalid configuration: " + "maxSingleShuffleLimit should be less than mergeThreshold "
+ "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit + "mergeThreshold: " +
this.mergeThreshold);
}
bool allowMemToMemMerge = jobConf.GetBoolean(MRJobConfig.ReduceMemtomemEnabled, false
);
if (allowMemToMemMerge)
{
this.memToMemMerger = new MergeManagerImpl.IntermediateMemoryToMemoryMerger(this,
this, memToMemMergeOutputsThreshold);
this.memToMemMerger.Start();
}
else
{
this.memToMemMerger = null;
}
this.inMemoryMerger = CreateInMemoryMerger();
this.inMemoryMerger.Start();
this.onDiskMerger = new MergeManagerImpl.OnDiskMerger(this, this);
this.onDiskMerger.Start();
this.mergePhase = mergePhase;
}