/// <summary>
/// Ensure that M/R 1.x applications can get and set task virtual memory with
/// old property names
/// </summary>
public virtual void TestDeprecatedPropertyNameForTaskVmem()
{
JobConf configuration = new JobConf();
configuration.SetLong(JobConf.MapredJobMapMemoryMbProperty, 1024);
configuration.SetLong(JobConf.MapredJobReduceMemoryMbProperty, 1024);
NUnit.Framework.Assert.AreEqual(1024, configuration.GetMemoryForMapTask());
NUnit.Framework.Assert.AreEqual(1024, configuration.GetMemoryForReduceTask());
// Make sure new property names aren't broken by the old ones
configuration.SetLong(JobConf.MapreduceJobMapMemoryMbProperty, 1025);
configuration.SetLong(JobConf.MapreduceJobReduceMemoryMbProperty, 1025);
NUnit.Framework.Assert.AreEqual(1025, configuration.GetMemoryForMapTask());
NUnit.Framework.Assert.AreEqual(1025, configuration.GetMemoryForReduceTask());
configuration.SetMemoryForMapTask(2048);
configuration.SetMemoryForReduceTask(2048);
NUnit.Framework.Assert.AreEqual(2048, configuration.GetLong(JobConf.MapredJobMapMemoryMbProperty
, -1));
NUnit.Framework.Assert.AreEqual(2048, configuration.GetLong(JobConf.MapredJobReduceMemoryMbProperty
, -1));
// Make sure new property names aren't broken by the old ones
NUnit.Framework.Assert.AreEqual(2048, configuration.GetLong(JobConf.MapreduceJobMapMemoryMbProperty
, -1));
NUnit.Framework.Assert.AreEqual(2048, configuration.GetLong(JobConf.MapreduceJobReduceMemoryMbProperty
, -1));
}
/// <summary>When no input dir is specified, generate random data.</summary>
/// <exception cref="System.IO.IOException"/>
protected internal static void ConfRandom(JobConf job)
{
// from RandomWriter
job.SetInputFormat(typeof(GenericMRLoadGenerator.RandomInputFormat));
job.SetMapperClass(typeof(GenericMRLoadGenerator.RandomMapOutput));
ClusterStatus cluster = new JobClient(job).GetClusterStatus();
int numMapsPerHost = job.GetInt(RandomTextWriter.MapsPerHost, 10);
long numBytesToWritePerMap = job.GetLong(RandomTextWriter.BytesPerMap, 1 * 1024 *
1024 * 1024);
if (numBytesToWritePerMap == 0)
{
throw new IOException("Cannot have " + RandomTextWriter.BytesPerMap + " set to 0"
);
}
long totalBytesToWrite = job.GetLong(RandomTextWriter.TotalBytes, numMapsPerHost
* numBytesToWritePerMap * cluster.GetTaskTrackers());
int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
if (numMaps == 0 && totalBytesToWrite > 0)
{
numMaps = 1;
job.SetLong(RandomTextWriter.BytesPerMap, totalBytesToWrite);
}
job.SetNumMapTasks(numMaps);
}
// fill keys, values with ~1.5 blocks for block-compressed seq fill
private static void FillBlocks(JobConf conf)
{
Random r = new Random();
long seed = conf.GetLong("filebench.seed", -1);
if (seed > 0)
{
r.SetSeed(seed);
}
int keylen = conf.GetInt("filebench.key.words", 5);
int vallen = conf.GetInt("filebench.val.words", 20);
int acc = (3 * conf.GetInt("io.seqfile.compress.blocksize", 1000000)) >> 1;
AList <string> k = new AList <string>();
AList <string> v = new AList <string>();
for (int i = 0; acc > 0; ++i)
{
string s = GenerateSentence(r, keylen);
acc -= s.Length;
k.AddItem(s);
s = GenerateSentence(r, vallen);
acc -= s.Length;
v.AddItem(s);
}
keys = Sharpen.Collections.ToArray(k, new string[0]);
values = Sharpen.Collections.ToArray(v, new string[0]);
}
/// <exception cref="System.Exception"/>
public virtual void TestNumInputs()
{
JobConf job = new JobConf(conf);
dfs = NewDFSCluster(job);
FileSystem fs = dfs.GetFileSystem();
System.Console.Out.WriteLine("FileSystem " + fs.GetUri());
Path inputDir = new Path("/foo/");
int numFiles = 10;
string fileNameBase = "part-0000";
for (int i = 0; i < numFiles; ++i)
{
CreateInputs(fs, inputDir, fileNameBase + i.ToString());
}
CreateInputs(fs, inputDir, "_meta");
CreateInputs(fs, inputDir, "_temp");
// split it using a file input format
TextInputFormat.AddInputPath(job, inputDir);
TextInputFormat inFormat = new TextInputFormat();
inFormat.Configure(job);
InputSplit[] splits = inFormat.GetSplits(job, 1);
NUnit.Framework.Assert.AreEqual("Expected value of " + FileInputFormat.NumInputFiles
, numFiles, job.GetLong(FileInputFormat.NumInputFiles, 0));
}
public ShuffleSchedulerImpl(JobConf job, TaskStatus status, TaskAttemptID reduceId
, ExceptionReporter reporter, Progress progress, Counters.Counter shuffledMapsCounter
, Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter)
{
referee = new ShuffleSchedulerImpl.Referee(this);
totalMaps = job.GetNumMapTasks();
abortFailureLimit = Math.Max(30, totalMaps / 10);
copyTimeTracker = new ShuffleSchedulerImpl.CopyTimeTracker();
remainingMaps = totalMaps;
finishedMaps = new bool[remainingMaps];
this.reporter = reporter;
this.status = status;
this.reduceId = reduceId;
this.progress = progress;
this.shuffledMapsCounter = shuffledMapsCounter;
this.reduceShuffleBytes = reduceShuffleBytes;
this.failedShuffleCounter = failedShuffleCounter;
this.startTime = Time.MonotonicNow();
lastProgressTime = startTime;
referee.Start();
this.maxFailedUniqueFetches = Math.Min(totalMaps, 5);
this.maxFetchFailuresBeforeReporting = job.GetInt(MRJobConfig.ShuffleFetchFailures
, ReportFailureLimit);
this.reportReadErrorImmediately = job.GetBoolean(MRJobConfig.ShuffleNotifyReaderror
, true);
this.maxDelay = job.GetLong(MRJobConfig.MaxShuffleFetchRetryDelay, MRJobConfig.DefaultMaxShuffleFetchRetryDelay
);
this.maxHostFailures = job.GetInt(MRJobConfig.MaxShuffleFetchHostFailures, MRJobConfig
.DefaultMaxShuffleFetchHostFailures);
}
public override void Configure(JobConf job)
{
numBytesToWrite = job.GetLong("test.tmb.bytes_per_map", 128 * 1024 * 1024);
minKeySize = job.GetInt("test.tmb.min_key", 10);
keySizeRange = job.GetInt("test.tmb.max_key", 10) - minKeySize;
minValueSize = job.GetInt("test.tmb.min_value", 10);
valueSizeRange = job.GetInt("test.tmb.max_value", 10) - minValueSize;
}
public override void Configure(JobConf job)
{
bytesToWrite = job.GetLong(RandomTextWriter.BytesPerMap, 1 * 1024 * 1024 * 1024);
keymin = job.GetInt(RandomTextWriter.MinKey, 5);
keymax = job.GetInt(RandomTextWriter.MaxKey, 10);
valmin = job.GetInt(RandomTextWriter.MinValue, 5);
valmax = job.GetInt(RandomTextWriter.MaxValue, 10);
}
/// <exception cref="System.Exception"/>
public virtual void TestLocality()
{
JobConf job = new JobConf(conf);
dfs = NewDFSCluster(job);
FileSystem fs = dfs.GetFileSystem();
System.Console.Out.WriteLine("FileSystem " + fs.GetUri());
Path inputDir = new Path("/foo/");
string fileName = "part-0000";
CreateInputs(fs, inputDir, fileName);
// split it using a file input format
TextInputFormat.AddInputPath(job, inputDir);
TextInputFormat inFormat = new TextInputFormat();
inFormat.Configure(job);
InputSplit[] splits = inFormat.GetSplits(job, 1);
FileStatus fileStatus = fs.GetFileStatus(new Path(inputDir, fileName));
BlockLocation[] locations = fs.GetFileBlockLocations(fileStatus, 0, fileStatus.GetLen
());
System.Console.Out.WriteLine("Made splits");
// make sure that each split is a block and the locations match
for (int i = 0; i < splits.Length; ++i)
{
FileSplit fileSplit = (FileSplit)splits[i];
System.Console.Out.WriteLine("File split: " + fileSplit);
foreach (string h in fileSplit.GetLocations())
{
System.Console.Out.WriteLine("Location: " + h);
}
System.Console.Out.WriteLine("Block: " + locations[i]);
NUnit.Framework.Assert.AreEqual(locations[i].GetOffset(), fileSplit.GetStart());
NUnit.Framework.Assert.AreEqual(locations[i].GetLength(), fileSplit.GetLength());
string[] blockLocs = locations[i].GetHosts();
string[] splitLocs = fileSplit.GetLocations();
NUnit.Framework.Assert.AreEqual(2, blockLocs.Length);
NUnit.Framework.Assert.AreEqual(2, splitLocs.Length);
NUnit.Framework.Assert.IsTrue((blockLocs[0].Equals(splitLocs[0]) && blockLocs[1].
Equals(splitLocs[1])) || (blockLocs[1].Equals(splitLocs[0]) && blockLocs[0].Equals
(splitLocs[1])));
}
NUnit.Framework.Assert.AreEqual("Expected value of " + FileInputFormat.NumInputFiles
, 1, job.GetLong(FileInputFormat.NumInputFiles, 0));
}
/// <exception cref="System.IO.IOException"/>
internal static long WriteBench(JobConf conf)
{
// OutputFormat instantiation
long filelen = conf.GetLong("filebench.file.bytes", 5 * 1024 * 1024 * 1024);
Org.Apache.Hadoop.IO.Text key = new Org.Apache.Hadoop.IO.Text();
Org.Apache.Hadoop.IO.Text val = new Org.Apache.Hadoop.IO.Text();
string fn = conf.Get("test.filebench.name", string.Empty);
Path outd = FileOutputFormat.GetOutputPath(conf);
conf.Set("mapred.work.output.dir", outd.ToString());
OutputFormat outf = conf.GetOutputFormat();
RecordWriter <Org.Apache.Hadoop.IO.Text, Org.Apache.Hadoop.IO.Text> rw = outf.GetRecordWriter
(outd.GetFileSystem(conf), conf, fn, Reporter.Null);
try
{
long acc = 0L;
DateTime start = new DateTime();
for (int i = 0; acc < filelen; ++i)
{
i %= keys.Length;
key.Set(keys[i]);
val.Set(values[i]);
rw.Write(key, val);
acc += keys[i].Length;
acc += values[i].Length;
}
DateTime end = new DateTime();
return(end.GetTime() - start.GetTime());
}
finally
{
rw.Close(Reporter.Null);
}
}
public override void Configure(JobConf conf)
{
loader = new TestJobCounters.MemoryLoader(conf.GetLong(TargetValue, -1));
}
/// <summary>get the input file name.</summary>
/// <param name="job">a job configuration object</param>
public override void Configure(JobConf job)
{
base.Configure(job);
maxNumItems = job.GetLong("aggregate.max.num.unique.values", long.MaxValue);
}
public MergeManagerImpl(TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS
, LocalDirAllocator localDirAllocator, Reporter reporter, CompressionCodec codec
, Type combinerClass, Task.CombineOutputCollector <K, V> combineCollector, Counters.Counter
spilledRecordsCounter, Counters.Counter reduceCombineInputCounter, Counters.Counter
mergedMapOutputsCounter, ExceptionReporter exceptionReporter, Progress mergePhase
, MapOutputFile mapOutputFile)
{
/* Maximum percentage of the in-memory limit that a single shuffle can
* consume*/
this.reduceId = reduceId;
this.jobConf = jobConf;
this.localDirAllocator = localDirAllocator;
this.exceptionReporter = exceptionReporter;
this.reporter = reporter;
this.codec = codec;
this.combinerClass = combinerClass;
this.combineCollector = combineCollector;
this.reduceCombineInputCounter = reduceCombineInputCounter;
this.spilledRecordsCounter = spilledRecordsCounter;
this.mergedMapOutputsCounter = mergedMapOutputsCounter;
this.mapOutputFile = mapOutputFile;
this.mapOutputFile.SetConf(jobConf);
this.localFS = localFS;
this.rfs = ((LocalFileSystem)localFS).GetRaw();
float maxInMemCopyUse = jobConf.GetFloat(MRJobConfig.ShuffleInputBufferPercent, MRJobConfig
.DefaultShuffleInputBufferPercent);
if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0)
{
throw new ArgumentException("Invalid value for " + MRJobConfig.ShuffleInputBufferPercent
+ ": " + maxInMemCopyUse);
}
// Allow unit tests to fix Runtime memory
this.memoryLimit = (long)(jobConf.GetLong(MRJobConfig.ReduceMemoryTotalBytes, Runtime
.GetRuntime().MaxMemory()) * maxInMemCopyUse);
this.ioSortFactor = jobConf.GetInt(MRJobConfig.IoSortFactor, 100);
float singleShuffleMemoryLimitPercent = jobConf.GetFloat(MRJobConfig.ShuffleMemoryLimitPercent
, DefaultShuffleMemoryLimitPercent);
if (singleShuffleMemoryLimitPercent <= 0.0f || singleShuffleMemoryLimitPercent >
1.0f)
{
throw new ArgumentException("Invalid value for " + MRJobConfig.ShuffleMemoryLimitPercent
+ ": " + singleShuffleMemoryLimitPercent);
}
usedMemory = 0L;
commitMemory = 0L;
this.maxSingleShuffleLimit = (long)(memoryLimit * singleShuffleMemoryLimitPercent
);
this.memToMemMergeOutputsThreshold = jobConf.GetInt(MRJobConfig.ReduceMemtomemThreshold
, ioSortFactor);
this.mergeThreshold = (long)(this.memoryLimit * jobConf.GetFloat(MRJobConfig.ShuffleMergePercent
, 0.90f));
Log.Info("MergerManager: memoryLimit=" + memoryLimit + ", " + "maxSingleShuffleLimit="
+ maxSingleShuffleLimit + ", " + "mergeThreshold=" + mergeThreshold + ", " + "ioSortFactor="
+ ioSortFactor + ", " + "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold
);
if (this.maxSingleShuffleLimit >= this.mergeThreshold)
{
throw new RuntimeException("Invalid configuration: " + "maxSingleShuffleLimit should be less than mergeThreshold "
+ "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit + "mergeThreshold: " +
this.mergeThreshold);
}
bool allowMemToMemMerge = jobConf.GetBoolean(MRJobConfig.ReduceMemtomemEnabled, false
);
if (allowMemToMemMerge)
{
this.memToMemMerger = new MergeManagerImpl.IntermediateMemoryToMemoryMerger(this,
this, memToMemMergeOutputsThreshold);
this.memToMemMerger.Start();
}
else
{
this.memToMemMerger = null;
}
this.inMemoryMerger = CreateInMemoryMerger();
this.inMemoryMerger.Start();
this.onDiskMerger = new MergeManagerImpl.OnDiskMerger(this, this);
this.onDiskMerger.Start();
this.mergePhase = mergePhase;
}
public override void Configure(JobConf conf)
{
// Mapper
base.Configure(conf);
skipSize = conf.GetLong("test.io.skip.size", 0);
}
/// <summary>
/// Splits files returned by
/// <see cref="FileInputFormat{K, V}.ListStatus(JobConf)"/>
/// when
/// they're too big.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual InputSplit[] GetSplits(JobConf job, int numSplits)
{
StopWatch sw = new StopWatch().Start();
FileStatus[] files = ListStatus(job);
// Save the number of input files for metrics/loadgen
job.SetLong(NumInputFiles, files.Length);
long totalSize = 0;
// compute total size
foreach (FileStatus file in files)
{
// check we have valid files
if (file.IsDirectory())
{
throw new IOException("Not a file: " + file.GetPath());
}
totalSize += file.GetLen();
}
long goalSize = totalSize / (numSplits == 0 ? 1 : numSplits);
long minSize = Math.Max(job.GetLong(FileInputFormat.SplitMinsize, 1), minSplitSize
);
// generate splits
AList <FileSplit> splits = new AList <FileSplit>(numSplits);
NetworkTopology clusterMap = new NetworkTopology();
foreach (FileStatus file_1 in files)
{
Path path = file_1.GetPath();
long length = file_1.GetLen();
if (length != 0)
{
FileSystem fs = path.GetFileSystem(job);
BlockLocation[] blkLocations;
if (file_1 is LocatedFileStatus)
{
blkLocations = ((LocatedFileStatus)file_1).GetBlockLocations();
}
else
{
blkLocations = fs.GetFileBlockLocations(file_1, 0, length);
}
if (IsSplitable(fs, path))
{
long blockSize = file_1.GetBlockSize();
long splitSize = ComputeSplitSize(goalSize, minSize, blockSize);
long bytesRemaining = length;
while (((double)bytesRemaining) / splitSize > SplitSlop)
{
string[][] splitHosts = GetSplitHostsAndCachedHosts(blkLocations, length - bytesRemaining
, splitSize, clusterMap);
splits.AddItem(MakeSplit(path, length - bytesRemaining, splitSize, splitHosts[0],
splitHosts[1]));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0)
{
string[][] splitHosts = GetSplitHostsAndCachedHosts(blkLocations, length - bytesRemaining
, bytesRemaining, clusterMap);
splits.AddItem(MakeSplit(path, length - bytesRemaining, bytesRemaining, splitHosts
[0], splitHosts[1]));
}
}
else
{
string[][] splitHosts = GetSplitHostsAndCachedHosts(blkLocations, 0, length, clusterMap
);
splits.AddItem(MakeSplit(path, 0, length, splitHosts[0], splitHosts[1]));
}
}
else
{
//Create empty hosts array for zero length files
splits.AddItem(MakeSplit(path, 0, length, new string[0]));
}
}
sw.Stop();
if (Log.IsDebugEnabled())
{
Log.Debug("Total # of splits generated by getSplits: " + splits.Count + ", TimeTaken: "
+ sw.Now(TimeUnit.Milliseconds));
}
return(Sharpen.Collections.ToArray(splits, new FileSplit[splits.Count]));
}