public virtual void ObsoleteMapOutput(TaskAttemptID mapId)
{
lock (this)
{
obsoleteMaps.AddItem(mapId);
}
}
public virtual void PutBackKnownMapOutput(MapHost host, TaskAttemptID mapId)
{
lock (this)
{
host.AddKnownMap(mapId);
}
}
public ShuffleSchedulerImpl(JobConf job, TaskStatus status, TaskAttemptID reduceId
, ExceptionReporter reporter, Progress progress, Counters.Counter shuffledMapsCounter
, Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter)
{
referee = new ShuffleSchedulerImpl.Referee(this);
totalMaps = job.GetNumMapTasks();
abortFailureLimit = Math.Max(30, totalMaps / 10);
copyTimeTracker = new ShuffleSchedulerImpl.CopyTimeTracker();
remainingMaps = totalMaps;
finishedMaps = new bool[remainingMaps];
this.reporter = reporter;
this.status = status;
this.reduceId = reduceId;
this.progress = progress;
this.shuffledMapsCounter = shuffledMapsCounter;
this.reduceShuffleBytes = reduceShuffleBytes;
this.failedShuffleCounter = failedShuffleCounter;
this.startTime = Time.MonotonicNow();
lastProgressTime = startTime;
referee.Start();
this.maxFailedUniqueFetches = Math.Min(totalMaps, 5);
this.maxFetchFailuresBeforeReporting = job.GetInt(MRJobConfig.ShuffleFetchFailures
, ReportFailureLimit);
this.reportReadErrorImmediately = job.GetBoolean(MRJobConfig.ShuffleNotifyReaderror
, true);
this.maxDelay = job.GetLong(MRJobConfig.MaxShuffleFetchRetryDelay, MRJobConfig.DefaultMaxShuffleFetchRetryDelay
);
this.maxHostFailures = job.GetInt(MRJobConfig.MaxShuffleFetchHostFailures, MRJobConfig
.DefaultMaxShuffleFetchHostFailures);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <IntWritable> values, Reducer.Context
context)
{
// Make one reducer slower for speculative execution
TaskAttemptID taid = context.GetTaskAttemptID();
long sleepTime = 100;
Configuration conf = context.GetConfiguration();
bool test_speculate_reduce = conf.GetBoolean(MRJobConfig.ReduceSpeculative, false
);
// IF TESTING REDUCE SPECULATIVE EXECUTION:
// Make the "*_r_000000_0" attempt take much longer than the others.
// When speculative execution is enabled, this should cause the attempt
// to be killed and restarted. At that point, the attempt ID will be
// "*_r_000000_1", so sleepTime will still remain 100ms.
if ((taid.GetTaskType() == TaskType.Reduce) && test_speculate_reduce && (taid.GetTaskID
().GetId() == 0) && (taid.GetId() == 0))
{
sleepTime = 10000;
}
try
{
Sharpen.Thread.Sleep(sleepTime);
}
catch (Exception)
{
}
// Ignore
context.Write(key, new IntWritable(0));
}
public TaskAttemptStartedEvent(TaskAttemptID attemptId, TaskType taskType, long startTime
, string trackerName, int httpPort, int shufflePort, string locality, string avataar
)
: this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort, ConverterUtils
.ToContainerId("container_-1_-1_-1_-1"), locality, avataar)
{
}
public virtual void Setup()
{
// mocked generics
Log.Info(">>>> " + name.GetMethodName());
job = new JobConf();
job.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, false);
jobWithRetry = new JobConf();
jobWithRetry.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, true);
id = TaskAttemptID.ForName("attempt_0_1_r_1_1");
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
mm = Org.Mockito.Mockito.Mock <MergeManagerImpl>();
r = Org.Mockito.Mockito.Mock <Reporter>();
metrics = Org.Mockito.Mockito.Mock <ShuffleClientMetrics>();
except = Org.Mockito.Mockito.Mock <ExceptionReporter>();
key = JobTokenSecretManager.CreateSecretKey(new byte[] { 0, 0, 0, 0 });
connection = Org.Mockito.Mockito.Mock <HttpURLConnection>();
allErrs = Org.Mockito.Mockito.Mock <Counters.Counter>();
Org.Mockito.Mockito.When(r.GetCounter(Matchers.AnyString(), Matchers.AnyString())
).ThenReturn(allErrs);
AList <TaskAttemptID> maps = new AList <TaskAttemptID>(1);
maps.AddItem(map1ID);
maps.AddItem(map2ID);
Org.Mockito.Mockito.When(ss.GetMapsForHost(host)).ThenReturn(maps);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestFailingMapper()
{
Log.Info("\n\n\nStarting testFailingMapper().");
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
Job job = RunFailingMapperJob();
TaskID taskID = new TaskID(job.GetJobID(), TaskType.Map, 0);
TaskAttemptID aId = new TaskAttemptID(taskID, 0);
System.Console.Out.WriteLine("Diagnostics for " + aId + " :");
foreach (string diag in job.GetTaskDiagnostics(aId))
{
System.Console.Out.WriteLine(diag);
}
aId = new TaskAttemptID(taskID, 1);
System.Console.Out.WriteLine("Diagnostics for " + aId + " :");
foreach (string diag_1 in job.GetTaskDiagnostics(aId))
{
System.Console.Out.WriteLine(diag_1);
}
TaskCompletionEvent[] events = job.GetTaskCompletionEvents(0, 2);
NUnit.Framework.Assert.AreEqual(TaskCompletionEvent.Status.Failed, events[0].GetStatus
());
NUnit.Framework.Assert.AreEqual(TaskCompletionEvent.Status.Tipfailed, events[1].GetStatus
());
NUnit.Framework.Assert.AreEqual(JobStatus.State.Failed, job.GetJobState());
VerifyFailingMapperCounters(job);
}
public ReduceAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, string taskStatus
, long shuffleFinishTime, long sortFinishTime, long finishTime, string hostname,
string state, Counters counters)
: this(id, taskType, taskStatus, shuffleFinishTime, sortFinishTime, finishTime, hostname
, -1, string.Empty, state, counters, null)
{
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override RecordReader CreateRecordReader(InputSplit split, TaskAttemptContext
taskContext)
{
try
{
if (!rrCstrMap.Contains(ident))
{
throw new IOException("No RecordReader for " + ident);
}
Configuration conf = GetConf(taskContext.GetConfiguration());
TaskAttemptContext context = new TaskAttemptContextImpl(conf, TaskAttemptID.ForName
(conf.Get(MRJobConfig.TaskAttemptId)), new Parser.WrappedStatusReporter(taskContext
));
return(rrCstrMap[ident].NewInstance(id, inf.CreateRecordReader(split, context), cmpcl
));
}
catch (MemberAccessException e)
{
throw new IOException(e);
}
catch (InstantiationException e)
{
throw new IOException(e);
}
catch (TargetInvocationException e)
{
throw new IOException(e);
}
}
public TaskAttemptUnsuccessfulCompletionEvent(TaskAttemptID id, TaskType taskType
, string status, long finishTime, string hostname, int port, string rackName, string
error, int[][] allSplits)
: this(id, taskType, status, finishTime, hostname, port, rackName, error, EmptyCounters
, null)
{
}
public TaskAttemptContextImpl(Configuration conf, TaskAttemptID taskId, StatusReporter
reporter)
: base(conf, taskId.GetJobID())
{
this.taskId = taskId;
this.reporter = reporter;
}
/// <exception cref="System.IO.IOException"/>
public virtual void CleanUpPartialOutputForTask(TaskAttemptContext context)
{
// we double check this is never invoked from a non-preemptable subclass.
// This should never happen, since the invoking codes is checking it too,
// but it is safer to double check. Errors handling this would produce
// inconsistent output.
if (!this.GetType().IsAnnotationPresent(typeof(Checkpointable)))
{
throw new InvalidOperationException("Invoking cleanUpPartialOutputForTask() " + "from non @Preemptable class"
);
}
FileSystem fs = FsFor(GetTaskAttemptPath(context), context.GetConfiguration());
Log.Info("cleanUpPartialOutputForTask: removing everything belonging to " + context
.GetTaskAttemptID().GetTaskID() + " in: " + GetCommittedTaskPath(context).GetParent
());
TaskAttemptID taid = context.GetTaskAttemptID();
TaskID tid = taid.GetTaskID();
Path pCommit = GetCommittedTaskPath(context).GetParent();
// remove any committed output
for (int i = 0; i < taid.GetId(); ++i)
{
TaskAttemptID oldId = new TaskAttemptID(tid, i);
Path pTask = new Path(pCommit, oldId.ToString());
if (fs.Exists(pTask) && !fs.Delete(pTask, true))
{
throw new IOException("Failed to delete " + pTask);
}
}
}
public MapOutput(TaskAttemptID mapId, long size, bool primaryMapOutput)
{
this.id = Id.IncrementAndGet();
this.mapId = mapId;
this.size = size;
this.primaryMapOutput = primaryMapOutput;
}
public virtual void TestTaskStartTimes()
{
TaskId taskId = Org.Mockito.Mockito.Mock <TaskId>();
JobHistoryParser.TaskInfo taskInfo = Org.Mockito.Mockito.Mock <JobHistoryParser.TaskInfo
>();
IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> taskAttempts = new SortedDictionary
<TaskAttemptID, JobHistoryParser.TaskAttemptInfo>();
TaskAttemptID id = new TaskAttemptID("0", 0, TaskType.Map, 0, 0);
JobHistoryParser.TaskAttemptInfo info = Org.Mockito.Mockito.Mock <JobHistoryParser.TaskAttemptInfo
>();
Org.Mockito.Mockito.When(info.GetAttemptId()).ThenReturn(id);
Org.Mockito.Mockito.When(info.GetStartTime()).ThenReturn(10l);
taskAttempts[id] = info;
id = new TaskAttemptID("1", 0, TaskType.Map, 1, 1);
info = Org.Mockito.Mockito.Mock <JobHistoryParser.TaskAttemptInfo>();
Org.Mockito.Mockito.When(info.GetAttemptId()).ThenReturn(id);
Org.Mockito.Mockito.When(info.GetStartTime()).ThenReturn(20l);
taskAttempts[id] = info;
Org.Mockito.Mockito.When(taskInfo.GetAllTaskAttempts()).ThenReturn(taskAttempts);
CompletedTask task = new CompletedTask(taskId, taskInfo);
TaskReport report = task.GetReport();
// Make sure the startTime returned by report is the lesser of the
// attempy launch times
NUnit.Framework.Assert.IsTrue(report.GetStartTime() == 10);
}
public MapAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, string taskStatus
, long mapFinishTime, long finishTime, string hostname, string state, Counters counters
)
: this(id, taskType, taskStatus, mapFinishTime, finishTime, hostname, -1, string.Empty
, state, counters, null)
{
}
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public ReduceContextImpl(Configuration conf, TaskAttemptID taskid, RawKeyValueIterator
input, Counter inputKeyCounter, Counter inputValueCounter, RecordWriter <KEYOUT,
VALUEOUT> output, OutputCommitter committer, StatusReporter reporter, RawComparator
<KEYIN> comparator, Type keyClass, Type valueClass)
: base(conf, taskid, output, committer, reporter)
{
iterable = new ReduceContextImpl.ValueIterable(this);
// current key
// current value
// first value in key
// more w/ this key
// more in file
this.input = input;
this.inputKeyCounter = inputKeyCounter;
this.inputValueCounter = inputValueCounter;
this.comparator = comparator;
this.serializationFactory = new SerializationFactory(conf);
this.keyDeserializer = serializationFactory.GetDeserializer(keyClass);
this.keyDeserializer.Open(buffer);
this.valueDeserializer = serializationFactory.GetDeserializer(valueClass);
this.valueDeserializer.Open(buffer);
hasMore = input.Next();
this.keyClass = keyClass;
this.valueClass = valueClass;
this.conf = conf;
this.taskid = taskid;
}
/// <summary>
/// Tests the class loader set by
/// <see cref="Org.Apache.Hadoop.Conf.Configuration.SetClassLoader(Sharpen.ClassLoader)
/// "/>
/// is inherited by any
/// <see cref="WrappedRecordReader{K, U}"/>
/// s created by
/// <see cref="CompositeRecordReader{K, V, X}"/>
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestClassLoader()
{
Configuration conf = new Configuration();
TestWrappedRRClassloader.Fake_ClassLoader classLoader = new TestWrappedRRClassloader.Fake_ClassLoader
();
conf.SetClassLoader(classLoader);
NUnit.Framework.Assert.IsTrue(conf.GetClassLoader() is TestWrappedRRClassloader.Fake_ClassLoader
);
FileSystem fs = FileSystem.Get(conf);
Path testdir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified
(fs);
Path @base = new Path(testdir, "/empty");
Path[] src = new Path[] { new Path(@base, "i0"), new Path("i1"), new Path("i2") };
conf.Set(CompositeInputFormat.JoinExpr, CompositeInputFormat.Compose("outer", typeof(
TestWrappedRRClassloader.IF_ClassLoaderChecker), src));
CompositeInputFormat <NullWritable> inputFormat = new CompositeInputFormat <NullWritable
>();
// create dummy TaskAttemptID
TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.Map, 0, 0);
conf.Set(MRJobConfig.TaskAttemptId, tid.ToString());
inputFormat.CreateRecordReader(inputFormat.GetSplits(Job.GetInstance(conf))[0], new
TaskAttemptContextImpl(conf, tid));
}
public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, RecordWriter
<KEYOUT, VALUEOUT> output, OutputCommitter committer, StatusReporter reporter)
: base(conf, taskid, reporter)
{
this.output = output;
this.committer = committer;
}
/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.mapred.MapReduceBase#configure(org.apache.hadoop.mapred
* .JobConf)
*/
public override void Configure(JobConf conf)
{
// MapReduceBase
try
{
config = new ConfigExtractor(conf);
ConfigExtractor.DumpOptions(config);
filesystem = config.GetBaseDirectory().GetFileSystem(conf);
}
catch (Exception e)
{
Log.Error("Unable to setup slive " + StringUtils.StringifyException(e));
throw new RuntimeException("Unable to setup slive configuration", e);
}
if (conf.Get(MRJobConfig.TaskAttemptId) != null)
{
this.taskId = TaskAttemptID.ForName(conf.Get(MRJobConfig.TaskAttemptId)).GetTaskID
().GetId();
}
else
{
// So that branch-1/0.20 can run this same code as well
this.taskId = TaskAttemptID.ForName(conf.Get("mapred.task.id")).GetTaskID().GetId
();
}
}
/// <summary>Do some basic verification on the input received -- Being defensive</summary>
/// <param name="compressedLength"/>
/// <param name="decompressedLength"/>
/// <param name="forReduce"/>
/// <param name="remaining"/>
/// <param name="mapId"/>
/// <returns>true/false, based on if the verification succeeded or not</returns>
private bool VerifySanity(long compressedLength, long decompressedLength, int forReduce
, ICollection <TaskAttemptID> remaining, TaskAttemptID mapId)
{
if (compressedLength < 0 || decompressedLength < 0)
{
wrongLengthErrs.Increment(1);
Log.Warn(GetName() + " invalid lengths in map output header: id: " + mapId + " len: "
+ compressedLength + ", decomp len: " + decompressedLength);
return(false);
}
if (forReduce != reduce)
{
wrongReduceErrs.Increment(1);
Log.Warn(GetName() + " data for the wrong reduce map: " + mapId + " len: " + compressedLength
+ " decomp len: " + decompressedLength + " for reduce " + forReduce);
return(false);
}
// Sanity check
if (!remaining.Contains(mapId))
{
wrongMapErrs.Increment(1);
Log.Warn("Invalid map-output! Received output for " + mapId);
return(false);
}
return(true);
}
internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V>
scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics
, ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id)
{
/* Default read timeout (in milliseconds) */
// Initiative value is 0, which means it hasn't retried yet.
this.jobConf = job;
this.reporter = reporter;
this.scheduler = scheduler;
this.merger = merger;
this.metrics = metrics;
this.exceptionReporter = exceptionReporter;
this.id = id;
this.reduce = reduceId.GetTaskID().GetId();
this.shuffleSecretKey = shuffleKey;
ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString
());
wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength
.ToString());
badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString
());
wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap
.ToString());
connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection
.ToString());
wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce
.ToString());
this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout
);
this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout);
this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig
.DefaultShuffleFetchRetryIntervalMs);
this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout
);
bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled
, YarnConfiguration.DefaultNmRecoveryEnabled);
this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault
);
SetName("fetcher#" + id);
SetDaemon(true);
lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher))
{
sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault
);
if (sslShuffle && sslFactory == null)
{
sslFactory = new SSLFactory(SSLFactory.Mode.Client, job);
try
{
sslFactory.Init();
}
catch (Exception ex)
{
sslFactory.Destroy();
throw new RuntimeException(ex);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
public override void Merge(IList <InMemoryMapOutput <K, V> > inputs)
{
if (inputs == null || inputs.Count == 0)
{
return;
}
TaskAttemptID dummyMapId = inputs[0].GetMapId();
IList <Merger.Segment <K, V> > inMemorySegments = new AList <Merger.Segment <K, V> >();
long mergeOutputSize = this._enclosing.CreateInMemorySegments(inputs, inMemorySegments
, 0);
int noInMemorySegments = inMemorySegments.Count;
InMemoryMapOutput <K, V> mergedMapOutputs = this._enclosing.UnconditionalReserve(dummyMapId
, mergeOutputSize, false);
IFile.Writer <K, V> writer = new InMemoryWriter <K, V>(mergedMapOutputs.GetArrayStream
());
MergeManagerImpl.Log.Info("Initiating Memory-to-Memory merge with " + noInMemorySegments
+ " segments of total-size: " + mergeOutputSize);
RawKeyValueIterator rIter = Merger.Merge(this._enclosing.jobConf, this._enclosing
.rfs, (Type)this._enclosing.jobConf.GetMapOutputKeyClass(), (Type)this._enclosing
.jobConf.GetMapOutputValueClass(), inMemorySegments, inMemorySegments.Count, new
Path(this._enclosing.reduceId.ToString()), (RawComparator <K>) this._enclosing.jobConf
.GetOutputKeyComparator(), this._enclosing.reporter, null, null, null);
Merger.WriteFile(rIter, writer, this._enclosing.reporter, this._enclosing.jobConf
);
writer.Close();
MergeManagerImpl.Log.Info(this._enclosing.reduceId + " Memory-to-Memory merge of the "
+ noInMemorySegments + " files in-memory complete.");
// Note the output of the merge
this._enclosing.CloseInMemoryMergedFile(mergedMapOutputs);
}
public Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler
, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter
exceptionReporter, SecretKey shuffleKey)
: this(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey
, ++nextId)
{
}
public MapContextImpl(Configuration conf, TaskAttemptID taskid, RecordReader <KEYIN
, VALUEIN> reader, RecordWriter <KEYOUT, VALUEOUT> writer, OutputCommitter committer
, StatusReporter reporter, InputSplit split)
: base(conf, taskid, writer, committer, reporter)
{
this.reader = reader;
this.split = split;
}
/// <exception cref="System.IO.IOException"/>
public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId, MergeManagerImpl
<K, V> merger, long size, JobConf conf, MapOutputFile mapOutputFile, int fetcher
, bool primaryMapOutput)
: this(mapId, reduceId, merger, size, conf, mapOutputFile, fetcher, primaryMapOutput
, FileSystem.GetLocal(conf).GetRaw(), mapOutputFile.GetInputFileForWrite(mapId.GetTaskID
(), size))
{
}
/// <exception cref="System.Exception"/>
private string ReadStdOut(JobConf conf)
{
TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName(conf.Get(MRJobConfig
.TaskAttemptId)));
FilePath stdOut = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stdout);
return(ReadFile(stdOut));
}
public FakeFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V
> scheduler, MergeManagerImpl <K, V> merger, Reporter reporter, ShuffleClientMetrics
metrics, ExceptionReporter exceptionReporter, SecretKey jobTokenSecret, HttpURLConnection
connection, int id)
: base(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, jobTokenSecret
, id)
{
this.connection = connection;
}
public virtual void CommitTask(TaskAttemptContext context, Path taskAttemptPath)
{
TaskAttemptID attemptId = context.GetTaskAttemptID();
if (HasOutputPath())
{
context.Progress();
if (taskAttemptPath == null)
{
taskAttemptPath = GetTaskAttemptPath(context);
}
FileSystem fs = taskAttemptPath.GetFileSystem(context.GetConfiguration());
FileStatus taskAttemptDirStatus;
try
{
taskAttemptDirStatus = fs.GetFileStatus(taskAttemptPath);
}
catch (FileNotFoundException)
{
taskAttemptDirStatus = null;
}
if (taskAttemptDirStatus != null)
{
if (algorithmVersion == 1)
{
Path committedTaskPath = GetCommittedTaskPath(context);
if (fs.Exists(committedTaskPath))
{
if (!fs.Delete(committedTaskPath, true))
{
throw new IOException("Could not delete " + committedTaskPath);
}
}
if (!fs.Rename(taskAttemptPath, committedTaskPath))
{
throw new IOException("Could not rename " + taskAttemptPath + " to " + committedTaskPath
);
}
Log.Info("Saved output of task '" + attemptId + "' to " + committedTaskPath);
}
else
{
// directly merge everything from taskAttemptPath to output directory
MergePaths(fs, taskAttemptDirStatus, outputPath);
Log.Info("Saved output of task '" + attemptId + "' to " + outputPath);
}
}
else
{
Log.Warn("No Output found for " + attemptId);
}
}
else
{
Log.Warn("Output Path is null in commitTask()");
}
}
public virtual void TestSucceedAndFailedCopyMap <K, V>()
{
JobConf job = new JobConf();
job.SetNumMapTasks(2);
//mock creation
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = job.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>();
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
TaskStatus status = new _TaskStatus_251();
Progress progress = new Progress();
ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status
, null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes
(), context.GetFailedShuffleCounter());
MapHost host1 = new MapHost("host1", null);
TaskAttemptID failedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0)
, TaskType.Map, 0), 0);
TaskAttemptID succeedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0
), TaskType.Map, 1), 1);
// handle output fetch failure for failedAttemptID, part I
scheduler.HostFailed(host1.GetHostName());
// handle output fetch succeed for succeedAttemptID
long bytes = (long)500 * 1024 * 1024;
scheduler.CopySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output);
// handle output fetch failure for failedAttemptID, part II
// for MAPREDUCE-6361: verify no NPE exception get thrown out
scheduler.CopyFailed(failedAttemptID, host1, true, false);
}
/// <summary>Unconditional Reserve is used by the Memory-to-Memory thread</summary>
/// <returns/>
private InMemoryMapOutput <K, V> UnconditionalReserve(TaskAttemptID mapId, long requestedSize
, bool primaryMapOutput)
{
lock (this)
{
usedMemory += requestedSize;
return(new InMemoryMapOutput <K, V>(jobConf, mapId, this, (int)requestedSize, codec
, primaryMapOutput));
}
}
