public virtual void TestSucceedAndFailedCopyMap <K, V>()
{
JobConf job = new JobConf();
job.SetNumMapTasks(2);
//mock creation
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = job.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>();
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
TaskStatus status = new _TaskStatus_251();
Progress progress = new Progress();
ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status
, null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes
(), context.GetFailedShuffleCounter());
MapHost host1 = new MapHost("host1", null);
TaskAttemptID failedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0)
, TaskType.Map, 0), 0);
TaskAttemptID succeedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0
), TaskType.Map, 1), 1);
// handle output fetch failure for failedAttemptID, part I
scheduler.HostFailed(host1.GetHostName());
// handle output fetch succeed for succeedAttemptID
long bytes = (long)500 * 1024 * 1024;
scheduler.CopySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output);
// handle output fetch failure for failedAttemptID, part II
// for MAPREDUCE-6361: verify no NPE exception get thrown out
scheduler.CopyFailed(failedAttemptID, host1, true, false);
}
private DataInputStream OpenShuffleUrl(MapHost host, ICollection <TaskAttemptID> remaining
, Uri url)
{
DataInputStream input = null;
try
{
SetupConnectionsWithRetry(host, remaining, url);
if (stopped)
{
AbortConnect(host, remaining);
}
else
{
input = new DataInputStream(connection.GetInputStream());
}
}
catch (IOException ie)
{
bool connectExcpt = ie is ConnectException;
ioErrs.Increment(1);
Log.Warn("Failed to connect to " + host + " with " + remaining.Count + " map outputs"
, ie);
// If connect did not succeed, just mark all the maps as failed,
// indirectly penalizing the host
scheduler.HostFailed(host.GetHostName());
foreach (TaskAttemptID left in remaining)
{
scheduler.CopyFailed(left, host, false, connectExcpt);
}
// Add back all the remaining maps, WITHOUT marking them as failed
foreach (TaskAttemptID left_1 in remaining)
{
scheduler.PutBackKnownMapOutput(host, left_1);
}
}
return(input);
}