public virtual void Setup()
{
// mocked generics
Log.Info(">>>> " + name.GetMethodName());
job = new JobConf();
job.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, false);
jobWithRetry = new JobConf();
jobWithRetry.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, true);
id = TaskAttemptID.ForName("attempt_0_1_r_1_1");
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
mm = Org.Mockito.Mockito.Mock <MergeManagerImpl>();
r = Org.Mockito.Mockito.Mock <Reporter>();
metrics = Org.Mockito.Mockito.Mock <ShuffleClientMetrics>();
except = Org.Mockito.Mockito.Mock <ExceptionReporter>();
key = JobTokenSecretManager.CreateSecretKey(new byte[] { 0, 0, 0, 0 });
connection = Org.Mockito.Mockito.Mock <HttpURLConnection>();
allErrs = Org.Mockito.Mockito.Mock <Counters.Counter>();
Org.Mockito.Mockito.When(r.GetCounter(Matchers.AnyString(), Matchers.AnyString())
).ThenReturn(allErrs);
AList <TaskAttemptID> maps = new AList <TaskAttemptID>(1);
maps.AddItem(map1ID);
maps.AddItem(map2ID);
Org.Mockito.Mockito.When(ss.GetMapsForHost(host)).ThenReturn(maps);
}
protected internal virtual void CopyFromHost(MapHost host)
{
// reset retryStartTime for a new host
retryStartTime = 0;
// Get completed maps on 'host'
IList <TaskAttemptID> maps = scheduler.GetMapsForHost(host);
// Sanity check to catch hosts with only 'OBSOLETE' maps,
// especially at the tail of large jobs
if (maps.Count == 0)
{
return;
}
if (Log.IsDebugEnabled())
{
Log.Debug("Fetcher " + id + " going to fetch from " + host + " for: " + maps);
}
// List of maps to be fetched yet
ICollection <TaskAttemptID> remaining = new HashSet <TaskAttemptID>(maps);
// Construct the url and connect
Uri url = GetMapOutputURL(host, maps);
DataInputStream input = OpenShuffleUrl(host, remaining, url);
if (input == null)
{
return;
}
try
{
// Loop through available map-outputs and fetch them
// On any error, faildTasks is not null and we exit
// after putting back the remaining maps to the
// yet_to_be_fetched list and marking the failed tasks.
TaskAttemptID[] failedTasks = null;
while (!remaining.IsEmpty() && failedTasks == null)
{
try
{
failedTasks = CopyMapOutput(host, input, remaining, fetchRetryEnabled);
}
catch (IOException)
{
//
// Setup connection again if disconnected by NM
connection.Disconnect();
// Get map output from remaining tasks only.
url = GetMapOutputURL(host, remaining);
input = OpenShuffleUrl(host, remaining, url);
if (input == null)
{
return;
}
}
}
if (failedTasks != null && failedTasks.Length > 0)
{
Log.Warn("copyMapOutput failed for tasks " + Arrays.ToString(failedTasks));
scheduler.HostFailed(host.GetHostName());
foreach (TaskAttemptID left in failedTasks)
{
scheduler.CopyFailed(left, host, true, false);
}
}
// Sanity check
if (failedTasks == null && !remaining.IsEmpty())
{
throw new IOException("server didn't return all expected map outputs: " + remaining
.Count + " left.");
}
input.Close();
input = null;
}
finally
{
if (input != null)
{
IOUtils.Cleanup(Log, input);
input = null;
}
foreach (TaskAttemptID left in remaining)
{
scheduler.PutBackKnownMapOutput(host, left);
}
}
}