/// <summary>
/// Queries the
/// <see cref="TaskTracker"/>
/// for a set of map-completion events
/// from a given event ID.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual int GetMapCompletionEvents()
{
int numNewMaps = 0;
TaskCompletionEvent[] events = null;
do
{
MapTaskCompletionEventsUpdate update = umbilical.GetMapCompletionEvents((JobID)reduce
.GetJobID(), fromEventIdx, maxEventsToFetch, (TaskAttemptID)reduce);
events = update.GetMapTaskCompletionEvents();
Log.Debug("Got " + events.Length + " map completion events from " + fromEventIdx);
System.Diagnostics.Debug.Assert(!update.ShouldReset(), "Unexpected legacy state");
// Update the last seen event ID
fromEventIdx += events.Length;
// Process the TaskCompletionEvents:
// 1. Save the SUCCEEDED maps in knownOutputs to fetch the outputs.
// 2. Save the OBSOLETE/FAILED/KILLED maps in obsoleteOutputs to stop
// fetching from those maps.
// 3. Remove TIPFAILED maps from neededOutputs since we don't need their
// outputs at all.
foreach (TaskCompletionEvent @event in events)
{
scheduler.Resolve(@event);
if (TaskCompletionEvent.Status.Succeeded == @event.GetTaskStatus())
{
++numNewMaps;
}
}
}while (events.Length == maxEventsToFetch);
return(numNewMaps);
}
public virtual void TestConsecutiveFetch()
{
int MaxEventsToFetch = 100;
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 1);
TaskUmbilicalProtocol umbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>
();
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.AnyInt(), Matchers.AnyInt(), Matchers.Any <TaskAttemptID>())).ThenReturn
(GetMockedCompletionEventsUpdate(0, 0));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid))).ThenReturn(GetMockedCompletionEventsUpdate
(0, MaxEventsToFetch));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid)))
.ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch, MaxEventsToFetch));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid
))).ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch * 2, 3));
ShuffleScheduler <string, string> scheduler = Org.Mockito.Mockito.Mock <ShuffleScheduler
>();
ExceptionReporter reporter = Org.Mockito.Mockito.Mock <ExceptionReporter>();
TestEventFetcher.EventFetcherForTest <string, string> ef = new TestEventFetcher.EventFetcherForTest
<string, string>(tid, umbilical, scheduler, reporter, MaxEventsToFetch);
ef.GetMapCompletionEvents();
Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Never()).ReportException
(Matchers.Any <Exception>());
InOrder inOrder = Org.Mockito.Mockito.InOrder(umbilical);
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(MaxEventsToFetch
* 2 + 3)).Resolve(Matchers.Any <TaskCompletionEvent>());
}
