public static TaskCompletionEvent FromYarn(TaskAttemptCompletionEvent newEvent)
{
return(new TaskCompletionEvent(newEvent.GetEventId(), FromYarn(newEvent.GetAttemptId
()), newEvent.GetAttemptId().GetId(), newEvent.GetAttemptId().GetTaskId().GetTaskType
().Equals(TaskType.Map), FromYarn(newEvent.GetStatus()), newEvent.GetMapOutputServerAddress
()));
}
private void ConstructTaskAttemptCompletionEvents()
{
LoadAllTasks();
completionEvents = new List <TaskAttemptCompletionEvent>();
IList <TaskAttempt> allTaskAttempts = new List <TaskAttempt>();
int numMapAttempts = 0;
foreach (KeyValuePair <TaskId, Task> taskEntry in tasks)
{
Task task = taskEntry.Value;
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttemptEntry in task.GetAttempts
())
{
TaskAttempt taskAttempt = taskAttemptEntry.Value;
allTaskAttempts.AddItem(taskAttempt);
if (task.GetType() == TaskType.Map)
{
++numMapAttempts;
}
}
}
allTaskAttempts.Sort(new _IComparer_237());
mapCompletionEvents = new AList <TaskAttemptCompletionEvent>(numMapAttempts);
int eventId = 0;
foreach (TaskAttempt taskAttempt_1 in allTaskAttempts)
{
TaskAttemptCompletionEvent tace = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <TaskAttemptCompletionEvent
>();
int attemptRunTime = -1;
if (taskAttempt_1.GetLaunchTime() != 0 && taskAttempt_1.GetFinishTime() != 0)
{
attemptRunTime = (int)(taskAttempt_1.GetFinishTime() - taskAttempt_1.GetLaunchTime
());
}
// Default to KILLED
TaskAttemptCompletionEventStatus taceStatus = TaskAttemptCompletionEventStatus.Killed;
string taStateString = taskAttempt_1.GetState().ToString();
try
{
taceStatus = TaskAttemptCompletionEventStatus.ValueOf(taStateString);
}
catch (Exception)
{
Log.Warn("Cannot constuct TACEStatus from TaskAtemptState: [" + taStateString + "] for taskAttemptId: ["
+ taskAttempt_1.GetID() + "]. Defaulting to KILLED");
}
tace.SetAttemptId(taskAttempt_1.GetID());
tace.SetAttemptRunTime(attemptRunTime);
tace.SetEventId(eventId++);
tace.SetMapOutputServerAddress(taskAttempt_1.GetAssignedContainerMgrAddress());
tace.SetStatus(taceStatus);
completionEvents.AddItem(tace);
if (taskAttempt_1.GetID().GetTaskId().GetTaskType() == TaskType.Map)
{
mapCompletionEvents.AddItem(tace);
}
}
}
private static TaskAttemptCompletionEvent[] GetAttemptCompletionEvents(IList <TaskAttemptCompletionEvent
> eventList, int startIndex, int maxEvents)
{
TaskAttemptCompletionEvent[] events = new TaskAttemptCompletionEvent[0];
if (eventList.Count > startIndex)
{
int actualMax = Math.Min(maxEvents, (eventList.Count - startIndex));
events = Sharpen.Collections.ToArray(eventList.SubList(startIndex, actualMax + startIndex
), events);
}
return(events);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetMapCompletionEvents()
{
TaskAttemptCompletionEvent[] empty = new TaskAttemptCompletionEvent[] { };
TaskAttemptCompletionEvent[] taskEvents = new TaskAttemptCompletionEvent[] { CreateTce
(0, true, TaskAttemptCompletionEventStatus.Obsolete), CreateTce(1, false, TaskAttemptCompletionEventStatus
.Failed), CreateTce(2, true, TaskAttemptCompletionEventStatus.Succeeded), CreateTce
(3, false, TaskAttemptCompletionEventStatus.Failed) };
TaskAttemptCompletionEvent[] mapEvents = new TaskAttemptCompletionEvent[] { taskEvents
[0], taskEvents[2] };
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 100)).ThenReturn
(taskEvents);
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 2)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 0, 2));
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(2, 100)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 2, 4));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 100)).ThenReturn
(TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 2)).ThenReturn(
TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(2, 100)).ThenReturn
(TypeConverter.FromYarn(empty));
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob
);
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_200(hbHandler
, appCtx, secret, rmHeartbeatHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JobID jid = new JobID("12345", 1);
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0);
MapTaskCompletionEventsUpdate update = listener.GetMapCompletionEvents(jid, 0, 100
, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 0, 2, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 2, 100, tid);
NUnit.Framework.Assert.AreEqual(0, update.events.Length);
}
private static TaskAttemptCompletionEvent CreateTce(int eventId, bool isMap, TaskAttemptCompletionEventStatus
status)
{
JobId jid = MRBuilderUtils.NewJobId(12345, 1, 1);
TaskId tid = MRBuilderUtils.NewTaskId(jid, 0, isMap ? TaskType.Map : TaskType.Reduce
);
TaskAttemptId attemptId = MRBuilderUtils.NewTaskAttemptId(tid, 0);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
TaskAttemptCompletionEvent tce = recordFactory.NewRecordInstance <TaskAttemptCompletionEvent
>();
tce.SetEventId(eventId);
tce.SetAttemptId(attemptId);
tce.SetStatus(status);
return(tce);
}
public JobTaskAttemptCompletedEvent(TaskAttemptCompletionEvent completionEvent)
: base(completionEvent.GetAttemptId().GetTaskId().GetJobId(), JobEventType.JobTaskAttemptCompleted
)
{
this.completionEvent = completionEvent;
}
private MRProtos.TaskAttemptCompletionEventProto ConvertToProtoFormat(TaskAttemptCompletionEvent
t)
{
return(((TaskAttemptCompletionEventPBImpl)t).GetProto());
}
public virtual void AddCompletionEvent(TaskAttemptCompletionEvent completionEvents
)
{
InitCompletionEvents();
this.completionEvents.AddItem(completionEvents);
}
