/// <summary>Ensure that a TASK_ID was passed into the page.</summary>
public virtual void RequireTask()
{
if ($(TaskId).IsEmpty())
{
BadRequest("missing task ID");
throw new RuntimeException("missing task ID");
}
TaskId taskID = MRApps.ToTaskID($(TaskId));
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.context.GetJob(taskID.GetJobId
());
app.SetJob(job);
if (app.GetJob() == null)
{
NotFound(MRApps.ToString(taskID.GetJobId()));
throw new RuntimeException("Not Found: " + $(JobId));
}
else
{
app.SetTask(app.GetJob().GetTask(taskID));
if (app.GetTask() == null)
{
NotFound($(TaskId));
throw new RuntimeException("Not Found: " + $(TaskId));
}
}
if (!CheckAccess(job))
{
AccessDenied("User " + Request().GetRemoteUser() + " does not have " + " permission to view job "
+ $(JobId));
throw new RuntimeException("Access denied: User " + Request().GetRemoteUser() + " does not have permission to view job "
+ $(JobId));
}
}
public virtual void TestToTaskID()
{
TaskId tid = MRApps.ToTaskID("task_1_2_r_3");
NUnit.Framework.Assert.AreEqual(1, tid.GetJobId().GetAppId().GetClusterTimestamp(
));
NUnit.Framework.Assert.AreEqual(2, tid.GetJobId().GetAppId().GetId());
NUnit.Framework.Assert.AreEqual(2, tid.GetJobId().GetId());
NUnit.Framework.Assert.AreEqual(TaskType.Reduce, tid.GetTaskType());
NUnit.Framework.Assert.AreEqual(3, tid.GetId());
tid = MRApps.ToTaskID("task_1_2_m_3");
NUnit.Framework.Assert.AreEqual(TaskType.Map, tid.GetTaskType());
}
/// <summary>Absorbs one TaskAttemptStatus</summary>
/// <param name="reportedStatus">
/// the status report that we got from a task attempt
/// that we want to fold into the speculation data for this job
/// </param>
/// <param name="timestamp">
/// the time this status corresponds to. This matters
/// because statuses contain progress.
/// </param>
protected internal virtual void StatusUpdate(TaskAttemptStatusUpdateEvent.TaskAttemptStatus
reportedStatus, long timestamp)
{
string stateString = reportedStatus.taskState.ToString();
TaskAttemptId attemptID = reportedStatus.id;
TaskId taskID = attemptID.GetTaskId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(taskID.GetJobId()
);
if (job == null)
{
return;
}
Task task = job.GetTask(taskID);
if (task == null)
{
return;
}
estimator.UpdateAttempt(reportedStatus, timestamp);
if (stateString.Equals(TaskAttemptState.Running.ToString()))
{
runningTasks.PutIfAbsent(taskID, true);
}
else
{
runningTasks.Remove(taskID, true);
if (!stateString.Equals(TaskAttemptState.Starting.ToString()))
{
Sharpen.Collections.Remove(runningTaskAttemptStatistics, attemptID);
}
}
}
/// <exception cref="System.IO.IOException"/>
private Task VerifyAndGetTask(TaskId taskID, JobACL accessType)
{
Task task = this.VerifyAndGetJob(taskID.GetJobId(), accessType, true).GetTask(taskID
);
if (task == null)
{
throw new IOException("Unknown Task " + taskID);
}
return(task);
}
public virtual void UpdateAttempt(TaskAttemptStatusUpdateEvent.TaskAttemptStatus
status, long timestamp)
{
TaskAttemptId attemptID = status.id;
TaskId taskID = attemptID.GetTaskId();
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
if (job == null)
{
return;
}
Task task = job.GetTask(taskID);
if (task == null)
{
return;
}
long boxedStart = startTimes[attemptID];
long start = boxedStart == null ? long.MinValue : boxedStart;
TaskAttempt taskAttempt = task.GetAttempt(attemptID);
if (taskAttempt.GetState() == TaskAttemptState.Succeeded)
{
bool isNew = false;
// is this a new success?
lock (doneTasks)
{
if (!doneTasks.Contains(task))
{
doneTasks.AddItem(task);
isNew = true;
}
}
// It's a new completion
// Note that if a task completes twice [because of a previous speculation
// and a race, or a success followed by loss of the machine with the
// local data] we only count the first one.
if (isNew)
{
long finish = timestamp;
if (start > 1L && finish > 1L && start <= finish)
{
long duration = finish - start;
DataStatistics statistics = DataStatisticsForTask(taskID);
if (statistics != null)
{
statistics.Add(duration);
}
}
}
}
}
internal virtual bool CanSpeculate(AppContext context, TaskId taskID)
{
// This class rejects speculating any task that already has speculations,
// or isn't running.
// Subclasses should call TaskSpeculationPredicate.canSpeculate(...) , but
// can be even more restrictive.
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
Task task = job.GetTask(taskID);
return(task.GetAttempts().Count == 1);
}
/* ************************************************************* */
// This section contains the code that gets run for a SpeculatorEvent
private AtomicInteger ContainerNeed(TaskId taskID)
{
JobId jobID = taskID.GetJobId();
TaskType taskType = taskID.GetTaskType();
ConcurrentMap <JobId, AtomicInteger> relevantMap = taskType == TaskType.Map ? mapContainerNeeds
: reduceContainerNeeds;
AtomicInteger result = relevantMap[jobID];
if (result == null)
{
relevantMap.PutIfAbsent(jobID, new AtomicInteger(0));
result = relevantMap[jobID];
}
return(result);
}
protected internal virtual DataStatistics DataStatisticsForTask(TaskId taskID)
{
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
if (job == null)
{
return(null);
}
Task task = job.GetTask(taskID);
if (task == null)
{
return(null);
}
return(task.GetType() == TaskType.Map ? mapperStatistics[job] : task.GetType() ==
TaskType.Reduce ? reducerStatistics[job] : null);
}
//TODO_get.set
public virtual void TestTaskIDtoString()
{
TaskId tid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <TaskId
>();
tid.SetJobId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobId
>());
tid.GetJobId().SetAppId(ApplicationId.NewInstance(0, 0));
tid.SetTaskType(TaskType.Map);
TaskType type = tid.GetTaskType();
System.Console.Error.WriteLine(type);
type = TaskType.Reduce;
System.Console.Error.WriteLine(type);
System.Console.Error.WriteLine(tid.GetTaskType());
NUnit.Framework.Assert.AreEqual("task_0_0000_m_000000", MRApps.ToString(tid));
tid.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual("task_0_0000_r_000000", MRApps.ToString(tid));
}
public virtual void TestSingleTaskCounterView()
{
AppContext appContext = new MockAppContext(0, 1, 1, 2);
IDictionary <string, string> @params = GetTaskParams(appContext);
@params[AMParams.CounterGroup] = "org.apache.hadoop.mapreduce.FileSystemCounter";
@params[AMParams.CounterName] = "HDFS_WRITE_OPS";
// remove counters from one task attempt
// to test handling of missing counters
TaskId taskID = MRApps.ToTaskID(@params[AMParams.TaskId]);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(taskID.GetJobId
());
Task task = job.GetTask(taskID);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
attempt.GetReport().SetCounters(null);
WebAppTests.TestPage <AppContext>(typeof(SingleCounterPage), appContext, @params);
}
public virtual long ThresholdRuntime(TaskId taskID)
{
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
TaskType type = taskID.GetTaskType();
DataStatistics statistics = DataStatisticsForTask(taskID);
int completedTasksOfType = type == TaskType.Map ? job.GetCompletedMaps() : job.GetCompletedReduces
();
int totalTasksOfType = type == TaskType.Map ? job.GetTotalMaps() : job.GetTotalReduces
();
if (completedTasksOfType < MinimumCompleteNumberToSpeculate || (((float)completedTasksOfType
) / totalTasksOfType) < MinimumCompleteProportionToSpeculate)
{
return(long.MaxValue);
}
long result = statistics == null ? long.MaxValue : (long)statistics.Outlier(slowTaskRelativeTresholds
[job]);
return(result);
}
private long StoredPerAttemptValue(IDictionary <TaskAttempt, AtomicLong> data, TaskAttemptId
attemptID)
{
TaskId taskID = attemptID.GetTaskId();
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
Task task = job.GetTask(taskID);
if (task == null)
{
return(-1L);
}
TaskAttempt taskAttempt = task.GetAttempt(attemptID);
if (taskAttempt == null)
{
return(-1L);
}
AtomicLong estimate = data[taskAttempt];
return(estimate == null ? -1L : estimate.Get());
}
private void PopulateMembers(AppContext ctx)
{
JobId jobID = null;
TaskId taskID = null;
string tid = $(AMParams.TaskId);
if (!tid.IsEmpty())
{
taskID = MRApps.ToTaskID(tid);
jobID = taskID.GetJobId();
}
else
{
string jid = $(AMParams.JobId);
if (!jid.IsEmpty())
{
jobID = MRApps.ToJobID(jid);
}
}
if (jobID == null)
{
return;
}
job = ctx.GetJob(jobID);
if (job == null)
{
return;
}
if (taskID != null)
{
task = job.GetTask(taskID);
if (task == null)
{
return;
}
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> entry in task.GetAttempts())
{
long value = 0;
Counters counters = entry.Value.GetCounters();
CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup
)) : null;
if (group != null)
{
Counter c = group.FindCounter($(AMParams.CounterName));
if (c != null)
{
value = c.GetValue();
}
}
values[MRApps.ToString(entry.Key)] = value;
}
return;
}
// Get all types of counters
IDictionary <TaskId, Task> tasks = job.GetTasks();
foreach (KeyValuePair <TaskId, Task> entry_1 in tasks)
{
long value = 0;
Counters counters = entry_1.Value.GetCounters();
CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup
)) : null;
if (group != null)
{
Counter c = group.FindCounter($(AMParams.CounterName));
if (c != null)
{
value = c.GetValue();
}
}
values[MRApps.ToString(entry_1.Key)] = value;
}
}
public JobMapTaskRescheduledEvent(TaskId taskID)
: base(taskID.GetJobId(), JobEventType.JobMapTaskRescheduled)
{
this.taskID = taskID;
}
public JobTaskEvent(TaskId taskID, TaskState taskState)
: base(taskID.GetJobId(), JobEventType.JobTaskCompleted)
{
this.taskID = taskID;
this.taskState = taskState;
}
/// <exception cref="System.IO.IOException"/>
public virtual GetTaskReportResponse GetTaskReport(GetTaskReportRequest request)
{
TaskId taskId = request.GetTaskId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(taskId.GetJobId
(), true);
GetTaskReportResponse response = this.recordFactory.NewRecordInstance <GetTaskReportResponse
>();
response.SetTaskReport(job.GetTask(taskId).GetReport());
return(response);
}
public static TaskID FromYarn(TaskId id)
{
return(new TaskID(FromYarn(id.GetJobId()), FromYarn(id.GetTaskType()), id.GetId()
));
}
public override void UpdateAttempt(TaskAttemptStatusUpdateEvent.TaskAttemptStatus
status, long timestamp)
{
base.UpdateAttempt(status, timestamp);
TaskAttemptId attemptID = status.id;
TaskId taskID = attemptID.GetTaskId();
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
if (job == null)
{
return;
}
Task task = job.GetTask(taskID);
if (task == null)
{
return;
}
TaskAttempt taskAttempt = task.GetAttempt(attemptID);
if (taskAttempt == null)
{
return;
}
long boxedStart = startTimes[attemptID];
long start = boxedStart == null ? long.MinValue : boxedStart;
// We need to do two things.
// 1: If this is a completion, we accumulate statistics in the superclass
// 2: If this is not a completion, we learn more about it.
// This is not a completion, but we're cooking.
//
if (taskAttempt.GetState() == TaskAttemptState.Running)
{
// See if this task is already in the registry
AtomicLong estimateContainer = attemptRuntimeEstimates[taskAttempt];
AtomicLong estimateVarianceContainer = attemptRuntimeEstimateVariances[taskAttempt
];
if (estimateContainer == null)
{
if (attemptRuntimeEstimates[taskAttempt] == null)
{
attemptRuntimeEstimates[taskAttempt] = new AtomicLong();
estimateContainer = attemptRuntimeEstimates[taskAttempt];
}
}
if (estimateVarianceContainer == null)
{
attemptRuntimeEstimateVariances.PutIfAbsent(taskAttempt, new AtomicLong());
estimateVarianceContainer = attemptRuntimeEstimateVariances[taskAttempt];
}
long estimate = -1;
long varianceEstimate = -1;
// This code assumes that we'll never consider starting a third
// speculative task attempt if two are already running for this task
if (start > 0 && timestamp > start)
{
estimate = (long)((timestamp - start) / Math.Max(0.0001, status.progress));
varianceEstimate = (long)(estimate * status.progress / 10);
}
if (estimateContainer != null)
{
estimateContainer.Set(estimate);
}
if (estimateVarianceContainer != null)
{
estimateVarianceContainer.Set(varianceEstimate);
}
}
}
/* ************************************************************* */
// This is the code section that runs periodically and adds speculations for
// those jobs that need them.
// This can return a few magic values for tasks that shouldn't speculate:
// returns ON_SCHEDULE if thresholdRuntime(taskID) says that we should not
// considering speculating this task
// returns ALREADY_SPECULATING if that is true. This has priority.
// returns TOO_NEW if our companion task hasn't gotten any information
// returns PROGRESS_IS_GOOD if the task is sailing through
// returns NOT_RUNNING if the task is not running
//
// All of these values are negative. Any value that should be allowed to
// speculate is 0 or positive.
private long SpeculationValue(TaskId taskID, long now)
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(taskID.GetJobId()
);
Task task = job.GetTask(taskID);
IDictionary <TaskAttemptId, TaskAttempt> attempts = task.GetAttempts();
long acceptableRuntime = long.MinValue;
long result = long.MinValue;
if (!mayHaveSpeculated.Contains(taskID))
{
acceptableRuntime = estimator.ThresholdRuntime(taskID);
if (acceptableRuntime == long.MaxValue)
{
return(OnSchedule);
}
}
TaskAttemptId runningTaskAttemptID = null;
int numberRunningAttempts = 0;
foreach (TaskAttempt taskAttempt in attempts.Values)
{
if (taskAttempt.GetState() == TaskAttemptState.Running || taskAttempt.GetState()
== TaskAttemptState.Starting)
{
if (++numberRunningAttempts > 1)
{
return(AlreadySpeculating);
}
runningTaskAttemptID = taskAttempt.GetID();
long estimatedRunTime = estimator.EstimatedRuntime(runningTaskAttemptID);
long taskAttemptStartTime = estimator.AttemptEnrolledTime(runningTaskAttemptID);
if (taskAttemptStartTime > now)
{
// This background process ran before we could process the task
// attempt status change that chronicles the attempt start
return(TooNew);
}
long estimatedEndTime = estimatedRunTime + taskAttemptStartTime;
long estimatedReplacementEndTime = now + estimator.EstimatedNewAttemptRuntime(taskID
);
float progress = taskAttempt.GetProgress();
DefaultSpeculator.TaskAttemptHistoryStatistics data = runningTaskAttemptStatistics
[runningTaskAttemptID];
if (data == null)
{
runningTaskAttemptStatistics[runningTaskAttemptID] = new DefaultSpeculator.TaskAttemptHistoryStatistics
(estimatedRunTime, progress, now);
}
else
{
if (estimatedRunTime == data.GetEstimatedRunTime() && progress == data.GetProgress
())
{
// Previous stats are same as same stats
if (data.NotHeartbeatedInAWhile(now))
{
// Stats have stagnated for a while, simulate heart-beat.
TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
taskAttemptStatus.id = runningTaskAttemptID;
taskAttemptStatus.progress = progress;
taskAttemptStatus.taskState = taskAttempt.GetState();
// Now simulate the heart-beat
HandleAttempt(taskAttemptStatus);
}
}
else
{
// Stats have changed - update our data structure
data.SetEstimatedRunTime(estimatedRunTime);
data.SetProgress(progress);
data.ResetHeartBeatTime(now);
}
}
if (estimatedEndTime < now)
{
return(ProgressIsGood);
}
if (estimatedReplacementEndTime >= estimatedEndTime)
{
return(TooLateToSpeculate);
}
result = estimatedEndTime - estimatedReplacementEndTime;
}
}
// If we are here, there's at most one task attempt.
if (numberRunningAttempts == 0)
{
return(NotRunning);
}
if (acceptableRuntime == long.MinValue)
{
acceptableRuntime = estimator.ThresholdRuntime(taskID);
if (acceptableRuntime == long.MaxValue)
{
return(OnSchedule);
}
}
return(result);
}
private void GetCounters(AppContext ctx)
{
JobId jobID = null;
TaskId taskID = null;
string tid = $(AMParams.TaskId);
if (!tid.IsEmpty())
{
taskID = MRApps.ToTaskID(tid);
jobID = taskID.GetJobId();
}
else
{
string jid = $(AMParams.JobId);
if (jid != null && !jid.IsEmpty())
{
jobID = MRApps.ToJobID(jid);
}
}
if (jobID == null)
{
return;
}
job = ctx.GetJob(jobID);
if (job == null)
{
return;
}
if (taskID != null)
{
task = job.GetTask(taskID);
if (task == null)
{
return;
}
total = task.GetCounters();
return;
}
// Get all types of counters
IDictionary <TaskId, Task> tasks = job.GetTasks();
total = job.GetAllCounters();
bool needTotalCounters = false;
if (total == null)
{
total = new Counters();
needTotalCounters = true;
}
map = new Counters();
reduce = new Counters();
foreach (Task t in tasks.Values)
{
Counters counters = t.GetCounters();
if (counters == null)
{
continue;
}
switch (t.GetType())
{
case TaskType.Map:
{
map.IncrAllCounters(counters);
break;
}
case TaskType.Reduce:
{
reduce.IncrAllCounters(counters);
break;
}
}
if (needTotalCounters)
{
total.IncrAllCounters(counters);
}
}
}
