public virtual void TestToTaskID()
{
TaskId tid = MRApps.ToTaskID("task_1_2_r_3");
NUnit.Framework.Assert.AreEqual(1, tid.GetJobId().GetAppId().GetClusterTimestamp(
));
NUnit.Framework.Assert.AreEqual(2, tid.GetJobId().GetAppId().GetId());
NUnit.Framework.Assert.AreEqual(2, tid.GetJobId().GetId());
NUnit.Framework.Assert.AreEqual(TaskType.Reduce, tid.GetTaskType());
NUnit.Framework.Assert.AreEqual(3, tid.GetId());
tid = MRApps.ToTaskID("task_1_2_m_3");
NUnit.Framework.Assert.AreEqual(TaskType.Map, tid.GetTaskType());
}
/// <returns>
/// The end of the JS map that is the jquery datatable config for the
/// attempts table.
/// </returns>
private string AttemptsTableInit()
{
TaskType type = null;
string symbol = $(AMParams.TaskType);
if (!symbol.IsEmpty())
{
type = MRApps.TaskType(symbol);
}
else
{
TaskId taskID = MRApps.ToTaskID($(AMParams.TaskId));
type = taskID.GetTaskType();
}
StringBuilder b = JQueryUI.TableInit().Append(", 'aaData': attemptsTableData").Append
(", bDeferRender: true").Append(", bProcessing: true").Append("\n,aoColumnDefs:[\n"
).Append("\n{'aTargets': [ 4 ]").Append(", 'bSearchable': false }").Append("\n, {'sType':'numeric', 'aTargets': [ 0 ]"
).Append(", 'mRender': parseHadoopAttemptID }").Append("\n, {'sType':'numeric', 'aTargets': [ 5, 6"
).Append(type == TaskType.Reduce ? ", 7, 8" : string.Empty).Append(" ], 'mRender': renderHadoopDate }"
).Append("\n, {'sType':'numeric', 'aTargets': [").Append(type == TaskType.Reduce
? "9, 10, 11, 12" : "7").Append(" ], 'mRender': renderHadoopElapsedTime }]").Append
("\n, aaSorting: [[0, 'asc']]").Append("}");
//logs column should not filterable (it includes container ID which may pollute searches)
//Column numbers are different for maps and reduces
// Sort by id upon page load
return(b.ToString());
}
//TODO_get.set
public virtual void TestTaskIDtoString()
{
TaskId tid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <TaskId
>();
tid.SetJobId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobId
>());
tid.GetJobId().SetAppId(ApplicationId.NewInstance(0, 0));
tid.SetTaskType(TaskType.Map);
TaskType type = tid.GetTaskType();
System.Console.Error.WriteLine(type);
type = TaskType.Reduce;
System.Console.Error.WriteLine(type);
System.Console.Error.WriteLine(tid.GetTaskType());
NUnit.Framework.Assert.AreEqual("task_0_0000_m_000000", MRApps.ToString(tid));
tid.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual("task_0_0000_r_000000", MRApps.ToString(tid));
}
/* ************************************************************* */
// This section contains the code that gets run for a SpeculatorEvent
private AtomicInteger ContainerNeed(TaskId taskID)
{
JobId jobID = taskID.GetJobId();
TaskType taskType = taskID.GetTaskType();
ConcurrentMap <JobId, AtomicInteger> relevantMap = taskType == TaskType.Map ? mapContainerNeeds
: reduceContainerNeeds;
AtomicInteger result = relevantMap[jobID];
if (result == null)
{
relevantMap.PutIfAbsent(jobID, new AtomicInteger(0));
result = relevantMap[jobID];
}
return(result);
}
public virtual long ThresholdRuntime(TaskId taskID)
{
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
TaskType type = taskID.GetTaskType();
DataStatistics statistics = DataStatisticsForTask(taskID);
int completedTasksOfType = type == TaskType.Map ? job.GetCompletedMaps() : job.GetCompletedReduces
();
int totalTasksOfType = type == TaskType.Map ? job.GetTotalMaps() : job.GetTotalReduces
();
if (completedTasksOfType < MinimumCompleteNumberToSpeculate || (((float)completedTasksOfType
) / totalTasksOfType) < MinimumCompleteProportionToSpeculate)
{
return(long.MaxValue);
}
long result = statistics == null ? long.MaxValue : (long)statistics.Outlier(slowTaskRelativeTresholds
[job]);
return(result);
}
public static TaskID FromYarn(TaskId id)
{
return(new TaskID(FromYarn(id.GetJobId()), FromYarn(id.GetTaskType()), id.GetId()
));
}
