public virtual void TestSingleEntryDataStatistics()
{
DataStatistics statistics = new DataStatistics(17.29);
NUnit.Framework.Assert.AreEqual(1, statistics.Count(), Tol);
NUnit.Framework.Assert.AreEqual(17.29, statistics.Mean(), Tol);
NUnit.Framework.Assert.AreEqual(0, statistics.Var(), Tol);
NUnit.Framework.Assert.AreEqual(0, statistics.Std(), Tol);
NUnit.Framework.Assert.AreEqual(17.29, statistics.Outlier(1.0f), Tol);
}
public virtual long EstimatedNewAttemptRuntime(TaskId id)
{
DataStatistics statistics = DataStatisticsForTask(id);
if (statistics == null)
{
return(-1L);
}
return((long)statistics.Mean());
}
public virtual void TestMutiEntryDataStatistics()
{
DataStatistics statistics = new DataStatistics();
statistics.Add(17);
statistics.Add(29);
NUnit.Framework.Assert.AreEqual(2, statistics.Count(), Tol);
NUnit.Framework.Assert.AreEqual(23.0, statistics.Mean(), Tol);
NUnit.Framework.Assert.AreEqual(36.0, statistics.Var(), Tol);
NUnit.Framework.Assert.AreEqual(6.0, statistics.Std(), Tol);
NUnit.Framework.Assert.AreEqual(29.0, statistics.Outlier(1.0f), Tol);
}
public virtual void UpdateAttempt(TaskAttemptStatusUpdateEvent.TaskAttemptStatus
status, long timestamp)
{
TaskAttemptId attemptID = status.id;
TaskId taskID = attemptID.GetTaskId();
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
if (job == null)
{
return;
}
Task task = job.GetTask(taskID);
if (task == null)
{
return;
}
long boxedStart = startTimes[attemptID];
long start = boxedStart == null ? long.MinValue : boxedStart;
TaskAttempt taskAttempt = task.GetAttempt(attemptID);
if (taskAttempt.GetState() == TaskAttemptState.Succeeded)
{
bool isNew = false;
// is this a new success?
lock (doneTasks)
{
if (!doneTasks.Contains(task))
{
doneTasks.AddItem(task);
isNew = true;
}
}
// It's a new completion
// Note that if a task completes twice [because of a previous speculation
// and a race, or a success followed by loss of the machine with the
// local data] we only count the first one.
if (isNew)
{
long finish = timestamp;
if (start > 1L && finish > 1L && start <= finish)
{
long duration = finish - start;
DataStatistics statistics = DataStatisticsForTask(taskID);
if (statistics != null)
{
statistics.Add(duration);
}
}
}
}
}
public virtual void TestUpdateStatistics()
{
DataStatistics statistics = new DataStatistics(17);
statistics.Add(29);
NUnit.Framework.Assert.AreEqual(2, statistics.Count(), Tol);
NUnit.Framework.Assert.AreEqual(23.0, statistics.Mean(), Tol);
NUnit.Framework.Assert.AreEqual(36.0, statistics.Var(), Tol);
statistics.UpdateStatistics(17, 29);
NUnit.Framework.Assert.AreEqual(2, statistics.Count(), Tol);
NUnit.Framework.Assert.AreEqual(29.0, statistics.Mean(), Tol);
NUnit.Framework.Assert.AreEqual(0.0, statistics.Var(), Tol);
}
public virtual void Contextualize(Configuration conf, AppContext context)
{
this.context = context;
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> allJobs = context.
GetAllJobs();
foreach (KeyValuePair <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> entry in
allJobs)
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = entry.Value;
mapperStatistics[job] = new DataStatistics();
reducerStatistics[job] = new DataStatistics();
slowTaskRelativeTresholds[job] = conf.GetFloat(MRJobConfig.SpeculativeSlowtaskThreshold
, 1.0f);
}
}
public virtual long ThresholdRuntime(TaskId taskID)
{
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
TaskType type = taskID.GetTaskType();
DataStatistics statistics = DataStatisticsForTask(taskID);
int completedTasksOfType = type == TaskType.Map ? job.GetCompletedMaps() : job.GetCompletedReduces
();
int totalTasksOfType = type == TaskType.Map ? job.GetTotalMaps() : job.GetTotalReduces
();
if (completedTasksOfType < MinimumCompleteNumberToSpeculate || (((float)completedTasksOfType
) / totalTasksOfType) < MinimumCompleteProportionToSpeculate)
{
return(long.MaxValue);
}
long result = statistics == null ? long.MaxValue : (long)statistics.Outlier(slowTaskRelativeTresholds
[job]);
return(result);
}
