/// <exception cref="System.IO.IOException"/>
public virtual void CleanUpPartialOutputForTask(TaskAttemptContext context)
{
// we double check this is never invoked from a non-preemptable subclass.
// This should never happen, since the invoking codes is checking it too,
// but it is safer to double check. Errors handling this would produce
// inconsistent output.
if (!this.GetType().IsAnnotationPresent(typeof(Checkpointable)))
{
throw new InvalidOperationException("Invoking cleanUpPartialOutputForTask() " + "from non @Preemptable class"
);
}
FileSystem fs = FsFor(GetTaskAttemptPath(context), context.GetConfiguration());
Log.Info("cleanUpPartialOutputForTask: removing everything belonging to " + context
.GetTaskAttemptID().GetTaskID() + " in: " + GetCommittedTaskPath(context).GetParent
());
TaskAttemptID taid = context.GetTaskAttemptID();
TaskID tid = taid.GetTaskID();
Path pCommit = GetCommittedTaskPath(context).GetParent();
// remove any committed output
for (int i = 0; i < taid.GetId(); ++i)
{
TaskAttemptID oldId = new TaskAttemptID(tid, i);
Path pTask = new Path(pCommit, oldId.ToString());
if (fs.Exists(pTask) && !fs.Delete(pTask, true))
{
throw new IOException("Failed to delete " + pTask);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <IntWritable> values, Reducer.Context
context)
{
// Make one reducer slower for speculative execution
TaskAttemptID taid = context.GetTaskAttemptID();
long sleepTime = 100;
Configuration conf = context.GetConfiguration();
bool test_speculate_reduce = conf.GetBoolean(MRJobConfig.ReduceSpeculative, false
);
// IF TESTING REDUCE SPECULATIVE EXECUTION:
// Make the "*_r_000000_0" attempt take much longer than the others.
// When speculative execution is enabled, this should cause the attempt
// to be killed and restarted. At that point, the attempt ID will be
// "*_r_000000_1", so sleepTime will still remain 100ms.
if ((taid.GetTaskType() == TaskType.Reduce) && test_speculate_reduce && (taid.GetTaskID
().GetId() == 0) && (taid.GetId() == 0))
{
sleepTime = 10000;
}
try
{
Sharpen.Thread.Sleep(sleepTime);
}
catch (Exception)
{
}
// Ignore
context.Write(key, new IntWritable(0));
}
