/// <summary>Validate job.end.retry.attempts for the localJobRunner.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestLocalJobRunnerRetryCount()
{
int retryAttempts = 3;
JobStatus jobStatus = CreateTestJobStatus("job_20130313155005308_0001", JobStatus
.Succeeded);
JobConf jobConf = CreateTestJobConf(new Configuration(), retryAttempts, baseUrl +
"fail");
JobEndNotifier.LocalRunnerNotification(jobConf, jobStatus);
// Validate params
NUnit.Framework.Assert.AreEqual(retryAttempts + 1, TestJobEndNotifier.FailServlet
.calledTimes);
}
/// <summary>Basic validation for localRunnerNotification.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestLocalJobRunnerUriSubstitution()
{
JobStatus jobStatus = CreateTestJobStatus("job_20130313155005308_0001", JobStatus
.Succeeded);
JobConf jobConf = CreateTestJobConf(new Configuration(), 0, baseUrl + "jobend?jobid=$jobId&status=$jobStatus"
);
JobEndNotifier.LocalRunnerNotification(jobConf, jobStatus);
// No need to wait for the notification to go thru since calls are
// synchronous
// Validate params
NUnit.Framework.Assert.AreEqual(1, TestJobEndNotifier.JobEndServlet.calledTimes);
NUnit.Framework.Assert.AreEqual("jobid=job_20130313155005308_0001&status=SUCCEEDED"
, TestJobEndNotifier.JobEndServlet.requestUri.GetQuery());
}
/// <summary>
/// Validate that the notification times out after reaching
/// mapreduce.job.end-notification.timeout.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestNotificationTimeout()
{
Configuration conf = new Configuration();
// Reduce the timeout to 1 second
conf.SetInt("mapreduce.job.end-notification.timeout", 1000);
JobStatus jobStatus = CreateTestJobStatus("job_20130313155005308_0001", JobStatus
.Succeeded);
JobConf jobConf = CreateTestJobConf(conf, 0, baseUrl + "delay");
long startTime = Runtime.CurrentTimeMillis();
JobEndNotifier.LocalRunnerNotification(jobConf, jobStatus);
long elapsedTime = Runtime.CurrentTimeMillis() - startTime;
// Validate params
NUnit.Framework.Assert.AreEqual(1, TestJobEndNotifier.DelayServlet.calledTimes);
// Make sure we timed out with time slightly above 1 second
// (default timeout is in terms of minutes, so we'll catch the problem)
NUnit.Framework.Assert.IsTrue(elapsedTime < 2000);
}
public override void Run()
{
JobID jobId = this.profile.GetJobID();
JobContext jContext = new JobContextImpl(this.job, jobId);
OutputCommitter outputCommitter = null;
try
{
outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper
(), jobId, this._enclosing.conf);
}
catch (Exception e)
{
LocalJobRunner.Log.Info("Failed to createOutputCommitter", e);
return;
}
try
{
JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, this.localFs, this._enclosing.conf, this.systemJobDir);
int numReduceTasks = this.job.GetNumReduceTasks();
outputCommitter.SetupJob(jContext);
this.status.SetSetupProgress(1.0f);
IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap
(new Dictionary <TaskAttemptID, MapOutputFile>());
IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables
(taskSplitMetaInfos, jobId, mapOutputFiles);
this.InitCounters(mapRunnables.Count, numReduceTasks);
ExecutorService mapService = this.CreateMapExecutor();
this.RunTasks(mapRunnables, mapService, "map");
try
{
if (numReduceTasks > 0)
{
IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables
(jobId, mapOutputFiles);
ExecutorService reduceService = this.CreateReduceExecutor();
this.RunTasks(reduceRunnables, reduceService, "reduce");
}
}
finally
{
foreach (MapOutputFile output in mapOutputFiles.Values)
{
output.RemoveAll();
}
}
// delete the temporary directory in output directory
outputCommitter.CommitJob(jContext);
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Succeeded);
}
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
catch (Exception t)
{
try
{
outputCommitter.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException)
{
LocalJobRunner.Log.Info("Error cleaning up job:" + this.id);
}
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Failed);
}
LocalJobRunner.Log.Warn(this.id, t);
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
finally
{
try
{
this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true);
// delete submit dir
this.localFs.Delete(this.localJobFile, true);
// delete local copy
// Cleanup distributed cache
this.localDistributedCacheManager.Close();
}
catch (IOException e)
{
LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e);
}
}
}
