/// <exception cref="System.Exception"/>
public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf
, bool mapSpeculative, bool reduceSpeculative)
{
string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser(
).GetShortUserName());
conf.Set(MRJobConfig.UserName, user);
conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString());
conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true);
// TODO: fix the bug where the speculator gets events with
// not-fully-constructed objects. For now, disable speculative exec
conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative);
conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative);
Init(conf);
Start();
DefaultMetricsSystem.Shutdown();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values
.GetEnumerator().Next();
if (assignedQueue != null)
{
job.SetQueueName(assignedQueue);
}
// Write job.xml
string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID()
));
Log.Info("Writing job conf to " + jobFile);
new FilePath(jobFile).GetParentFile().Mkdirs();
conf.WriteXml(new FileOutputStream(jobFile));
return(job);
}
public virtual void TestKillJob()
{
CountDownLatch latch = new CountDownLatch(1);
MRApp app = new TestKill.BlockingMRApp(1, 0, latch);
//this will start the job but job won't complete as task is
//blocked
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
//wait and vailidate for Job to become RUNNING
app.WaitForState(job, JobState.Running);
//send the kill signal to Job
app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType.
JobKill));
//unblock Task
latch.CountDown();
//wait and validate for Job to be KILLED
app.WaitForState(job, JobState.Killed);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Killed, task.
GetReport().GetTaskState());
IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 1, attempts.Count
);
IEnumerator <TaskAttempt> it = attempts.Values.GetEnumerator();
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Killed
, it.Next().GetReport().GetTaskAttemptState());
}
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetJob()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(Runtime.CurrentTimeMillis()
, 4);
jobId.SetAppId(appId);
jobId.SetId(1);
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
JobReport report = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100010L);
Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100015L);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(report);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("JobName");
Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("UserName");
Org.Mockito.Mockito.When(job.GetQueueName()).ThenReturn("QueueName");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(3);
Org.Mockito.Mockito.When(job.GetCompletedMaps()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
return(job);
}
public virtual void TestHistoryEvents()
{
Configuration conf = new Configuration();
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(2, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
//make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps
*/
HistoryContext context = new JobHistory();
// test start and stop states
((JobHistory)context).Init(conf);
((JobHistory)context).Start();
NUnit.Framework.Assert.IsTrue(context.GetStartTime() > 0);
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Started);
// get job before stopping JobHistory
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId);
// stop JobHistory
((JobHistory)context).Stop();
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Stopped);
NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 2, parsedJob.GetCompletedMaps
());
NUnit.Framework.Assert.AreEqual(Runtime.GetProperty("user.name"), parsedJob.GetUserName
());
IDictionary <TaskId, Task> tasks = parsedJob.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 3, tasks.Count);
foreach (Task task in tasks.Values)
{
VerifyTask(task);
}
IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map);
NUnit.Framework.Assert.AreEqual("No of maps not correct", 2, maps.Count);
IDictionary <TaskId, Task> reduces = parsedJob.GetTasks(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual("No of reduces not correct", 1, reduces.Count);
NUnit.Framework.Assert.AreEqual("CompletedReduce not correct", 1, parsedJob.GetCompletedReduces
());
NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob
.GetState());
}
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
{
this.id = MRApps.ToString(job.GetID());
JobReport report = job.GetReport();
this.mapsTotal = job.GetTotalMaps();
this.mapsCompleted = job.GetCompletedMaps();
this.reducesTotal = job.GetTotalReduces();
this.reducesCompleted = job.GetCompletedReduces();
this.submitTime = report.GetSubmitTime();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.name = job.GetName().ToString();
this.queue = job.GetQueueName();
this.user = job.GetUserName();
this.state = job.GetState().ToString();
this.acls = new AList <ConfEntryInfo>();
if (job is CompletedJob)
{
avgMapTime = 0l;
avgReduceTime = 0l;
avgShuffleTime = 0l;
avgMergeTime = 0l;
failedReduceAttempts = 0;
killedReduceAttempts = 0;
successfulReduceAttempts = 0;
failedMapAttempts = 0;
killedMapAttempts = 0;
successfulMapAttempts = 0;
CountTasksAndAttempts(job);
this.uberized = job.IsUber();
this.diagnostics = string.Empty;
IList <string> diagnostics = job.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
this.diagnostics = b.ToString();
}
IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
if (allacls != null)
{
foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
{
this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
()));
}
}
}
}
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess)
{
// ok for any user to see
// these should only be seen if acls allow
this.id = MRApps.ToString(job.GetID());
JobReport report = job.GetReport();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime);
if (this.elapsedTime == -1)
{
this.elapsedTime = 0;
}
this.name = job.GetName().ToString();
this.user = job.GetUserName();
this.state = job.GetState();
this.mapsTotal = job.GetTotalMaps();
this.mapsCompleted = job.GetCompletedMaps();
this.mapProgress = report.GetMapProgress() * 100;
this.mapProgressPercent = StringHelper.Percent(report.GetMapProgress());
this.reducesTotal = job.GetTotalReduces();
this.reducesCompleted = job.GetCompletedReduces();
this.reduceProgress = report.GetReduceProgress() * 100;
this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress());
this.acls = new AList <ConfEntryInfo>();
if (hasAccess)
{
this.diagnostics = string.Empty;
CountTasksAndAttempts(job);
this.uberized = job.IsUber();
IList <string> diagnostics = job.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
this.diagnostics = b.ToString();
}
IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
if (allacls != null)
{
foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
{
this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
()));
}
}
}
}
public static IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> NewJobs
(ApplicationId appID, int numJobsPerApp, int numTasksPerJob, int numAttemptsPerTask
)
{
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> map = Maps.NewHashMap
();
for (int j = 0; j < numJobsPerApp; ++j)
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = NewJob(appID, j, numTasksPerJob,
numAttemptsPerTask);
map[job.GetID()] = job;
}
return(map);
}
public virtual AMAttemptsInfo GetJobAttempts(string jid)
{
Init();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = AMWebServices.GetJobFromJobIdString
(jid, ctx);
AMAttemptsInfo amAttempts = new AMAttemptsInfo();
foreach (AMInfo amInfo in job.GetAMInfos())
{
AMAttemptInfo attempt = new AMAttemptInfo(amInfo, MRApps.ToString(job.GetID()), job
.GetUserName(), uriInfo.GetBaseUri().ToString(), webapp.Name());
amAttempts.Add(attempt);
}
return(amAttempts);
}
/// <summary>test clean old history files.</summary>
/// <remarks>
/// test clean old history files. Files should be deleted after 1 week by
/// default.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDeleteFileInfo()
{
Log.Info("STARTING testDeleteFileInfo");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
HistoryFileManager hfm = new HistoryFileManager();
hfm.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId);
hfm.InitExisting();
// wait for move files form the done_intermediate directory to the gone
// directory
while (fileInfo.IsMovePending())
{
Sharpen.Thread.Sleep(300);
}
NUnit.Framework.Assert.IsNotNull(hfm.jobListCache.Values());
// try to remove fileInfo
hfm.Clean();
// check that fileInfo does not deleted
NUnit.Framework.Assert.IsFalse(fileInfo.IsDeleted());
// correct live time
hfm.SetMaxHistoryAge(-1);
hfm.Clean();
hfm.Stop();
NUnit.Framework.Assert.IsTrue("Thread pool shutdown", hfm.moveToDoneExecutor.IsTerminated
());
// should be deleted !
NUnit.Framework.Assert.IsTrue("file should be deleted ", fileInfo.IsDeleted());
}
finally
{
Log.Info("FINISHED testDeleteFileInfo");
}
}
/// <exception cref="System.Exception"/>
public virtual void TestScanningOldDirs()
{
Log.Info("STARTING testScanningOldDirs");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
TestJobHistoryParsing.HistoryFileManagerForTest hfm = new TestJobHistoryParsing.HistoryFileManagerForTest
();
hfm.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId);
NUnit.Framework.Assert.IsNotNull("Unable to locate job history", fileInfo);
// force the manager to "forget" the job
hfm.DeleteJobFromJobListCache(fileInfo);
int msecPerSleep = 10;
int msecToSleep = 10 * 1000;
while (fileInfo.IsMovePending() && msecToSleep > 0)
{
NUnit.Framework.Assert.IsTrue(!fileInfo.DidMoveFail());
msecToSleep -= msecPerSleep;
Sharpen.Thread.Sleep(msecPerSleep);
}
NUnit.Framework.Assert.IsTrue("Timeout waiting for history move", msecToSleep > 0
);
fileInfo = hfm.GetFileInfo(jobId);
hfm.Stop();
NUnit.Framework.Assert.IsNotNull("Unable to locate old job history", fileInfo);
NUnit.Framework.Assert.IsTrue("HistoryFileManager not shutdown properly", hfm.moveToDoneExecutor
.IsTerminated());
}
finally
{
Log.Info("FINISHED testScanningOldDirs");
}
}
public virtual void TestSingleCounterView()
{
AppContext appContext = new MockAppContext(0, 1, 1, 1);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetAllJobs().Values.GetEnumerator
().Next();
// add a failed task to the job without any counters
Task failedTask = MockJobs.NewTask(job.GetID(), 2, 1, true);
IDictionary <TaskId, Task> tasks = job.GetTasks();
tasks[failedTask.GetID()] = failedTask;
IDictionary <string, string> @params = GetJobParams(appContext);
@params[AMParams.CounterGroup] = "org.apache.hadoop.mapreduce.FileSystemCounter";
@params[AMParams.CounterName] = "HDFS_WRITE_OPS";
WebAppTests.TestPage <AppContext>(typeof(SingleCounterPage), appContext, @params);
}
public virtual void TestJobRebootNotLastRetryOnUnregistrationFailure()
{
MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
//send an reboot event
app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType.
JobAmReboot));
// return exteranl state as RUNNING since otherwise the JobClient will
// prematurely exit.
app.WaitForState(job, JobState.Running);
}
public virtual void TestAverageReduceTime()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob
>();
Task task1 = Org.Mockito.Mockito.Mock <Task>();
Task task2 = Org.Mockito.Mockito.Mock <Task>();
JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1);
TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce);
TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce);
TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1);
TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2);
TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>();
JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L);
Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L);
Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L);
Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L);
Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L);
Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L);
Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L);
Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1
, taskAttempt1));
Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2
, taskAttempt2));
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1,
task1, taskId2, task2));
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
JobInfo jobInfo = new JobInfo(job);
NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime());
}
public JobCounterInfo(AppContext ctx, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
GetCounters(ctx, job);
counterGroup = new AList <CounterGroupInfo>();
this.id = MRApps.ToString(job.GetID());
if (total != null)
{
foreach (CounterGroup g in total)
{
if (g != null)
{
CounterGroup mg = map == null ? null : map.GetGroup(g.GetName());
CounterGroup rg = reduce == null ? null : reduce.GetGroup(g.GetName());
CounterGroupInfo cginfo = new CounterGroupInfo(g.GetName(), g, mg, rg);
counterGroup.AddItem(cginfo);
}
}
}
}
public virtual void Setup()
{
this.conf = new JobConf();
this.conf.Set(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestHsWebServicesAcls.NullGroupsProvider
).FullName);
this.conf.SetBoolean(MRConfig.MrAclsEnabled, true);
Groups.GetUserToGroupsMappingService(conf);
this.ctx = BuildHistoryContext(this.conf);
WebApp webApp = Org.Mockito.Mockito.Mock <HsWebApp>();
Org.Mockito.Mockito.When(webApp.Name()).ThenReturn("hsmockwebapp");
this.hsWebServices = new HsWebServices(ctx, conf, webApp);
this.hsWebServices.SetResponse(Org.Mockito.Mockito.Mock <HttpServletResponse>());
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = ctx.GetAllJobs().Values.GetEnumerator
().Next();
this.jobIdStr = job.GetID().ToString();
Task task = job.GetTasks().Values.GetEnumerator().Next();
this.taskIdStr = task.GetID().ToString();
this.taskAttemptIdStr = task.GetAttempts().Keys.GetEnumerator().Next().ToString();
}
public virtual void TestHsJobsBlock()
{
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary
<JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetJob();
jobs[job.GetID()] = job;
Org.Mockito.Mockito.When(ctx.GetAllJobs()).ThenReturn(jobs);
HsJobsBlock block = new TestBlocks.HsJobsBlockForTest(this, ctx);
PrintWriter pWriter = new PrintWriter(data);
HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
, 0, false);
block.Render(html);
pWriter.Flush();
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("JobName"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("UserName"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("QueueName"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
}
public virtual void TestJobRebootOnLastRetryOnUnregistrationFailure()
{
// make startCount as 2 since this is last retry which equals to
// DEFAULT_MAX_AM_RETRY
// The last param mocks the unregistration failure
MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true, 2, false);
Configuration conf = new Configuration();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
//send an reboot event
app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType.
JobAmReboot));
app.WaitForInternalState((JobImpl)job, JobStateInternal.Reboot);
// return exteranl state as RUNNING if this is the last retry while
// unregistration fails
app.WaitForState(job, JobState.Running);
}
public virtual void TestAssignedQueue()
{
Configuration conf = new Configuration();
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(2, 1, true, this.GetType().
FullName, true, "assignedQueue");
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
//make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps
*/
HistoryContext context = new JobHistory();
// test start and stop states
((JobHistory)context).Init(conf);
((JobHistory)context).Start();
NUnit.Framework.Assert.IsTrue(context.GetStartTime() > 0);
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Started);
// get job before stopping JobHistory
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId);
// stop JobHistory
((JobHistory)context).Stop();
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Stopped);
NUnit.Framework.Assert.AreEqual("QueueName not correct", "assignedQueue", parsedJob
.GetQueueName());
}
public virtual void TestEventsFlushOnStop()
{
Configuration conf = new Configuration();
MRApp app = new TestJobHistoryEvents.MRAppWithSpecialHistoryHandler(1, 0, true, this
.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps
*/
HistoryContext context = new JobHistory();
((JobHistory)context).Init(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId);
NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 1, parsedJob.GetCompletedMaps
());
IDictionary <TaskId, Task> tasks = parsedJob.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, tasks.Count);
VerifyTask(tasks.Values.GetEnumerator().Next());
IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map);
NUnit.Framework.Assert.AreEqual("No of maps not correct", 1, maps.Count);
NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob
.GetState());
}
/// <exception cref="System.Exception"/>
private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps
)
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name"));
long amStartTimeEst = Runtime.CurrentTimeMillis();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true,
this.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf);
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
}
catch (IOException ioe)
{
Log.Info("Can not get FileContext", ioe);
throw (new Exception("Can not get File Context"));
}
if (numMaps == numSuccessfulMaps)
{
string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId);
Path summaryFile = new Path(jobhistoryDir, summaryFileName);
string jobSummaryString = GetJobSummary(fc, summaryFile);
NUnit.Framework.Assert.IsNotNull(jobSummaryString);
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100"));
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100")
);
IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>();
StringTokenizer strToken = new StringTokenizer(jobSummaryString, ",");
while (strToken.HasMoreTokens())
{
string keypair = strToken.NextToken();
jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1];
}
NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements
["jobId"]);
NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements
["jobName"]);
NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements
["submitTime"]) != 0);
NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements
["launchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstMapTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstReduceTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements
["finishTime"]) != 0);
NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32
(jobSummaryElements["numMaps"]));
NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32
(jobSummaryElements["numReduces"]));
NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name"
), jobSummaryElements["user"]);
NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements
["queue"]);
NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements
["status"]);
}
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser.JobInfo jobInfo;
long numFinishedMaps;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
Log.Info("JobHistoryFile is: " + historyFilePath);
try
{
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
JobHistoryParser parser = new JobHistoryParser(@in);
EventReader realReader = new EventReader(@in);
EventReader reader = Org.Mockito.Mockito.Mock <EventReader>();
if (numMaps == numSuccessfulMaps)
{
reader = realReader;
}
else
{
AtomicInteger numFinishedEvents = new AtomicInteger(0);
// Hack!
Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader
, numFinishedEvents, numSuccessfulMaps));
}
jobInfo = parser.Parse(reader);
numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);
if (numFinishedMaps != numMaps)
{
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException
);
}
}
NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name"
), jobInfo.GetUsername());
NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname(
));
NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName
());
NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath
());
NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps
);
NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo
.GetFinishedReduces());
NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized
());
IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks();
int totalTasks = allTasks.Count;
NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps
+ numReduces), totalTasks);
// Verify aminfo
NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count);
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost
());
JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0];
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort()
);
NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId());
NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId()
.GetApplicationAttemptId());
NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis(
) && amInfo.GetStartTime() >= amStartTimeEst);
ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1);
// Assert at taskAttempt level
foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values)
{
int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count;
NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount
);
JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator
().Next();
NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId());
// Verify the wrong ctor is not being used. Remove after mrv1 is removed.
NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid));
}
// Deep compare Job and JobInfo
foreach (Task task in job.GetTasks().Values)
{
JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID
())];
NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1);
foreach (TaskAttempt taskAttempt in task.GetAttempts().Values)
{
JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts(
)[TypeConverter.FromYarn((taskAttempt.GetID()))];
NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo);
NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt
.GetShufflePort(), taskAttemptInfo.GetShufflePort());
if (numMaps == numSuccessfulMaps)
{
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname());
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort());
// Verify rack-name
NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname
(), RackName);
}
}
}
// test output for HistoryViewer
TextWriter stdps = System.Console.Out;
try
{
Runtime.SetOut(new TextWriter(outContent));
HistoryViewer viewer;
lock (fileInfo)
{
viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString()
, conf, true);
}
viewer.Print();
foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values)
{
string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus
()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId(
).GetJobID();
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0);
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId(
).ToString()) > 0);
}
}
finally
{
Runtime.SetOut(stdps);
}
}
public virtual void Test()
{
TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService
(this, 1, 0, false);
Configuration conf = new Configuration();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
// send the diagnostic
string diagnostic1 = "Diagnostic1";
string diagnostic2 = "Diagnostic2";
app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt
.GetID(), diagnostic1));
// send the status update
TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
taskAttemptStatus.id = attempt.GetID();
taskAttemptStatus.progress = 0.5f;
taskAttemptStatus.stateString = "RUNNING";
taskAttemptStatus.taskState = TaskAttemptState.Running;
taskAttemptStatus.phase = Phase.Map;
// send the status update
app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt
.GetID(), taskAttemptStatus));
//verify that all object are fully populated by invoking RPCs.
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
app.clientService.GetBindAddress(), conf);
GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest
>();
gcRequest.SetJobId(job.GetID());
NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest)
.GetCounters());
GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest
>();
gjrRequest.SetJobId(job.GetID());
JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport();
VerifyJobReport(jr);
GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance
<GetTaskAttemptCompletionEventsRequest>();
gtaceRequest.SetJobId(job.GetID());
gtaceRequest.SetFromEventId(0);
gtaceRequest.SetMaxEvents(10);
NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents
(gtaceRequest).GetCompletionEventList());
GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
>();
gdRequest.SetTaskAttemptId(attempt.GetID());
NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest
).GetDiagnosticsList());
GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
gtarRequest.SetTaskAttemptId(attempt.GetID());
TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport
();
VerifyTaskAttemptReport(tar);
GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest
>();
gtrRequest.SetTaskId(task.GetID());
NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest
).GetTaskReport());
GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Map);
NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count);
NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString
());
TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount
());
//send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
.Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
// For invalid jobid, throw IOException
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
)));
gtreportsRequest.SetTaskType(TaskType.Reduce);
try
{
proxy.GetTaskReports(gtreportsRequest);
NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
}
catch (IOException)
{
}
}
/// <exception cref="System.Exception"/>
public virtual void TestHistoryParsingForFailedAttempts()
{
Log.Info("STARTING testHistoryParsingForFailedAttempts");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithFailedAttempt(2, 1, true
, this.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser parser;
JobHistoryParser.JobInfo jobInfo;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
parser = new JobHistoryParser(@in);
jobInfo = parser.Parse();
}
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException
);
int noOffailedAttempts = 0;
IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks();
foreach (Task task in job.GetTasks().Values)
{
JobHistoryParser.TaskInfo taskInfo = allTasks[TypeConverter.FromYarn(task.GetID()
)];
foreach (TaskAttempt taskAttempt in task.GetAttempts().Values)
{
JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo.GetAllTaskAttempts()[
TypeConverter.FromYarn((taskAttempt.GetID()))];
// Verify rack-name for all task attempts
NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname
(), RackName);
if (taskAttemptInfo.GetTaskStatus().Equals("FAILED"))
{
noOffailedAttempts++;
}
}
}
NUnit.Framework.Assert.AreEqual("No of Failed tasks doesn't match.", 2, noOffailedAttempts
);
}
finally
{
Log.Info("FINISHED testHistoryParsingForFailedAttempts");
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDiagnosticsForKilledJob()
{
Log.Info("STARTING testDiagnosticsForKilledJob");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithJobKilled(2, 1, true, this
.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Killed);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser parser;
JobHistoryParser.JobInfo jobInfo;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
parser = new JobHistoryParser(@in);
jobInfo = parser.Parse();
}
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException
);
IList <string> originalDiagnostics = job.GetDiagnostics();
string historyError = jobInfo.GetErrorInfo();
NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics
!= null && !originalDiagnostics.IsEmpty());
NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError
);
foreach (string diagString in originalDiagnostics)
{
NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString));
}
NUnit.Framework.Assert.IsTrue("No killed message in diagnostics", historyError.Contains
(JobImpl.JobKilledDiag));
}
finally
{
Log.Info("FINISHED testDiagnosticsForKilledJob");
}
}
//Test reports of JobHistoryServer. History server should get log files from MRApp and read them
/// <exception cref="System.Exception"/>
public virtual void TestReports()
{
Configuration config = new Configuration();
config.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey
, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(config);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
FullName, true);
app.Submit(config);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
app.WaitForState(job, JobState.Succeeded);
historyServer = new JobHistoryServer();
historyServer.Init(config);
historyServer.Start();
// search JobHistory service
JobHistory jobHistory = null;
foreach (Org.Apache.Hadoop.Service.Service service in historyServer.GetServices())
{
if (service is JobHistory)
{
jobHistory = (JobHistory)service;
}
}
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = jobHistory.
GetAllJobs();
NUnit.Framework.Assert.AreEqual(1, jobs.Count);
NUnit.Framework.Assert.AreEqual("job_0_0000", jobs.Keys.GetEnumerator().Next().ToString
());
Task task = job.GetTasks().Values.GetEnumerator().Next();
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
HistoryClientService historyService = historyServer.GetClientService();
MRClientProtocol protocol = historyService.GetClientHandler();
GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
// test getTaskAttemptReport
TaskAttemptId taId = attempt.GetID();
taId.SetTaskId(task.GetID());
taId.GetTaskId().SetJobId(job.GetID());
gtarRequest.SetTaskAttemptId(taId);
GetTaskAttemptReportResponse response = protocol.GetTaskAttemptReport(gtarRequest
);
NUnit.Framework.Assert.AreEqual("container_0_0000_01_000000", response.GetTaskAttemptReport
().GetContainerId().ToString());
NUnit.Framework.Assert.IsTrue(response.GetTaskAttemptReport().GetDiagnosticInfo()
.IsEmpty());
// counters
NUnit.Framework.Assert.IsNotNull(response.GetTaskAttemptReport().GetCounters().GetCounter
(TaskCounter.PhysicalMemoryBytes));
NUnit.Framework.Assert.AreEqual(taId.ToString(), response.GetTaskAttemptReport().
GetTaskAttemptId().ToString());
// test getTaskReport
GetTaskReportRequest request = recordFactory.NewRecordInstance <GetTaskReportRequest
>();
TaskId taskId = task.GetID();
taskId.SetJobId(job.GetID());
request.SetTaskId(taskId);
GetTaskReportResponse reportResponse = protocol.GetTaskReport(request);
NUnit.Framework.Assert.AreEqual(string.Empty, reportResponse.GetTaskReport().GetDiagnosticsList
().GetEnumerator().Next());
// progress
NUnit.Framework.Assert.AreEqual(1.0f, reportResponse.GetTaskReport().GetProgress(
), 0.01);
// report has corrected taskId
NUnit.Framework.Assert.AreEqual(taskId.ToString(), reportResponse.GetTaskReport()
.GetTaskId().ToString());
// Task state should be SUCCEEDED
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, reportResponse.GetTaskReport
().GetTaskState());
// For invalid jobid, throw IOException
GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
>();
gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
)));
gtreportsRequest.SetTaskType(TaskType.Reduce);
try
{
protocol.GetTaskReports(gtreportsRequest);
NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
}
catch (IOException)
{
}
// Expected
// test getTaskAttemptCompletionEvents
GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory.NewRecordInstance
<GetTaskAttemptCompletionEventsRequest>();
taskAttemptRequest.SetJobId(job.GetID());
GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol
.GetTaskAttemptCompletionEvents(taskAttemptRequest);
NUnit.Framework.Assert.AreEqual(0, taskAttemptCompletionEventsResponse.GetCompletionEventCount
());
// test getDiagnostics
GetDiagnosticsRequest diagnosticRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
>();
diagnosticRequest.SetTaskAttemptId(taId);
GetDiagnosticsResponse diagnosticResponse = protocol.GetDiagnostics(diagnosticRequest
);
// it is strange : why one empty string ?
NUnit.Framework.Assert.AreEqual(1, diagnosticResponse.GetDiagnosticsCount());
NUnit.Framework.Assert.AreEqual(string.Empty, diagnosticResponse.GetDiagnostics(0
));
}
/// <exception cref="System.IO.IOException"/>
private void CheckAccess(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, JobACL jobOperation
)
{
UserGroupInformation callerUGI;
callerUGI = UserGroupInformation.GetCurrentUser();
if (!job.CheckAccess(callerUGI, jobOperation))
{
throw new IOException(new AccessControlException("User " + callerUGI.GetShortUserName
() + " cannot perform operation " + jobOperation.ToString() + " on " + job.GetID
()));
}
}
public virtual JobId GetID()
{
return(mockJob.GetID());
}
/// <exception cref="System.IO.IOException"/>
public MockCompletedJob(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
: base(new Configuration(), job.GetID(), null, true, job.GetUserName(), null, null
)
{
this.job = job;
}
public override JobId GetID()
{
return(job.GetID());
}
protected override void Render(HtmlBlock.Block html)
{
if (job == null)
{
html.P().("Sorry, no counters for nonexistent", $(AMParams.JobId, "job")).();
return;
}
if (!$(AMParams.TaskId).IsEmpty() && task == null)
{
html.P().("Sorry, no counters for nonexistent", $(AMParams.TaskId, "task")).();
return;
}
if (total == null || total.GetGroupNames() == null || total.CountCounters() == 0)
{
string type = $(AMParams.TaskId);
if (type == null || type.IsEmpty())
{
type = $(AMParams.JobId, "the job");
}
html.P().("Sorry it looks like ", type, " has no counters.").();
return;
}
string urlBase;
string urlId;
if (task != null)
{
urlBase = "singletaskcounter";
urlId = MRApps.ToString(task.GetID());
}
else
{
urlBase = "singlejobcounter";
urlId = MRApps.ToString(job.GetID());
}
int numGroups = 0;
Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet>
> > tbody = html.Div(JQueryUI.InfoWrap).Table("#counters").Thead().Tr().Th(".group.ui-state-default"
, "Counter Group").Th(".ui-state-default", "Counters").().().Tbody();
foreach (CounterGroup g in total)
{
CounterGroup mg = map == null ? null : map.GetGroup(g.GetName());
CounterGroup rg = reduce == null ? null : reduce.GetGroup(g.GetName());
++numGroups;
// This is mostly for demonstration :) Typically we'd introduced
// a CounterGroup block to reduce the verbosity. OTOH, this
// serves as an indicator of where we're in the tag hierarchy.
Hamlet.TR <Hamlet.THEAD <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE
<Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupHeadRow = tbody
.Tr().Th().$title(g.GetName()).$class("ui-state-default").(FixGroupDisplayName(g
.GetDisplayName())).().Td().$class(JQueryUI.CTable).Table(".dt-counters").$id(job
.GetID() + "." + g.GetName()).Thead().Tr().Th(".name", "Name");
if (map != null)
{
groupHeadRow.Th("Map").Th("Reduce");
}
// Ditto
Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV
<Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > group = groupHeadRow.Th(map
== null ? "Value" : "Total").().().Tbody();
foreach (Counter counter in g)
{
// Ditto
Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE
<Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupRow = group
.Tr();
if (task == null && mg == null && rg == null)
{
groupRow.Td().$title(counter.GetName()).(counter.GetDisplayName()).();
}
else
{
groupRow.Td().$title(counter.GetName()).A(Url(urlBase, urlId, g.GetName(), counter
.GetName()), counter.GetDisplayName()).();
}
if (map != null)
{
Counter mc = mg == null ? null : mg.FindCounter(counter.GetName());
Counter rc = rg == null ? null : rg.FindCounter(counter.GetName());
groupRow.Td(mc == null ? "0" : string.Format("%,d", mc.GetValue())).Td(rc == null
? "0" : string.Format("%,d", rc.GetValue()));
}
groupRow.Td(string.Format("%,d", counter.GetValue())).();
}
group.().().().();
}
tbody.().().();
}
public virtual void TestUpdatedNodes()
{
int runCount = 0;
MRApp app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName
, true, ++runCount);
Configuration conf = new Configuration();
// after half of the map completion, reduce will start
conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f);
// uberization forces full slowstart (1.0), so disable that
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask1 = it.Next();
Task mapTask2 = it.Next();
// all maps must be running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next();
TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
NodeId node1 = task1Attempt.GetNodeId();
NodeId node2 = task2Attempt.GetNodeId();
NUnit.Framework.Assert.AreEqual(node1, node2);
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
// all maps must be succeeded
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Succeeded);
TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for success", 2, events
.Length);
// send updated nodes info
AList <NodeReport> updatedNodes = new AList <NodeReport>();
NodeReport nr = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <NodeReport
>();
nr.SetNodeId(node1);
nr.SetNodeState(NodeState.Unhealthy);
updatedNodes.AddItem(nr);
app.GetContext().GetEventHandler().Handle(new JobUpdatedNodesEvent(job.GetID(), updatedNodes
));
app.WaitForState(task1Attempt, TaskAttemptState.Killed);
app.WaitForState(task2Attempt, TaskAttemptState.Killed);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for killed",
4, events.Length);
// all maps must be back to running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
IEnumerator <TaskAttempt> itr = mapTask1.GetAttempts().Values.GetEnumerator();
itr.Next();
task1Attempt = itr.Next();
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
// map1 must be succeeded. map2 must be running
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
5, events.Length);
// Crash the app again.
app.Stop();
// rerun
// in rerun the 1st map will be recovered from previous run
app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName,
false, ++runCount);
conf = new Configuration();
conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 4, job.GetTasks().Count
);
it = job.GetTasks().Values.GetEnumerator();
mapTask1 = it.Next();
mapTask2 = it.Next();
Task reduceTask1 = it.Next();
Task reduceTask2 = it.Next();
// map 1 will be recovered, no need to send done
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for killed & success of map1"
, 2, events.Length);
task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(mapTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
3, events.Length);
app.WaitForState(reduceTask1, TaskState.Running);
app.WaitForState(reduceTask2, TaskState.Running);
TaskAttempt task3Attempt = reduceTask1.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask1, TaskState.Succeeded);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaKill));
app.WaitForState(reduceTask1, TaskState.Succeeded);
TaskAttempt task4Attempt = reduceTask2.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task4Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for reduce success"
, 5, events.Length);
// job succeeds
app.WaitForState(job, JobState.Succeeded);
}
