private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetJob()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(Runtime.CurrentTimeMillis()
, 4);
jobId.SetAppId(appId);
jobId.SetId(1);
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
JobReport report = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100010L);
Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100015L);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(report);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("JobName");
Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("UserName");
Org.Mockito.Mockito.When(job.GetQueueName()).ThenReturn("QueueName");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(3);
Org.Mockito.Mockito.When(job.GetCompletedMaps()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
return(job);
}
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
{
this.id = MRApps.ToString(job.GetID());
JobReport report = job.GetReport();
this.mapsTotal = job.GetTotalMaps();
this.mapsCompleted = job.GetCompletedMaps();
this.reducesTotal = job.GetTotalReduces();
this.reducesCompleted = job.GetCompletedReduces();
this.submitTime = report.GetSubmitTime();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.name = job.GetName().ToString();
this.queue = job.GetQueueName();
this.user = job.GetUserName();
this.state = job.GetState().ToString();
this.acls = new AList <ConfEntryInfo>();
if (job is CompletedJob)
{
avgMapTime = 0l;
avgReduceTime = 0l;
avgShuffleTime = 0l;
avgMergeTime = 0l;
failedReduceAttempts = 0;
killedReduceAttempts = 0;
successfulReduceAttempts = 0;
failedMapAttempts = 0;
killedMapAttempts = 0;
successfulMapAttempts = 0;
CountTasksAndAttempts(job);
this.uberized = job.IsUber();
this.diagnostics = string.Empty;
IList <string> diagnostics = job.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
this.diagnostics = b.ToString();
}
IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
if (allacls != null)
{
foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
{
this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
()));
}
}
}
}
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess)
{
// ok for any user to see
// these should only be seen if acls allow
this.id = MRApps.ToString(job.GetID());
JobReport report = job.GetReport();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime);
if (this.elapsedTime == -1)
{
this.elapsedTime = 0;
}
this.name = job.GetName().ToString();
this.user = job.GetUserName();
this.state = job.GetState();
this.mapsTotal = job.GetTotalMaps();
this.mapsCompleted = job.GetCompletedMaps();
this.mapProgress = report.GetMapProgress() * 100;
this.mapProgressPercent = StringHelper.Percent(report.GetMapProgress());
this.reducesTotal = job.GetTotalReduces();
this.reducesCompleted = job.GetCompletedReduces();
this.reduceProgress = report.GetReduceProgress() * 100;
this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress());
this.acls = new AList <ConfEntryInfo>();
if (hasAccess)
{
this.diagnostics = string.Empty;
CountTasksAndAttempts(job);
this.uberized = job.IsUber();
IList <string> diagnostics = job.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
this.diagnostics = b.ToString();
}
IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
if (allacls != null)
{
foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
{
this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
()));
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void WaitForState(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job,
JobState finalState)
{
int timeoutSecs = 0;
JobReport report = job.GetReport();
while (!finalState.Equals(report.GetJobState()) && timeoutSecs++ < 20)
{
System.Console.Out.WriteLine("Job State is : " + report.GetJobState() + " Waiting for state : "
+ finalState + " map progress : " + report.GetMapProgress() + " reduce progress : "
+ report.GetReduceProgress());
report = job.GetReport();
Sharpen.Thread.Sleep(500);
}
System.Console.Out.WriteLine("Job State is : " + report.GetJobState());
NUnit.Framework.Assert.AreEqual("Job state is not correct (timedout)", finalState
, job.GetState());
}
public virtual void TestAverageReduceTime()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob
>();
Task task1 = Org.Mockito.Mockito.Mock <Task>();
Task task2 = Org.Mockito.Mockito.Mock <Task>();
JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1);
TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce);
TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce);
TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1);
TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2);
TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>();
JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L);
Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L);
Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L);
Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L);
Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L);
Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L);
Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L);
Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1
, taskAttempt1));
Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2
, taskAttempt2));
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1,
task1, taskId2, task2));
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
JobInfo jobInfo = new JobInfo(job);
NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime());
}
public virtual void TestEventsFlushOnStop()
{
Configuration conf = new Configuration();
MRApp app = new TestJobHistoryEvents.MRAppWithSpecialHistoryHandler(1, 0, true, this
.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps
*/
HistoryContext context = new JobHistory();
((JobHistory)context).Init(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId);
NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 1, parsedJob.GetCompletedMaps
());
IDictionary <TaskId, Task> tasks = parsedJob.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, tasks.Count);
VerifyTask(tasks.Values.GetEnumerator().Next());
IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map);
NUnit.Framework.Assert.AreEqual("No of maps not correct", 1, maps.Count);
NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob
.GetState());
}
public virtual void TestHistoryEvents()
{
Configuration conf = new Configuration();
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(2, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
//make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps
*/
HistoryContext context = new JobHistory();
// test start and stop states
((JobHistory)context).Init(conf);
((JobHistory)context).Start();
NUnit.Framework.Assert.IsTrue(context.GetStartTime() > 0);
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Started);
// get job before stopping JobHistory
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId);
// stop JobHistory
((JobHistory)context).Stop();
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Stopped);
NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 2, parsedJob.GetCompletedMaps
());
NUnit.Framework.Assert.AreEqual(Runtime.GetProperty("user.name"), parsedJob.GetUserName
());
IDictionary <TaskId, Task> tasks = parsedJob.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 3, tasks.Count);
foreach (Task task in tasks.Values)
{
VerifyTask(task);
}
IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map);
NUnit.Framework.Assert.AreEqual("No of maps not correct", 2, maps.Count);
IDictionary <TaskId, Task> reduces = parsedJob.GetTasks(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual("No of reduces not correct", 1, reduces.Count);
NUnit.Framework.Assert.AreEqual("CompletedReduce not correct", 1, parsedJob.GetCompletedReduces
());
NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob
.GetState());
}
public override JobState GetState()
{
return(job.GetState());
}
public virtual JobState GetState()
{
return(mockJob.GetState());
}
