public virtual void TestNotificationOnLastRetryUnregistrationFailure()
{
HttpServer2 server = StartHttpServer();
MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator
(this, 2, 2, false, this.GetType().FullName, true, 2, false));
// Currently, we will have isLastRetry always equals to false at beginning
// of MRAppMaster, except staging area exists or commit already started at
// the beginning.
// Now manually set isLastRetry to true and this should reset to false when
// unregister failed.
app.isLastAMRetry = true;
Org.Mockito.Mockito.DoNothing().When(app).Sysexit();
JobConf conf = new JobConf();
conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl
+ "jobend?jobid=$jobId&status=$jobStatus");
JobImpl job = (JobImpl)app.Submit(conf);
app.WaitForState(job, JobState.Running);
app.GetContext().GetEventHandler().Handle(new JobEvent(app.GetJobId(), JobEventType
.JobAmReboot));
app.WaitForInternalState(job, JobStateInternal.Reboot);
// Now shutdown. User should see FAILED state.
// Unregistration fails: isLastAMRetry is recalculated, this is
///reboot will stop service internally, we don't need to shutdown twice
app.WaitForServiceToStop(10000);
NUnit.Framework.Assert.IsFalse(app.IsLastAMRetry());
// Since it's not last retry, JobEndServlet didn't called
NUnit.Framework.Assert.AreEqual(0, TestJobEndNotifier.JobEndServlet.calledTimes);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.requestUri);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.foundJobState);
server.Stop();
}
public virtual void TestAbsentNotificationOnNotLastRetryUnregistrationFailure()
{
HttpServer2 server = StartHttpServer();
MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator
(this, 2, 2, false, this.GetType().FullName, true, 1, false));
Org.Mockito.Mockito.DoNothing().When(app).Sysexit();
JobConf conf = new JobConf();
conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl
+ "jobend?jobid=$jobId&status=$jobStatus");
JobImpl job = (JobImpl)app.Submit(conf);
app.WaitForState(job, JobState.Running);
app.GetContext().GetEventHandler().Handle(new JobEvent(app.GetJobId(), JobEventType
.JobAmReboot));
app.WaitForInternalState(job, JobStateInternal.Reboot);
// Now shutdown.
// Unregistration fails: isLastAMRetry is recalculated, this is not
app.ShutDownJob();
// Not the last AM attempt. So user should that the job is still running.
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.IsFalse(app.IsLastAMRetry());
NUnit.Framework.Assert.AreEqual(0, TestJobEndNotifier.JobEndServlet.calledTimes);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.requestUri);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.foundJobState);
server.Stop();
}
public virtual void TestNotificationOnLastRetryNormalShutdown()
{
HttpServer2 server = StartHttpServer();
// Act like it is the second attempt. Default max attempts is 2
MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator
(this, 2, 2, true, this.GetType().FullName, true, 2, true));
Org.Mockito.Mockito.DoNothing().When(app).Sysexit();
JobConf conf = new JobConf();
conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl
+ "jobend?jobid=$jobId&status=$jobStatus");
JobImpl job = (JobImpl)app.Submit(conf);
app.WaitForInternalState(job, JobStateInternal.Succeeded);
// Unregistration succeeds: successfullyUnregistered is set
app.ShutDownJob();
NUnit.Framework.Assert.IsTrue(app.IsLastAMRetry());
NUnit.Framework.Assert.AreEqual(1, TestJobEndNotifier.JobEndServlet.calledTimes);
NUnit.Framework.Assert.AreEqual("jobid=" + job.GetID() + "&status=SUCCEEDED", TestJobEndNotifier.JobEndServlet
.requestUri.GetQuery());
NUnit.Framework.Assert.AreEqual(JobState.Succeeded.ToString(), TestJobEndNotifier.JobEndServlet
.foundJobState);
server.Stop();
}
/// <summary>
/// Create a NULL job to conform to Null Object pattern
/// </summary>
/// <param name="customer"></param>
/// <returns></returns>
public static JobImpl Empty(this JobImpl job)
{
return(new JobImpl()
{
JobId = NULL_ID, CustomerId = NULL_ID, Description = "", State = 1, Customer = new CustomerImpl().Empty(), Components = new List <ComponentImpl>()
});
}
protected internal override Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job CreateJob(
Configuration conf, JobStateInternal forcedState, string diagnostic)
{
this.spiedJob = Org.Mockito.Mockito.Spy((JobImpl)base.CreateJob(conf, forcedState
, diagnostic));
((AppContext)this.GetContext()).GetAllJobs()[this.spiedJob.GetID()] = this.spiedJob;
return(this.spiedJob);
}
/// <summary>
/// Create a JobImpl from a regular Job class
/// </summary>
/// <param name="job"></param>
/// <param name="customer"></param>
/// <returns></returns>
public static JobImpl ToJobImpl(this Job job, CustomerImpl customer)
{
var jobImpl = new JobImpl()
{
JobId = job.Id, Customer = customer, CustomerId = customer.CustomerId, Description = job.Description.TrimEnd(), State = job.State
};
return(jobImpl);
}
protected internal override JobSplit.TaskSplitMetaInfo[] CreateSplits(JobImpl job
, JobId jobId)
{
JobSplit.TaskSplitMetaInfo[] splits = new JobSplit.TaskSplitMetaInfo[maps];
for (int i = 0; i < maps; i++)
{
splits[i] = new JobSplit.TaskSplitMetaInfo();
}
return(splits);
}
private void JobStatusChanged(JobImpl job, Job.StatusEnum status)
{
// Notify that Status and all dependent properties have changed
PropertyChanged.Notify(() => Status);
PropertyChanged.Notify(() => AllowReRun);
PropertyChanged.Notify(() => AllowAbort);
PropertyChanged.Notify(() => ProgressMessage);
PropertyChanged.Notify(() => ProgressCurrent);
PropertyChanged.Notify(() => ProgressTotal);
PropertyChanged.Notify(() => ProgressIsIndeterminate);
}
public virtual void TestJobSuccess()
{
MRApp app = new MRApp(2, 2, true, this.GetType().FullName, true, false);
JobImpl job = (JobImpl)app.Submit(new Configuration());
app.WaitForInternalState(job, JobStateInternal.Succeeded);
// AM is not unregistered
NUnit.Framework.Assert.AreEqual(JobState.Running, job.GetState());
// imitate that AM is unregistered
app.successfullyUnregistered.Set(true);
app.WaitForState(job, JobState.Succeeded);
}
/// <summary>
/// Create a Job from a JobImpl
/// </summary>
/// <param name="jobImpl"></param>
/// <param name="customer"></param>
/// <returns></returns>
public static Job ToJob(this JobImpl jobImpl, List <Component> components, Customer customer)
{
customer.Address = TrimValue(customer.Address);
customer.Name = TrimValue(customer.Name);
var job = new Job()
{
Id = jobImpl.JobId, Description = jobImpl.Description.TrimEnd(), Customer = customer, Components = components
};
return(job);
}
public static Counters GetCounters(ICollection <Task> tasks)
{
IList <Task> completedTasks = new AList <Task>();
foreach (Task task in tasks)
{
if (task.GetCounters() != null)
{
completedTasks.AddItem(task);
}
}
Counters counters = new Counters();
return(JobImpl.IncrTaskCounters(counters, completedTasks));
}
protected internal override Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job CreateJob(
Configuration conf, JobStateInternal forcedState, string diagnostic)
{
JobImpl jobImpl = Org.Mockito.Mockito.Mock <JobImpl>();
Org.Mockito.Mockito.When(jobImpl.GetInternalState()).ThenReturn(this.jobStateInternal
);
Org.Mockito.Mockito.When(jobImpl.GetAllCounters()).ThenReturn(new Counters());
JobID jobID = JobID.ForName("job_1234567890000_0001");
JobId jobId = TypeConverter.ToYarn(jobID);
Org.Mockito.Mockito.When(jobImpl.GetID()).ThenReturn(jobId);
((AppContext)this.GetContext()).GetAllJobs()[jobImpl.GetID()] = jobImpl;
return(jobImpl);
}
public virtual void TestCountersOnJobFinish()
{
TestMRApp.MRAppWithSpiedJob app = new TestMRApp.MRAppWithSpiedJob(this, 1, 1, true
, this.GetType().FullName, true);
JobImpl job = (JobImpl)app.Submit(new Configuration());
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
System.Console.Out.WriteLine(job.GetAllCounters());
// Just call getCounters
job.GetAllCounters();
job.GetAllCounters();
// Should be called only once
Org.Mockito.Mockito.Verify(job, Org.Mockito.Mockito.Times(1)).ConstructFinalFullcounters
();
}
/// <exception cref="System.Exception"/>
public virtual void WaitForInternalState(JobImpl job, JobStateInternal finalState
)
{
int timeoutSecs = 0;
JobStateInternal iState = job.GetInternalState();
while (!finalState.Equals(iState) && timeoutSecs++ < 20)
{
System.Console.Out.WriteLine("Job Internal State is : " + iState + " Waiting for Internal state : "
+ finalState);
Sharpen.Thread.Sleep(500);
iState = job.GetInternalState();
}
System.Console.Out.WriteLine("Task Internal State is : " + iState);
NUnit.Framework.Assert.AreEqual("Task Internal state is not correct (timedout)",
finalState, iState);
}
/// <exception cref="System.Exception"/>
public virtual void TestStagingCleanupOrder()
{
TestStagingCleanup.MRAppTestCleanup app = new TestStagingCleanup.MRAppTestCleanup
(this, 1, 1, true, this.GetType().FullName, true);
JobImpl job = (JobImpl)app.Submit(new Configuration());
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
int waitTime = 20 * 1000;
while (waitTime > 0 && app.numStops < 2)
{
Sharpen.Thread.Sleep(100);
waitTime -= 100;
}
// assert ContainerAllocatorStopped and then tagingDirCleanedup
NUnit.Framework.Assert.AreEqual(1, app.ContainerAllocatorStopped);
NUnit.Framework.Assert.AreEqual(2, app.stagingDirCleanedup);
}
/// <exception cref="System.Exception"/>
public virtual void TestComponentStopOrder()
{
TestMRAppComponentDependencies.TestMRApp app = new TestMRAppComponentDependencies.TestMRApp
(this, 1, 1, true, this.GetType().FullName, true);
JobImpl job = (JobImpl)app.Submit(new Configuration());
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
int waitTime = 20 * 1000;
while (waitTime > 0 && app.numStops < 2)
{
Sharpen.Thread.Sleep(100);
waitTime -= 100;
}
// assert JobHistoryEventHandlerStopped and then clientServiceStopped
NUnit.Framework.Assert.AreEqual(1, app.JobHistoryEventHandlerStopped);
NUnit.Framework.Assert.AreEqual(2, app.clientServiceStopped);
}
private void OnJobProgressChanged(JobImpl job, string progressMessage, int progressCurrent, int progressTotal)
{
if (!string.IsNullOrEmpty(ProgressMessageHistory))
{
ProgressMessageHistory += "\n";
}
if (!ProgressIsIndeterminate)
{
ProgressMessageHistory += $"({ProgressCurrent}/{ProgressTotal}) {ProgressMessage}";
}
else
{
ProgressMessageHistory += $"({ProgressCurrent}) {ProgressMessage}";
}
PropertyChanged.Notify(() => ProgressMessage);
PropertyChanged.Notify(() => ProgressCurrent);
PropertyChanged.Notify(() => ProgressTotal);
PropertyChanged.Notify(() => ProgressIsIndeterminate);
PropertyChanged.Notify(() => ProgressMessageHistory);
}
/// <exception cref="System.IO.IOException"/>
protected internal override void Setup(JobImpl job)
{
base.Setup(job);
job.conf.SetInt(MRJobConfig.NumReduces, reduces);
job.remoteJobConfFile = new Path("test");
}
public void JobStatusChanged(Job job_, Job.StatusEnum status)
{
try
{
JobImpl job = (JobImpl)job_;
Trace.TraceInformation("Form JobStatusChanged was called for: {0}, which changed status to {1}", job.Title, status.ToString());
autoresize_counter--;
lvJobQueue.SafeInvoke(delegate
{
Color color = Color.White;
switch (status)
{
case Job.StatusEnum.Ready:
color = Color.PeachPuff;
break;
case Job.StatusEnum.QueuedLocal:
case Job.StatusEnum.QueuedOnServer:
case Job.StatusEnum.StartedOnServer:
color = Color.LightYellow;
break;
case Job.StatusEnum.UploadPackage:
case Job.StatusEnum.DownloadResults:
color = Color.PaleTurquoise;
break;
case Job.StatusEnum.PostedToServer:
color = Color.Khaki;
break;
case Job.StatusEnum.RunningOnServer:
case Job.StatusEnum.RunningLocal:
color = Color.LightSkyBlue;
break;
case Job.StatusEnum.Succeeded:
color = Color.LightGreen;
break;
case Job.StatusEnum.Failed:
case Job.StatusEnum.FailedAbortOnServer:
case Job.StatusEnum.FailedAbortByUser:
case Job.StatusEnum.FailedExecution:
case Job.StatusEnum.FailedToUploadServer:
case Job.StatusEnum.FailedToDownload:
case Job.StatusEnum.AbortOnServerRequested:
color = Color.Tomato;
break;
default:
color = Color.White;
break;
}
var item = lvJobQueue.Items.
Cast <ListViewItem>().
FirstOrDefault(x => x.SubItems[(int)Headers.Id].Text == job.Id.ToString());
item.BackColor = color;
item.SubItems[(int)Headers.Status].Text = status.ToString();
AutoResizeColumns();
});
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
Trace.TraceError(ex.ToString());
}
}
/// <summary>
/// Update the given job
/// </summary>
/// <param name="id"></param>
public void UpdateJob(JobImpl job)
{
}
public override void CreateJob(JobImpl job)
{
return;
}
/// <summary>
/// Determine if this is a null pattern object
/// </summary>
/// <param name="customer"></param>
/// <returns></returns>
public static bool IsEmpty(this JobImpl job)
{
return(job.JobId == NULL_ID);
}
/// <summary>
/// Save a job by class
/// </summary>
/// <param name="job"></param>
public void CreateJob(JobImpl job)
{
db.JobImpls.Add(job);
db.SaveChanges();
}
protected internal virtual void DoUnregistration()
{
FinalApplicationStatus finishState = FinalApplicationStatus.Undefined;
JobImpl jobImpl = (JobImpl)job;
if (jobImpl.GetInternalState() == JobStateInternal.Succeeded)
{
finishState = FinalApplicationStatus.Succeeded;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState
() == JobStateInternal.Running && isSignalled))
{
finishState = FinalApplicationStatus.Killed;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState
() == JobStateInternal.Error)
{
finishState = FinalApplicationStatus.Failed;
}
}
}
StringBuilder sb = new StringBuilder();
foreach (string s in job.GetDiagnostics())
{
sb.Append(s).Append("\n");
}
Log.Info("Setting job diagnostics to " + sb.ToString());
string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(),
context.GetApplicationID());
Log.Info("History url is " + historyUrl);
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
(finishState, sb.ToString(), historyUrl);
try
{
while (true)
{
FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request
);
if (response.GetIsUnregistered())
{
// When excepting ClientService, other services are already stopped,
// it is safe to let clients know the final states. ClientService
// should wait for some time so clients have enough time to know the
// final states.
MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context;
raContext.MarkSuccessfulUnregistration();
break;
}
Log.Info("Waiting for application to be successfully unregistered.");
Sharpen.Thread.Sleep(rmPollInterval);
}
}
catch (ApplicationMasterNotRegisteredException)
{
// RM might have restarted or failed over and so lost the fact that AM had
// registered before.
Register();
DoUnregistration();
}
}
public void TestJobImplEmpty()
{
var job = new JobImpl().Empty();
Assert.IsTrue(job.IsEmpty());
}
public virtual void UpdateJob(JobImpl job)
{
throw new NotImplementedException();
}
private void JobStatusChanged(JobImpl job, Job.StatusEnum status)
{
InvokeOnMainThread(InvokeTrackedJobsChanged);
}
/// <summary>
/// Add a job by class
/// </summary>
/// <param name="job"></param>
public void CreateJob(JobImpl job)
{
}