/// <exception cref="System.Exception"/>
public virtual void TestResetTimer()
{
YarnConfiguration conf = new YarnConfiguration();
UserGroupInformation.SetConfiguration(conf);
conf.Set(YarnConfiguration.RecoveryEnabled, "true");
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, true);
conf.SetInt(YarnConfiguration.RmAmExpiryIntervalMs, 6000);
ControlledClock clock = new ControlledClock(new SystemClock());
clock.SetTime(0);
MemoryRMStateStore memStore = new _MemoryRMStateStore_46(clock);
memStore.Init(conf);
ApplicationAttemptId attemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId>();
Dispatcher dispatcher = Org.Mockito.Mockito.Mock <Dispatcher>();
bool[] expired = new bool[] { false };
AMLivelinessMonitor monitor = new _AMLivelinessMonitor_58(attemptId, expired, dispatcher
, clock);
monitor.Register(attemptId);
MockRM rm = new _MockRM_66(monitor, conf, memStore);
rm.Start();
// make sure that monitor has started
while (monitor.GetServiceState() != Service.STATE.Started)
{
Sharpen.Thread.Sleep(100);
}
// expired[0] would be set to true without resetTimer
NUnit.Framework.Assert.IsFalse(expired[0]);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestCreateDirsWithFileSystemNotBecomingAvailBeforeTimeout()
{
dfsCluster.GetFileSystem().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter
);
NUnit.Framework.Assert.IsTrue(dfsCluster.GetFileSystem().IsInSafeMode());
ControlledClock clock = new ControlledClock(new SystemClock());
clock.SetTime(1);
new _Thread_199(clock).Start();
TestCreateHistoryDirs(dfsCluster.GetConfiguration(0), clock);
}
/// <exception cref="System.Exception"/>
public virtual void TestRMAppAttemptFailuresValidityInterval()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 2.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 2);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// set window size to a larger number : 20s
// we will verify the app should be failed if
// two continuous attempts failed in 20s.
RMApp app = rm1.SubmitApp(200, 20000);
MockAM am = MockRM.LaunchAM(app, rm1, nm1);
// Fail current attempt normally
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Failed);
// launch the second attempt
rm1.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
NUnit.Framework.Assert.AreEqual(2, app.GetAppAttempts().Count);
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)app.GetCurrentAppAttempt()).MayBeLastAttempt
());
MockAM am_2 = MockRM.LaunchAndRegisterAM(app, rm1, nm1);
am_2.WaitForState(RMAppAttemptState.Running);
nm1.NodeHeartbeat(am_2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am_2.WaitForState(RMAppAttemptState.Failed);
// current app should be failed.
rm1.WaitForState(app.GetApplicationId(), RMAppState.Failed);
ControlledClock clock = new ControlledClock(new SystemClock());
// set window size to 6s
RMAppImpl app1 = (RMAppImpl)rm1.SubmitApp(200, 6000);
app1.SetSystemClock(clock);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// Fail attempt1 normally
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
am1.WaitForState(RMAppAttemptState.Failed);
// launch the second attempt
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
NUnit.Framework.Assert.AreEqual(2, app1.GetAppAttempts().Count);
RMAppAttempt attempt2 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt2).MayBeLastAttempt());
MockAM am2 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
am2.WaitForState(RMAppAttemptState.Running);
// wait for 6 seconds
clock.SetTime(Runtime.CurrentTimeMillis() + 6 * 1000);
// Fail attempt2 normally
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
// can launch the third attempt successfully
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
NUnit.Framework.Assert.AreEqual(3, app1.GetAppAttempts().Count);
RMAppAttempt attempt3 = app1.GetCurrentAppAttempt();
clock.Reset();
MockAM am3 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
am3.WaitForState(RMAppAttemptState.Running);
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
// re-register the NM
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
>();
status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
status.SetContainerId(attempt3.GetMasterContainer().GetId());
status.SetContainerState(ContainerState.Complete);
status.SetDiagnostics(string.Empty);
nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
rm2.WaitForState(attempt3.GetAppAttemptId(), RMAppAttemptState.Failed);
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// Lauch Attempt 4
MockAM am4 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 4, nm1);
// wait for 6 seconds
clock.SetTime(Runtime.CurrentTimeMillis() + 6 * 1000);
// Fail attempt4 normally
nm1.NodeHeartbeat(am4.GetApplicationAttemptId(), 1, ContainerState.Complete);
am4.WaitForState(RMAppAttemptState.Failed);
// can launch the 5th attempt successfully
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am5 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 5, nm1);
clock.Reset();
am5.WaitForState(RMAppAttemptState.Running);
// Fail attempt5 normally
nm1.NodeHeartbeat(am5.GetApplicationAttemptId(), 1, ContainerState.Complete);
am5.WaitForState(RMAppAttemptState.Failed);
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Failed);
rm1.Stop();
rm2.Stop();
}
public _MemoryRMStateStore_46(ControlledClock clock)
{
this.clock = clock;
}
public _Supplier_111(Task taskToBeSpeculated, ControlledClock clock)
{
this.taskToBeSpeculated = taskToBeSpeculated;
this.clock = clock;
}
public virtual void TestSpeculateSuccessfulWithoutUpdateEvents()
{
Clock actualClock = new SystemClock();
ControlledClock clock = new ControlledClock(actualClock);
clock.SetTime(Runtime.CurrentTimeMillis());
MRApp app = new MRApp(NumMappers, NumReducers, false, "test", true, clock);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration(),
true, true);
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", NumMappers + NumReducers
, tasks.Count);
IEnumerator <Task> taskIter = tasks.Values.GetEnumerator();
while (taskIter.HasNext())
{
app.WaitForState(taskIter.Next(), TaskState.Running);
}
// Process the update events
clock.SetTime(Runtime.CurrentTimeMillis() + 2000);
EventHandler appEventHandler = app.GetContext().GetEventHandler();
foreach (KeyValuePair <TaskId, Task> mapTask in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask.Value.GetAttempts
())
{
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.8, TaskAttemptState.Running);
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
Random generator = new Random();
object[] taskValues = Sharpen.Collections.ToArray(tasks.Values);
Task taskToBeSpeculated = (Task)taskValues[generator.Next(taskValues.Length)];
// Other than one random task, finish every other task.
foreach (KeyValuePair <TaskId, Task> mapTask_1 in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask_1.Value.
GetAttempts())
{
if (mapTask_1.Key != taskToBeSpeculated.GetID())
{
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaDone));
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaContainerCleaned));
app.WaitForState(taskAttempt.Value, TaskAttemptState.Succeeded);
}
}
}
GenericTestUtils.WaitFor(new _Supplier_111(taskToBeSpeculated, clock), 1000, 60000
);
// finish 1st TA, 2nd will be killed
TaskAttempt[] ta = MakeFirstAttemptWin(appEventHandler, taskToBeSpeculated);
VerifySpeculationMessage(app, ta);
app.WaitForState(Service.STATE.Stopped);
}
public _Supplier_205(Task speculatedTaskConst, ControlledClock clock)
{
this.speculatedTaskConst = speculatedTaskConst;
this.clock = clock;
}
public virtual void TestSepculateSuccessfulWithUpdateEvents()
{
Clock actualClock = new SystemClock();
ControlledClock clock = new ControlledClock(actualClock);
clock.SetTime(Runtime.CurrentTimeMillis());
MRApp app = new MRApp(NumMappers, NumReducers, false, "test", true, clock);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration(),
true, true);
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", NumMappers + NumReducers
, tasks.Count);
IEnumerator <Task> taskIter = tasks.Values.GetEnumerator();
while (taskIter.HasNext())
{
app.WaitForState(taskIter.Next(), TaskState.Running);
}
// Process the update events
clock.SetTime(Runtime.CurrentTimeMillis() + 1000);
EventHandler appEventHandler = app.GetContext().GetEventHandler();
foreach (KeyValuePair <TaskId, Task> mapTask in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask.Value.GetAttempts
())
{
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.5, TaskAttemptState.Running);
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
Task speculatedTask = null;
int numTasksToFinish = NumMappers + NumReducers - 1;
clock.SetTime(Runtime.CurrentTimeMillis() + 1000);
foreach (KeyValuePair <TaskId, Task> task in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in task.Value.GetAttempts
())
{
if (numTasksToFinish > 0)
{
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaDone));
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaContainerCleaned));
numTasksToFinish--;
app.WaitForState(taskAttempt.Value, TaskAttemptState.Succeeded);
}
else
{
// The last task is chosen for speculation
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.75, TaskAttemptState.Running);
speculatedTask = task.Value;
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
}
clock.SetTime(Runtime.CurrentTimeMillis() + 15000);
foreach (KeyValuePair <TaskId, Task> task_1 in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in task_1.Value.GetAttempts
())
{
if (taskAttempt.Value.GetState() != TaskAttemptState.Succeeded)
{
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.75, TaskAttemptState.Running);
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
}
Task speculatedTaskConst = speculatedTask;
GenericTestUtils.WaitFor(new _Supplier_205(speculatedTaskConst, clock), 1000, 60000
);
TaskAttempt[] ta = MakeFirstAttemptWin(appEventHandler, speculatedTask);
VerifySpeculationMessage(app, ta);
app.WaitForState(Service.STATE.Stopped);
}
public _Thread_199(ControlledClock clock)
{
this.clock = clock;
}