public virtual void TestNormalContainerAllocationWhenDNSUnavailable()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
// acquire the container.
SecurityUtilTestHelper.SetTokenServiceUseIp(true);
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
// not able to fetch the container;
NUnit.Framework.Assert.AreEqual(0, containers.Count);
SecurityUtilTestHelper.SetTokenServiceUseIp(false);
containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>())
.GetAllocatedContainers();
// should be able to fetch the container;
NUnit.Framework.Assert.AreEqual(1, containers.Count);
}
/// <exception cref="System.Exception"/>
private LogAggregationContext GetLogAggregationContextFromContainerToken(MockRM rm1
, MockNM nm1, LogAggregationContext logAggregationContext)
{
RMApp app2 = rm1.SubmitApp(200, logAggregationContext);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(true);
// request a container.
am2.Allocate("127.0.0.1", 512, 1, new AList <ContainerId>());
ContainerId containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(
), 2);
rm1.WaitForState(nm1, containerId, RMContainerState.Allocated);
// acquire the container.
IList <Container> containers = am2.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(containerId, containers[0].GetId());
// container token is generated.
NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
ContainerTokenIdentifier token = BuilderUtils.NewContainerTokenIdentifier(containers
[0].GetContainerToken());
return(token.GetLogAggregationContext());
}
public static TaskAttemptReport NewTaskAttemptReport(TaskAttemptId id)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(id.GetTaskId
().GetJobId().GetAppId(), 0);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 0);
TaskAttemptReport report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <TaskAttemptReport
>();
report.SetTaskAttemptId(id);
report.SetStartTime(Runtime.CurrentTimeMillis() - (int)(Math.Random() * Dt));
report.SetFinishTime(Runtime.CurrentTimeMillis() + (int)(Math.Random() * Dt) + 1);
if (id.GetTaskId().GetTaskType() == TaskType.Reduce)
{
report.SetShuffleFinishTime((report.GetFinishTime() + report.GetStartTime()) / 2);
report.SetSortFinishTime((report.GetFinishTime() + report.GetShuffleFinishTime())
/ 2);
}
report.SetPhase(Phases.Next());
report.SetTaskAttemptState(TaskAttemptStates.Next());
report.SetProgress((float)Math.Random());
report.SetCounters(TypeConverter.ToYarn(NewCounters()));
report.SetContainerId(containerId);
report.SetDiagnosticInfo(Diags.Next());
report.SetStateString("Moving average " + Math.Random());
return(report);
}
public virtual void TestContainerTokenGeneratedOnPullRequest()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000);
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
RMContainer container = rm1.GetResourceScheduler().GetRMContainer(containerId2);
// no container token is generated.
NUnit.Framework.Assert.AreEqual(containerId2, container.GetContainerId());
NUnit.Framework.Assert.IsNull(container.GetContainer().GetContainerToken());
// acquire the container.
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(containerId2, containers[0].GetId());
// container token is generated.
NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
rm1.Stop();
}
public virtual void TestHandleContainerStatusInvalidCompletions()
{
rm = new MockRM(new YarnConfiguration());
rm.Start();
EventHandler handler = Org.Mockito.Mockito.Spy(rm.GetRMContext().GetDispatcher().
GetEventHandler());
// Case 1: Unmanaged AM
RMApp app = rm.SubmitApp(1024, true);
// Case 1.1: AppAttemptId is null
NMContainerStatus report = NMContainerStatus.NewInstance(ContainerId.NewContainerId
(ApplicationAttemptId.NewInstance(app.GetApplicationId(), 2), 1), ContainerState
.Complete, Resource.NewInstance(1024, 1), "Dummy Completed", 0, Priority.NewInstance
(10), 1234);
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 1.2: Master container is null
RMAppAttemptImpl currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();
currentAttempt.SetMasterContainer(null);
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
, 0, Priority.NewInstance(10), 1234);
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 2: Managed AM
app = rm.SubmitApp(1024);
// Case 2.1: AppAttemptId is null
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(ApplicationAttemptId
.NewInstance(app.GetApplicationId(), 2), 1), ContainerState.Complete, Resource.NewInstance
(1024, 1), "Dummy Completed", 0, Priority.NewInstance(10), 1234);
try
{
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
}
catch (Exception)
{
}
// expected - ignore
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 2.2: Master container is null
currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();
currentAttempt.SetMasterContainer(null);
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
, 0, Priority.NewInstance(10), 1234);
try
{
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
}
catch (Exception)
{
}
// expected - ignore
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
}
public virtual void TestContainerMetricsFinished()
{
MetricsSystemImpl system = new MetricsSystemImpl();
system.Init("test");
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ApplicationId appId = ApplicationId.NewInstance(1234, 3);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 4);
ContainerId containerId1 = ContainerId.NewContainerId(appAttemptId, 1);
ContainerMetrics metrics1 = ContainerMetrics.ForContainer(system, containerId1, 1
, 0);
ContainerId containerId2 = ContainerId.NewContainerId(appAttemptId, 2);
ContainerMetrics metrics2 = ContainerMetrics.ForContainer(system, containerId2, 1
, 0);
ContainerId containerId3 = ContainerId.NewContainerId(appAttemptId, 3);
ContainerMetrics metrics3 = ContainerMetrics.ForContainer(system, containerId3, 1
, 0);
metrics1.Finished();
metrics2.Finished();
system.SampleMetrics();
system.SampleMetrics();
Sharpen.Thread.Sleep(100);
system.Stop();
// verify metrics1 is unregistered
NUnit.Framework.Assert.IsTrue(metrics1 != ContainerMetrics.ForContainer(system, containerId1
, 1, 0));
// verify metrics2 is unregistered
NUnit.Framework.Assert.IsTrue(metrics2 != ContainerMetrics.ForContainer(system, containerId2
, 1, 0));
// verify metrics3 is still registered
NUnit.Framework.Assert.IsTrue(metrics3 == ContainerMetrics.ForContainer(system, containerId3
, 1, 0));
system.Shutdown();
}
/// <exception cref="System.IO.IOException"/>
public AllocateResponse Allocate(AllocateRequest request)
{
AllocateResponse response = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateResponse
>();
IList <ResourceRequest> askList = request.GetAskList();
IList <Container> containers = new AList <Container>();
foreach (ResourceRequest req in askList)
{
if (!ResourceRequest.IsAnyLocation(req.GetResourceName()))
{
continue;
}
int numContainers = req.GetNumContainers();
for (int i = 0; i < numContainers; i++)
{
ContainerId containerId = ContainerId.NewContainerId(this._enclosing.GetContext()
.GetApplicationAttemptId(), request.GetResponseId() + i);
containers.AddItem(Container.NewInstance(containerId, NodeId.NewInstance("host" +
containerId.GetContainerId(), 2345), "host" + containerId.GetContainerId() + ":5678"
, req.GetCapability(), req.GetPriority(), null));
}
}
response.SetAllocatedContainers(containers);
response.SetResponseId(request.GetResponseId() + 1);
response.SetNumClusterNodes(350);
return(response);
}
/// <exception cref="System.Exception"/>
public virtual void TestPostExecuteAfterReacquisition()
{
// make up some bogus container ID
ApplicationId appId = ApplicationId.NewInstance(12345, 67890);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 54321);
ContainerId cid = ContainerId.NewContainerId(attemptId, 9876);
Configuration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.NmLinuxContainerResourcesHandler, typeof(TestLinuxContainerExecutor.TestResourceHandler
), typeof(LCEResourcesHandler));
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.SetConf(conf);
try
{
lce.Init();
}
catch (IOException)
{
}
// expected if LCE isn't setup right, but not necessary for this test
lce.ReacquireContainer("foouser", cid);
NUnit.Framework.Assert.IsTrue("postExec not called after reacquisition", TestLinuxContainerExecutor.TestResourceHandler
.postExecContainers.Contains(cid));
}
public virtual void TestRegisterNodeManagerRequest()
{
ApplicationId appId = ApplicationId.NewInstance(123456789, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(attemptId, 1);
NMContainerStatus containerReport = NMContainerStatus.NewInstance(containerId, ContainerState
.Running, Resource.NewInstance(1024, 1), "diagnostics", 0, Priority.NewInstance(
10), 1234);
IList <NMContainerStatus> reports = Arrays.AsList(containerReport);
RegisterNodeManagerRequest request = RegisterNodeManagerRequest.NewInstance(NodeId
.NewInstance("1.1.1.1", 1000), 8080, Resource.NewInstance(1024, 1), "NM-version-id"
, reports, Arrays.AsList(appId));
RegisterNodeManagerRequest requestProto = new RegisterNodeManagerRequestPBImpl(((
RegisterNodeManagerRequestPBImpl)request).GetProto());
NUnit.Framework.Assert.AreEqual(containerReport, requestProto.GetNMContainerStatuses
()[0]);
NUnit.Framework.Assert.AreEqual(8080, requestProto.GetHttpPort());
NUnit.Framework.Assert.AreEqual("NM-version-id", requestProto.GetNMVersion());
NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("1.1.1.1", 1000), requestProto
.GetNodeId());
NUnit.Framework.Assert.AreEqual(Resource.NewInstance(1024, 1), requestProto.GetResource
());
NUnit.Framework.Assert.AreEqual(1, requestProto.GetRunningApplications().Count);
NUnit.Framework.Assert.AreEqual(appId, requestProto.GetRunningApplications()[0]);
}
/// <exception cref="System.Exception"/>
private static void PrepareTimelineStore(TimelineStore store, int scale)
{
for (int i = 1; i <= scale; ++i)
{
TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.NewInstance(0, i);
if (i == 2)
{
entities.AddEntity(CreateApplicationTimelineEntity(appId, true, false, false));
}
else
{
entities.AddEntity(CreateApplicationTimelineEntity(appId, false, false, false));
}
store.Put(entities);
for (int j = 1; j <= scale; ++j)
{
entities = new TimelineEntities();
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
entities.AddEntity(CreateAppAttemptTimelineEntity(appAttemptId));
store.Put(entities);
for (int k = 1; k <= scale; ++k)
{
entities = new TimelineEntities();
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
entities.AddEntity(CreateContainerEntity(containerId));
store.Put(entities);
}
}
}
}
public virtual void TestExistenceOfResourceRequestInRMContainer()
{
Configuration conf = new Configuration();
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(1024);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
ResourceScheduler scheduler = rm1.GetResourceScheduler();
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
// Verify whether list of ResourceRequest is present in RMContainer
// while moving to ALLOCATED state
NUnit.Framework.Assert.IsNotNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
// Allocate container
am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired);
// After RMContainer moving to ACQUIRED state, list of ResourceRequest will
// be empty
NUnit.Framework.Assert.IsNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
}
/// <exception cref="System.Exception"/>
internal virtual ApplicationHistoryClientService MockApplicationHistoryClientService
(int numApps, int numAppAttempts, int numContainers)
{
ApplicationHistoryManager ahManager = new TestAHSWebApp.MockApplicationHistoryManagerImpl
(this, store);
ApplicationHistoryClientService historyClientService = new ApplicationHistoryClientService
(ahManager);
for (int i = 1; i <= numApps; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
WriteApplicationStartData(appId);
for (int j = 1; j <= numAppAttempts; ++j)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
WriteApplicationAttemptStartData(appAttemptId);
for (int k = 1; k <= numContainers; ++k)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
WriteContainerStartData(containerId);
WriteContainerFinishData(containerId);
}
WriteApplicationAttemptFinishData(appAttemptId);
}
WriteApplicationFinishData(appId);
}
return(historyClientService);
}
/// <exception cref="System.IO.IOException"/>
private void TestWriteHistoryData(int num, bool missingContainer, bool missingApplicationAttempt
)
{
// write application history data
for (int i = 1; i <= num; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
WriteApplicationStartData(appId);
// write application attempt history data
for (int j = 1; j <= num; ++j)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
WriteApplicationAttemptStartData(appAttemptId);
if (missingApplicationAttempt && j == num)
{
continue;
}
// write container history data
for (int k = 1; k <= num; ++k)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
WriteContainerStartData(containerId);
if (missingContainer && k == num)
{
continue;
}
WriteContainerFinishData(containerId);
}
WriteApplicationAttemptFinishData(appAttemptId);
}
WriteApplicationFinishData(appId);
}
}
public virtual void Handle(ContainerAllocatorEvent @event)
{
if (@event.GetType() == ContainerAllocator.EventType.ContainerReq)
{
Log.Info("Processing the event " + @event.ToString());
// Assign the same container ID as the AM
ContainerId cID = ContainerId.NewContainerId(GetContext().GetApplicationAttemptId
(), this.containerId.GetContainerId());
Container container = recordFactory.NewRecordInstance <Container>();
container.SetId(cID);
NodeId nodeId = NodeId.NewInstance(this.nmHost, this.nmPort);
container.SetNodeId(nodeId);
container.SetContainerToken(null);
container.SetNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
// send the container-assigned event to task attempt
if (@event.GetAttemptID().GetTaskId().GetTaskType() == TaskType.Map)
{
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(@event.GetAttemptID().GetTaskId
().GetJobId());
// TODO Setting OTHER_LOCAL_MAP for now.
jce.AddCounterUpdate(JobCounter.OtherLocalMaps, 1);
eventHandler.Handle(jce);
}
eventHandler.Handle(new TaskAttemptContainerAssignedEvent(@event.GetAttemptID(),
container, applicationACLs));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithLabels()
{
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
MockNM nm2 = rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm3);
// request a container.
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h2
RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm3);
// request a container.
am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue c1 (label = ""), and check all container will
// be allocated in h3
RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3);
// request a container.
am3.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
rm1.Close();
}
// Test RM restarts after AM container is preempted, new RM should not count
// AM preemption failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestPreemptedAMRestartOnRMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler();
ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 1);
// Forcibly preempt the am container;
scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
am1.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// state store has 1 attempt stored.
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount());
// attempt stored has the preempted container exit status.
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt
(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode();
rm2.Start();
// Restarted RM should re-launch the am.
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
public static ContainerId CreateContainerId()
{
ApplicationId appId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 0);
return(containerId);
}
private ContainerId CreateContainerId(int id)
{
ApplicationId appId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, id);
return(containerId);
}
public virtual void TestCreateAbnormalContainerStatus()
{
ContainerStatus cd = SchedulerUtils.CreateAbnormalContainerStatus(ContainerId.NewContainerId
(ApplicationAttemptId.NewInstance(ApplicationId.NewInstance(Runtime.CurrentTimeMillis
(), 1), 1), 1), "x");
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, cd.GetExitStatus());
}
public ContainerId GetAssignedContainerID()
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(taid.GetTaskId
().GetJobId().GetAppId(), 0);
ContainerId id = ContainerId.NewContainerId(appAttemptId, 0);
return(id);
}
private static ContainerId GetContainerId(ApplicationId applicationId, int startCount
)
{
ApplicationAttemptId appAttemptId = GetApplicationAttemptId(applicationId, startCount
);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, startCount);
return(containerId);
}
public static ContainerId NewContainerId(int appId, int appAttemptId, long timestamp
, long containerId)
{
ApplicationId applicationId = ApplicationId.NewInstance(timestamp, appId);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, appAttemptId);
return(ContainerId.NewContainerId(applicationAttemptId, containerId));
}
public TestMRApp(TestStagingCleanup _enclosing, ApplicationAttemptId applicationAttemptId
, ContainerAllocator allocator)
: base(applicationAttemptId, ContainerId.NewContainerId(applicationAttemptId, 1),
"testhost", 2222, 3333, Runtime.CurrentTimeMillis())
{
this._enclosing = _enclosing;
this.allocator = allocator;
this.successfullyUnregistered.Set(true);
}
// do nothing
private static AMInfo CreateAMInfo(int attempt)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(100, 1), attempt);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 1);
return(MRBuilderUtils.NewAMInfo(appAttemptId, Runtime.CurrentTimeMillis(), containerId
, NmHost, NmPort, NmHttpPort));
}
/// <exception cref="System.IO.IOException"/>
private void TestReadHistoryData(int num, bool missingContainer, bool missingApplicationAttempt
)
{
// read application history data
NUnit.Framework.Assert.AreEqual(num, store.GetAllApplications().Count);
for (int i = 1; i <= num; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
ApplicationHistoryData appData = store.GetApplication(appId);
NUnit.Framework.Assert.IsNotNull(appData);
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetApplicationName());
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetDiagnosticsInfo());
// read application attempt history data
NUnit.Framework.Assert.AreEqual(num, store.GetApplicationAttempts(appId).Count);
for (int j = 1; j <= num; ++j)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
ApplicationAttemptHistoryData attemptData = store.GetApplicationAttempt(appAttemptId
);
NUnit.Framework.Assert.IsNotNull(attemptData);
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetHost());
if (missingApplicationAttempt && j == num)
{
NUnit.Framework.Assert.IsNull(attemptData.GetDiagnosticsInfo());
continue;
}
else
{
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetDiagnosticsInfo
());
}
// read container history data
NUnit.Framework.Assert.AreEqual(num, store.GetContainers(appAttemptId).Count);
for (int k = 1; k <= num; ++k)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
ContainerHistoryData containerData = store.GetContainer(containerId);
NUnit.Framework.Assert.IsNotNull(containerData);
NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), containerData
.GetPriority());
if (missingContainer && k == num)
{
NUnit.Framework.Assert.IsNull(containerData.GetDiagnosticsInfo());
}
else
{
NUnit.Framework.Assert.AreEqual(containerId.ToString(), containerData.GetDiagnosticsInfo
());
}
}
ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId);
NUnit.Framework.Assert.IsNotNull(masterContainer);
NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer
.GetContainerId());
}
}
}
/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerTokenWithEpoch(Configuration conf)
{
Log.Info("Running test for serializing/deserializing containerIds");
NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager
().GetRMContext().GetNMTokenSecretManager();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L);
NodeManager nm = yarnCluster.GetNodeManager(0);
NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager
();
string user = "******";
WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
NodeId nodeId = nm.GetNMContext().GetNodeId();
// Both id should be equal.
NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId
(), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId());
// Creating a normal Container Token
RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager
().GetRMContext().GetContainerTokenSecretManager();
Resource r = Resource.NewInstance(1230, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager
.CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
);
byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array());
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
containerTokenIdentifier.ReadFields(dib);
NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID());
NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID
().ToString());
Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken
(appAttemptId, nodeId, user);
YarnRPC rpc = YarnRPC.Create(conf);
TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
IList <ContainerId> containerIds = new List <ContainerId>();
containerIds.AddItem(cId);
ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken
, nodeId, user);
GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest
.NewInstance(containerIds));
NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]);
NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId
());
NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId
().ToString());
}
public static NMContainerStatus CreateNMContainerStatus(ApplicationAttemptId appAttemptId
, int id, ContainerState containerState, int memory)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, id);
NMContainerStatus containerReport = NMContainerStatus.NewInstance(containerId, containerState
, Resource.NewInstance(memory, 1), "recover container", 0, Priority.NewInstance(
0), 0);
return(containerReport);
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
private Container MockContainer(int i)
{
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(attemptId, i);
nodeId = NodeId.NewInstance("localhost", 0);
// Create an empty record
containerToken = recordFactory.NewRecordInstance <Token>();
return(Container.NewInstance(containerId, nodeId, null, null, null, containerToken
));
}
private RMContainer CreateRMContainer(ApplicationAttemptId appAttId, int id, Resource
resource)
{
ContainerId containerId = ContainerId.NewContainerId(appAttId, id);
RMContainer rmContainer = Org.Mockito.Mockito.Mock <RMContainer>();
Container container = Org.Mockito.Mockito.Mock <Container>();
Org.Mockito.Mockito.When(container.GetResource()).ThenReturn(resource);
Org.Mockito.Mockito.When(container.GetNodeId()).ThenReturn(nodeId);
Org.Mockito.Mockito.When(rmContainer.GetContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(rmContainer.GetContainerId()).ThenReturn(containerId);
return(rmContainer);
}
