public virtual void TestRetriesOnFailures()
{
ContainerManagementProtocol mockProxy = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(mockProxy.StartContainers(Matchers.Any <StartContainersRequest
>())).ThenThrow(new NMNotYetReadyException("foo")).ThenReturn(mockResponse);
Configuration conf = new Configuration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
conf.SetInt(YarnConfiguration.ClientNmConnectRetryIntervalMs, 1);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_206(dispatcher, mockProxy, conf, null
);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId();
// kick the scheduling
nm1.NodeHeartbeat(true);
dispatcher.Await();
rm.WaitForState(appAttemptId, RMAppAttemptState.Launched, 500);
}
/// <summary>Get the fair share assigned to the appAttemptId.</summary>
/// <param name="appAttemptId"/>
/// <returns>
/// The fair share assigned to the appAttemptId,
/// <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does
/// not know about this application attempt.
/// </returns>
public virtual int GetAppFairShare(ApplicationAttemptId appAttemptId)
{
FSAppAttempt fsAppAttempt = scheduler.GetSchedulerApp(appAttemptId);
return(fsAppAttempt == null ? InvalidFairShare : fsAppAttempt.GetFairShare().GetMemory
());
}
public virtual void TestSingleAttempt()
{
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
WebResource r = Resource();
ClientResponse response = r.Path("ws").Path("v1").Path("applicationhistory").Path
("apps").Path(appId.ToString()).Path("appattempts").Path(appAttemptId.ToString()
).QueryParam("user.name", Users[round]).Accept(MediaType.ApplicationJson).Get <ClientResponse
>();
if (round == 1)
{
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Forbidden, response.GetClientResponseStatus
());
return;
}
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
JSONObject appAttempt = json.GetJSONObject("appAttempt");
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), appAttempt.GetString("appAttemptId"
));
NUnit.Framework.Assert.AreEqual("test host", appAttempt.GetString("host"));
NUnit.Framework.Assert.AreEqual("test diagnostics info", appAttempt.GetString("diagnosticsInfo"
));
NUnit.Framework.Assert.AreEqual("test tracking url", appAttempt.GetString("trackingUrl"
));
NUnit.Framework.Assert.AreEqual(YarnApplicationAttemptState.Finished.ToString(),
appAttempt.Get("appAttemptState"));
}
public virtual IList <Container> GetTransferredContainers(ApplicationAttemptId currentAttempt
)
{
ApplicationId appId = currentAttempt.GetApplicationId();
SchedulerApplication <T> app = applications[appId];
IList <Container> containerList = new AList <Container>();
RMApp appImpl = this.rmContext.GetRMApps()[appId];
if (appImpl.GetApplicationSubmissionContext().GetUnmanagedAM())
{
return(containerList);
}
if (app == null)
{
return(containerList);
}
ICollection <RMContainer> liveContainers = app.GetCurrentAppAttempt().GetLiveContainers
();
ContainerId amContainerId = rmContext.GetRMApps()[appId].GetCurrentAppAttempt().GetMasterContainer
().GetId();
foreach (RMContainer rmContainer in liveContainers)
{
if (!rmContainer.GetContainerId().Equals(amContainerId))
{
containerList.AddItem(rmContainer.GetContainer());
}
}
return(containerList);
}
/// <exception cref="System.Exception"/>
public virtual void TestSchedulingDelay()
{
// Add one node
string host = "127.0.0.1";
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4096, 4), 1, host
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.Handle(nodeUpdateEvent);
// Create one application and submit one each of node-local, rack-local
// and ANY requests
ApplicationAttemptId appAttemptId = CreateAppAttemptId(this.AppId++, this.AttemptId
++);
CreateMockRMApp(appAttemptId);
scheduler.AddApplication(appAttemptId.GetApplicationId(), "queue11", "user11", false
);
scheduler.AddApplicationAttempt(appAttemptId, false, false);
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ask.AddItem(CreateResourceRequest(1024, 1, ResourceRequest.Any, 1, 1, true));
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
FSAppAttempt app = scheduler.GetSchedulerApp(appAttemptId);
// Advance time and let continuous scheduling kick in
mockClock.Tick(1);
while (1024 != app.GetCurrentConsumption().GetMemory())
{
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.AreEqual(1024, app.GetCurrentConsumption().GetMemory());
}
/// <exception cref="System.Exception"/>
private static void PrepareTimelineStore(TimelineStore store, int scale)
{
for (int i = 1; i <= scale; ++i)
{
TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.NewInstance(0, i);
if (i == 2)
{
entities.AddEntity(CreateApplicationTimelineEntity(appId, true, false, false));
}
else
{
entities.AddEntity(CreateApplicationTimelineEntity(appId, false, false, false));
}
store.Put(entities);
for (int j = 1; j <= scale; ++j)
{
entities = new TimelineEntities();
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
entities.AddEntity(CreateAppAttemptTimelineEntity(appAttemptId));
store.Put(entities);
for (int k = 1; k <= scale; ++k)
{
entities = new TimelineEntities();
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
entities.AddEntity(CreateContainerEntity(containerId));
store.Put(entities);
}
}
}
}
private static TimelineEntity CreateApplicationTimelineEntity(ApplicationId appId
, bool emptyACLs, bool noAttemptId, bool wrongAppId)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(ApplicationMetricsConstants.EntityType);
if (wrongAppId)
{
entity.SetEntityId("wrong_app_id");
}
else
{
entity.SetEntityId(appId.ToString());
}
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddPrimaryFilter(TimelineStore.SystemFilter.EntityOwner.ToString(), "yarn"
);
IDictionary <string, object> entityInfo = new Dictionary <string, object>();
entityInfo[ApplicationMetricsConstants.NameEntityInfo] = "test app";
entityInfo[ApplicationMetricsConstants.TypeEntityInfo] = "test app type";
entityInfo[ApplicationMetricsConstants.UserEntityInfo] = "user1";
entityInfo[ApplicationMetricsConstants.QueueEntityInfo] = "test queue";
entityInfo[ApplicationMetricsConstants.SubmittedTimeEntityInfo] = int.MaxValue +
1L;
entityInfo[ApplicationMetricsConstants.AppMemMetrics] = 123;
entityInfo[ApplicationMetricsConstants.AppCpuMetrics] = 345;
if (emptyACLs)
{
entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo] = string.Empty;
}
else
{
entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo] = "user2";
}
entity.SetOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.SetEventType(ApplicationMetricsConstants.CreatedEventType);
tEvent.SetTimestamp(int.MaxValue + 2L + appId.GetId());
entity.AddEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.SetEventType(ApplicationMetricsConstants.FinishedEventType);
tEvent.SetTimestamp(int.MaxValue + 3L + appId.GetId());
IDictionary <string, object> eventInfo = new Dictionary <string, object>();
eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo] = "test diagnostics info";
eventInfo[ApplicationMetricsConstants.FinalStatusEventInfo] = FinalApplicationStatus
.Undefined.ToString();
eventInfo[ApplicationMetricsConstants.StateEventInfo] = YarnApplicationState.Finished
.ToString();
if (!noAttemptId)
{
eventInfo[ApplicationMetricsConstants.LatestAppAttemptEventInfo] = ApplicationAttemptId
.NewInstance(appId, 1);
}
tEvent.SetEventInfo(eventInfo);
entity.AddEvent(tEvent);
return(entity);
}
public RMAppAttemptContainerFinishedEvent(ApplicationAttemptId appAttemptId, ContainerStatus
containerStatus, NodeId nodeId)
: base(appAttemptId, RMAppAttemptEventType.ContainerFinished)
{
this.containerStatus = containerStatus;
this.nodeId = nodeId;
}
public virtual void TestHandleContainerStatusInvalidCompletions()
{
rm = new MockRM(new YarnConfiguration());
rm.Start();
EventHandler handler = Org.Mockito.Mockito.Spy(rm.GetRMContext().GetDispatcher().
GetEventHandler());
// Case 1: Unmanaged AM
RMApp app = rm.SubmitApp(1024, true);
// Case 1.1: AppAttemptId is null
NMContainerStatus report = NMContainerStatus.NewInstance(ContainerId.NewContainerId
(ApplicationAttemptId.NewInstance(app.GetApplicationId(), 2), 1), ContainerState
.Complete, Resource.NewInstance(1024, 1), "Dummy Completed", 0, Priority.NewInstance
(10), 1234);
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 1.2: Master container is null
RMAppAttemptImpl currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();
currentAttempt.SetMasterContainer(null);
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
, 0, Priority.NewInstance(10), 1234);
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 2: Managed AM
app = rm.SubmitApp(1024);
// Case 2.1: AppAttemptId is null
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(ApplicationAttemptId
.NewInstance(app.GetApplicationId(), 2), 1), ContainerState.Complete, Resource.NewInstance
(1024, 1), "Dummy Completed", 0, Priority.NewInstance(10), 1234);
try
{
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
}
catch (Exception)
{
}
// expected - ignore
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 2.2: Master container is null
currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();
currentAttempt.SetMasterContainer(null);
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
, 0, Priority.NewInstance(10), 1234);
try
{
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
}
catch (Exception)
{
}
// expected - ignore
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
}
/// <exception cref="System.IO.IOException"/>
private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app)
{
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 2);
nmContext.GetContainers()[container1.GetContainerId()] = container1;
nmContext.GetContainers()[container2.GetContainerId()] = container2;
app.GetContainers()[container1.GetContainerId()] = container1;
app.GetContainers()[container2.GetContainerId()] = container2;
Dictionary <string, string> hash = new Dictionary <string, string>();
hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString
();
hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString
();
return(hash);
}
public static TaskAttemptReport NewTaskAttemptReport(TaskAttemptId id)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(id.GetTaskId
().GetJobId().GetAppId(), 0);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 0);
TaskAttemptReport report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <TaskAttemptReport
>();
report.SetTaskAttemptId(id);
report.SetStartTime(Runtime.CurrentTimeMillis() - (int)(Math.Random() * Dt));
report.SetFinishTime(Runtime.CurrentTimeMillis() + (int)(Math.Random() * Dt) + 1);
if (id.GetTaskId().GetTaskType() == TaskType.Reduce)
{
report.SetShuffleFinishTime((report.GetFinishTime() + report.GetStartTime()) / 2);
report.SetSortFinishTime((report.GetFinishTime() + report.GetShuffleFinishTime())
/ 2);
}
report.SetPhase(Phases.Next());
report.SetTaskAttemptState(TaskAttemptStates.Next());
report.SetProgress((float)Math.Random());
report.SetCounters(TypeConverter.ToYarn(NewCounters()));
report.SetContainerId(containerId);
report.SetDiagnosticInfo(Diags.Next());
report.SetStateString("Moving average " + Math.Random());
return(report);
}
protected internal virtual ApplicationAttemptId CreateAppAttemptId(int appId, int
attemptId)
{
ApplicationId appIdImpl = ApplicationId.NewInstance(0, appId);
return(ApplicationAttemptId.NewInstance(appIdImpl, attemptId));
}
protected internal virtual ApplicationAttemptId CreateSchedulingRequest(int memory
, int vcores, string queueId, string userId, int numContainers, int priority)
{
ApplicationAttemptId id = CreateAppAttemptId(this.AppId++, this.AttemptId++);
scheduler.AddApplication(id.GetApplicationId(), queueId, userId, false);
// This conditional is for testAclSubmitApplication where app is rejected
// and no app is added.
if (scheduler.GetSchedulerApplications().Contains(id.GetApplicationId()))
{
scheduler.AddApplicationAttempt(id, false, false);
}
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest request = CreateResourceRequest(memory, vcores, ResourceRequest.Any
, priority, numContainers, true);
ask.AddItem(request);
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttempt rmAppAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(rmApp.GetCurrentAppAttempt()).ThenReturn(rmAppAttempt);
Org.Mockito.Mockito.When(rmAppAttempt.GetRMAppAttemptMetrics()).ThenReturn(new RMAppAttemptMetrics
(id, resourceManager.GetRMContext()));
resourceManager.GetRMContext().GetRMApps()[id.GetApplicationId()] = rmApp;
scheduler.Allocate(id, ask, new AList <ContainerId>(), null, null);
return(id);
}
private ConcurrentMap <ContainerId, ContainerHistoryData> GetSubMap(ApplicationAttemptId
appAttemptId)
{
containerData.PutIfAbsent(appAttemptId, new ConcurrentHashMap <ContainerId, ContainerHistoryData
>());
return(containerData[appAttemptId]);
}
/// <exception cref="System.IO.IOException"/>
private void DoneApplicationAttempt(ApplicationAttemptId applicationAttemptId, RMAppAttemptState
rmAppAttemptFinalState, bool keepContainers)
{
lock (this)
{
FiCaSchedulerApp attempt = GetApplicationAttempt(applicationAttemptId);
SchedulerApplication <FiCaSchedulerApp> application = applications[applicationAttemptId
.GetApplicationId()];
if (application == null || attempt == null)
{
throw new IOException("Unknown application " + applicationAttemptId + " has completed!"
);
}
// Kill all 'live' containers
foreach (RMContainer container in attempt.GetLiveContainers())
{
if (keepContainers && container.GetState().Equals(RMContainerState.Running))
{
// do not kill the running container in the case of work-preserving AM
// restart.
Log.Info("Skip killing " + container.GetContainerId());
continue;
}
CompletedContainer(container, SchedulerUtils.CreateAbnormalContainerStatus(container
.GetContainerId(), SchedulerUtils.CompletedApplication), RMContainerEventType.Kill
);
}
// Clean up pending requests, metrics etc.
attempt.Stop(rmAppAttemptFinalState);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestPostExecuteAfterReacquisition()
{
// make up some bogus container ID
ApplicationId appId = ApplicationId.NewInstance(12345, 67890);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 54321);
ContainerId cid = ContainerId.NewContainerId(attemptId, 9876);
Configuration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.NmLinuxContainerResourcesHandler, typeof(TestLinuxContainerExecutor.TestResourceHandler
), typeof(LCEResourcesHandler));
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.SetConf(conf);
try
{
lce.Init();
}
catch (IOException)
{
}
// expected if LCE isn't setup right, but not necessary for this test
lce.ReacquireContainer("foouser", cid);
NUnit.Framework.Assert.IsTrue("postExec not called after reacquisition", TestLinuxContainerExecutor.TestResourceHandler
.postExecContainers.Contains(cid));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override ApplicationAttemptReport GetApplicationAttemptReport(ApplicationAttemptId
appAttemptId)
{
try
{
GetApplicationAttemptReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.
NewRecord <GetApplicationAttemptReportRequest>();
request.SetApplicationAttemptId(appAttemptId);
GetApplicationAttemptReportResponse response = rmClient.GetApplicationAttemptReport
(request);
return(response.GetApplicationAttemptReport());
}
catch (YarnException e)
{
if (!historyServiceEnabled)
{
// Just throw it as usual if historyService is not enabled.
throw;
}
// Even if history-service is enabled, treat all exceptions still the same
// except the following
if (e.GetType() != typeof(ApplicationNotFoundException))
{
throw;
}
return(historyClient.GetApplicationAttemptReport(appAttemptId));
}
}
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string
user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
{
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
// receive the release request form AM before it receives the container status
// from NM for recovery. In this case, the to-be-recovered containers reported
// by NM should not be recovered.
// Time of the last container scheduled at the current allowed level
Preconditions.CheckNotNull(rmContext, "RMContext should not be null");
this.rmContext = rmContext;
this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue,
activeUsersManager, rmContext.GetEpoch());
this.queue = queue;
this.pendingRelease = new HashSet <ContainerId>();
this.attemptId = applicationAttemptId;
if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId
.GetApplicationId()))
{
ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId
.GetApplicationId()].GetApplicationSubmissionContext();
if (appSubmissionContext != null)
{
unmanagedAM = appSubmissionContext.GetUnmanagedAM();
this.logAggregationContext = appSubmissionContext.GetLogAggregationContext();
}
}
}
public virtual void TestGetContainers()
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(0, 1), 1);
ICollection <ContainerReport> containers;
if (callerUGI == null)
{
containers = historyManager.GetContainers(appAttemptId).Values;
}
else
{
try
{
containers = callerUGI.DoAs(new _PrivilegedExceptionAction_390(this, appAttemptId
));
if (callerUGI != null && callerUGI.GetShortUserName().Equals("user3"))
{
// The exception is expected
NUnit.Framework.Assert.Fail();
}
}
catch (AuthorizationException e)
{
if (callerUGI != null && callerUGI.GetShortUserName().Equals("user3"))
{
// The exception is expected
return;
}
throw;
}
}
NUnit.Framework.Assert.IsNotNull(containers);
NUnit.Framework.Assert.AreEqual(Scale, containers.Count);
}
public RMAppAttemptUnregistrationEvent(ApplicationAttemptId appAttemptId, string
trackingUrl, FinalApplicationStatus finalStatus, string diagnostics)
: base(appAttemptId, RMAppAttemptEventType.Unregistered, diagnostics)
{
this.finalTrackingUrl = trackingUrl;
this.finalStatus = finalStatus;
}
public virtual T GetApplicationAttempt(ApplicationAttemptId applicationAttemptId)
{
SchedulerApplication <T> app = applications[applicationAttemptId.GetApplicationId(
)];
return(app == null ? null : app.GetCurrentAppAttempt());
}
/// <summary>Create an event to record the start of an MR AppMaster</summary>
/// <param name="appAttemptId">the application attempt id.</param>
/// <param name="startTime">the start time of the AM.</param>
/// <param name="containerId">the containerId of the AM.</param>
/// <param name="nodeManagerHost">the node on which the AM is running.</param>
/// <param name="nodeManagerPort">the port on which the AM is running.</param>
/// <param name="nodeManagerHttpPort">the httpPort for the node running the AM.</param>
public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime, ContainerId
containerId, string nodeManagerHost, int nodeManagerPort, int nodeManagerHttpPort
, long submitTime)
: this(appAttemptId, startTime, containerId, nodeManagerHost, nodeManagerPort, nodeManagerHttpPort
, null, submitTime)
{
}
public virtual void TestRegisterNodeManagerRequest()
{
ApplicationId appId = ApplicationId.NewInstance(123456789, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(attemptId, 1);
NMContainerStatus containerReport = NMContainerStatus.NewInstance(containerId, ContainerState
.Running, Resource.NewInstance(1024, 1), "diagnostics", 0, Priority.NewInstance(
10), 1234);
IList <NMContainerStatus> reports = Arrays.AsList(containerReport);
RegisterNodeManagerRequest request = RegisterNodeManagerRequest.NewInstance(NodeId
.NewInstance("1.1.1.1", 1000), 8080, Resource.NewInstance(1024, 1), "NM-version-id"
, reports, Arrays.AsList(appId));
RegisterNodeManagerRequest requestProto = new RegisterNodeManagerRequestPBImpl(((
RegisterNodeManagerRequestPBImpl)request).GetProto());
NUnit.Framework.Assert.AreEqual(containerReport, requestProto.GetNMContainerStatuses
()[0]);
NUnit.Framework.Assert.AreEqual(8080, requestProto.GetHttpPort());
NUnit.Framework.Assert.AreEqual("NM-version-id", requestProto.GetNMVersion());
NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("1.1.1.1", 1000), requestProto
.GetNodeId());
NUnit.Framework.Assert.AreEqual(Resource.NewInstance(1024, 1), requestProto.GetResource
());
NUnit.Framework.Assert.AreEqual(1, requestProto.GetRunningApplications().Count);
NUnit.Framework.Assert.AreEqual(appId, requestProto.GetRunningApplications()[0]);
}
public virtual void TestNMTokenIdentifier()
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(1, 1), 1);
NodeId nodeId = NodeId.NewInstance("host0", 0);
string applicationSubmitter = "usr0";
int masterKeyId = 1;
NMTokenIdentifier token = new NMTokenIdentifier(appAttemptId, nodeId, applicationSubmitter
, masterKeyId);
NMTokenIdentifier anotherToken = new NMTokenIdentifier();
byte[] tokenContent = token.GetBytes();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenContent, tokenContent.Length);
anotherToken.ReadFields(dib);
// verify the whole record equals with original record
NUnit.Framework.Assert.AreEqual("Token is not the same after serialization " + "and deserialization."
, token, anotherToken);
// verify all properties are the same as original
NUnit.Framework.Assert.AreEqual("appAttemptId from proto is not the same with original token"
, anotherToken.GetApplicationAttemptId(), appAttemptId);
NUnit.Framework.Assert.AreEqual("NodeId from proto is not the same with original token"
, anotherToken.GetNodeId(), nodeId);
NUnit.Framework.Assert.AreEqual("applicationSubmitter from proto is not the same with original token"
, anotherToken.GetApplicationSubmitter(), applicationSubmitter);
NUnit.Framework.Assert.AreEqual("masterKeyId from proto is not the same with original token"
, anotherToken.GetKeyId(), masterKeyId);
}
public WritingApplicationAttemptFinishEvent(ApplicationAttemptId appAttemptId, ApplicationAttemptFinishData
appAttemptFinish)
: base(WritingHistoryEventType.AppAttemptFinish)
{
this.appAttemptId = appAttemptId;
this.appAttemptFinish = appAttemptFinish;
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerReport GetAMContainer(ApplicationAttemptId appAttemptId)
{
ApplicationReport app = GetApplication(appAttemptId.GetApplicationId());
return(ConvertToContainerReport(historyStore.GetAMContainer(appAttemptId), app ==
null ? null : app.GetUser()));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestMRAppMasterMaxAppAttempts()
{
// No matter what's the maxAppAttempt or attempt id, the isLastRetry always
// equals to false
bool[] expectedBools = new bool[] { false, false, false };
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
for (int i = 0; i < expectedBools.Length; ++i)
{
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools
[i], appMaster.IsLastAMRetry());
}
}
public virtual void AddApplicationAttempt(ApplicationAttemptId appAttemptId, bool
transferStateFromPreviousAttempt, bool isAttemptRecovering)
{
lock (this)
{
SchedulerApplication <FiCaSchedulerApp> application = applications[appAttemptId.GetApplicationId
()];
string user = application.GetUser();
// TODO: Fix store
FiCaSchedulerApp schedulerApp = new FiCaSchedulerApp(appAttemptId, user, DefaultQueue
, activeUsersManager, this.rmContext);
if (transferStateFromPreviousAttempt)
{
schedulerApp.TransferStateFromPreviousAttempt(application.GetCurrentAppAttempt());
}
application.SetCurrentAppAttempt(schedulerApp);
metrics.SubmitAppAttempt(user);
Log.Info("Added Application Attempt " + appAttemptId + " to scheduler from user "
+ application.GetUser());
if (isAttemptRecovering)
{
if (Log.IsDebugEnabled())
{
Log.Debug(appAttemptId + " is recovering. Skipping notifying ATTEMPT_ADDED");
}
}
else
{
rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
, RMAppAttemptEventType.AttemptAdded));
}
}
}
public virtual void TestMultipleContainers()
{
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
WebResource r = Resource();
ClientResponse response = r.Path("ws").Path("v1").Path("applicationhistory").Path
("apps").Path(appId.ToString()).Path("appattempts").Path(appAttemptId.ToString()
).Path("containers").QueryParam("user.name", Users[round]).Accept(MediaType.ApplicationJson
).Get <ClientResponse>();
if (round == 1)
{
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Forbidden, response.GetClientResponseStatus
());
return;
}
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
JSONObject containers = json.GetJSONObject("containers");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, containers.Length
());
JSONArray array = containers.GetJSONArray("container");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 5, array.Length()
);
}
public virtual void TestContainerMetricsFinished()
{
MetricsSystemImpl system = new MetricsSystemImpl();
system.Init("test");
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ApplicationId appId = ApplicationId.NewInstance(1234, 3);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 4);
ContainerId containerId1 = ContainerId.NewContainerId(appAttemptId, 1);
ContainerMetrics metrics1 = ContainerMetrics.ForContainer(system, containerId1, 1
, 0);
ContainerId containerId2 = ContainerId.NewContainerId(appAttemptId, 2);
ContainerMetrics metrics2 = ContainerMetrics.ForContainer(system, containerId2, 1
, 0);
ContainerId containerId3 = ContainerId.NewContainerId(appAttemptId, 3);
ContainerMetrics metrics3 = ContainerMetrics.ForContainer(system, containerId3, 1
, 0);
metrics1.Finished();
metrics2.Finished();
system.SampleMetrics();
system.SampleMetrics();
Sharpen.Thread.Sleep(100);
system.Stop();
// verify metrics1 is unregistered
NUnit.Framework.Assert.IsTrue(metrics1 != ContainerMetrics.ForContainer(system, containerId1
, 1, 0));
// verify metrics2 is unregistered
NUnit.Framework.Assert.IsTrue(metrics2 != ContainerMetrics.ForContainer(system, containerId2
, 1, 0));
// verify metrics3 is still registered
NUnit.Framework.Assert.IsTrue(metrics3 == ContainerMetrics.ForContainer(system, containerId3
, 1, 0));
system.Shutdown();
}
