public virtual IList <Container> GetTransferredContainers(ApplicationAttemptId currentAttempt
)
{
ApplicationId appId = currentAttempt.GetApplicationId();
SchedulerApplication <T> app = applications[appId];
IList <Container> containerList = new AList <Container>();
RMApp appImpl = this.rmContext.GetRMApps()[appId];
if (appImpl.GetApplicationSubmissionContext().GetUnmanagedAM())
{
return(containerList);
}
if (app == null)
{
return(containerList);
}
ICollection <RMContainer> liveContainers = app.GetCurrentAppAttempt().GetLiveContainers
();
ContainerId amContainerId = rmContext.GetRMApps()[appId].GetCurrentAppAttempt().GetMasterContainer
().GetId();
foreach (RMContainer rmContainer in liveContainers)
{
if (!rmContainer.GetContainerId().Equals(amContainerId))
{
containerList.AddItem(rmContainer.GetContainer());
}
}
return(containerList);
}
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string
user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
{
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
// receive the release request form AM before it receives the container status
// from NM for recovery. In this case, the to-be-recovered containers reported
// by NM should not be recovered.
// Time of the last container scheduled at the current allowed level
Preconditions.CheckNotNull(rmContext, "RMContext should not be null");
this.rmContext = rmContext;
this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue,
activeUsersManager, rmContext.GetEpoch());
this.queue = queue;
this.pendingRelease = new HashSet <ContainerId>();
this.attemptId = applicationAttemptId;
if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId
.GetApplicationId()))
{
ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId
.GetApplicationId()].GetApplicationSubmissionContext();
if (appSubmissionContext != null)
{
unmanagedAM = appSubmissionContext.GetUnmanagedAM();
this.logAggregationContext = appSubmissionContext.GetLogAggregationContext();
}
}
}
// resourceLimit: updated when LeafQueue#recoverContainer#allocateResource
// is called.
// newlyAllocatedContainers.add(rmContainer);
// schedulingOpportunities
// lastScheduledContainer
public virtual void IncNumAllocatedContainers(NodeType containerType, NodeType requestType
)
{
RMAppAttempt attempt = rmContext.GetRMApps()[attemptId.GetApplicationId()].GetCurrentAppAttempt
();
if (attempt != null)
{
attempt.GetRMAppAttemptMetrics().IncNumAllocatedContainers(containerType, requestType
);
}
}
public virtual void Handle(NodesListManagerEvent @event)
{
RMNode eventNode = @event.GetNode();
switch (@event.GetType())
{
case NodesListManagerEventType.NodeUnusable:
{
Log.Debug(eventNode + " reported unusable");
unusableRMNodesConcurrentSet.AddItem(eventNode);
foreach (RMApp app in rmContext.GetRMApps().Values)
{
if (!app.IsAppFinalStateStored())
{
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppNodeUpdateEvent(
app.GetApplicationId(), eventNode, RMAppNodeUpdateEvent.RMAppNodeUpdateType.NodeUnusable
));
}
}
break;
}
case NodesListManagerEventType.NodeUsable:
{
if (unusableRMNodesConcurrentSet.Contains(eventNode))
{
Log.Debug(eventNode + " reported usable");
unusableRMNodesConcurrentSet.Remove(eventNode);
}
foreach (RMApp app_1 in rmContext.GetRMApps().Values)
{
if (!app_1.IsAppFinalStateStored())
{
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppNodeUpdateEvent(
app_1.GetApplicationId(), eventNode, RMAppNodeUpdateEvent.RMAppNodeUpdateType.NodeUsable
));
}
}
break;
}
default:
{
Log.Error("Ignoring invalid eventtype " + @event.GetType());
break;
}
}
}
public static ClientRMService MockClientRMService(RMContext rmContext)
{
ClientRMService clientRMService = Org.Mockito.Mockito.Mock <ClientRMService>();
IList <ApplicationReport> appReports = new AList <ApplicationReport>();
foreach (RMApp app in rmContext.GetRMApps().Values)
{
ApplicationReport appReport = ApplicationReport.NewInstance(app.GetApplicationId(
), (ApplicationAttemptId)null, app.GetUser(), app.GetQueue(), app.GetName(), (string
)null, 0, (Token)null, app.CreateApplicationState(), app.GetDiagnostics().ToString
(), (string)null, app.GetStartTime(), app.GetFinishTime(), app.GetFinalApplicationStatus
(), (ApplicationResourceUsageReport)null, app.GetTrackingUrl(), app.GetProgress(
), app.GetApplicationType(), (Token)null);
appReports.AddItem(appReport);
}
GetApplicationsResponse response = Org.Mockito.Mockito.Mock <GetApplicationsResponse
>();
Org.Mockito.Mockito.When(response.GetApplicationList()).ThenReturn(appReports);
try
{
Org.Mockito.Mockito.When(clientRMService.GetApplications(Matchers.Any <GetApplicationsRequest
>())).ThenReturn(response);
}
catch (YarnException)
{
NUnit.Framework.Assert.Fail("Exception is not expteced.");
}
return(clientRMService);
}
public virtual void TestStateStoreAppLimitLargerThanMemoryAppLimit()
{
long now = Runtime.CurrentTimeMillis();
RMContext rmContext = MockRMContext(10, now - 20000);
Configuration conf = new YarnConfiguration();
int maxAppsInMemory = 8;
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, maxAppsInMemory);
// larger than maxCompletedAppsInMemory, reset to RM_MAX_COMPLETED_APPLICATIONS.
conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 1000);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
AddToCompletedApps(appMonitor, rmContext);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect", 10, appMonitor
.GetCompletedAppsListSize());
appMonitor.CheckAppNumCompletedLimit();
int numRemoveApps = 10 - maxAppsInMemory;
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, maxAppsInMemory, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
maxAppsInMemory, appMonitor.GetCompletedAppsListSize());
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(numRemoveApps
)).RemoveApplication(Matchers.IsA <RMApp>());
NUnit.Framework.Assert.AreEqual(maxAppsInMemory, appMonitor.GetCompletedAppsInStateStore
());
}
/// <exception cref="System.Exception"/>
public virtual void WaitForState(RMAppAttemptState finalState)
{
RMApp app = context.GetRMApps()[attemptId.GetApplicationId()];
RMAppAttempt attempt = app.GetRMAppAttempt(attemptId);
int timeoutSecs = 0;
while (!finalState.Equals(attempt.GetAppAttemptState()) && timeoutSecs++ < 40)
{
System.Console.Out.WriteLine("AppAttempt : " + attemptId + " State is : " + attempt
.GetAppAttemptState() + " Waiting for state : " + finalState);
Sharpen.Thread.Sleep(1000);
}
System.Console.Out.WriteLine("AppAttempt State is : " + attempt.GetAppAttemptState
());
NUnit.Framework.Assert.AreEqual("AppAttempt state is not correct (timedout)", finalState
, attempt.GetAppAttemptState());
}
// Protected. For tests.
protected internal virtual ContainerManagementProtocol GetContainerMgrProxy(ContainerId
containerId)
{
NodeId node = masterContainer.GetNodeId();
IPEndPoint containerManagerConnectAddress = NetUtils.CreateSocketAddrForHost(node
.GetHost(), node.GetPort());
YarnRPC rpc = GetYarnRPC();
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(containerId
.GetApplicationAttemptId().ToString());
string user = rmContext.GetRMApps()[containerId.GetApplicationAttemptId().GetApplicationId
()].GetUser();
Token token = rmContext.GetNMTokenSecretManager().CreateNMToken(containerId.GetApplicationAttemptId
(), node, user);
currentUser.AddToken(ConverterUtils.ConvertFromYarn(token, containerManagerConnectAddress
));
return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, currentUser, rpc,
containerManagerConnectAddress));
}
protected internal virtual void AddToCompletedApps(TestAppManager.TestRMAppManager
appMonitor, RMContext rmContext)
{
foreach (RMApp app in rmContext.GetRMApps().Values)
{
if (app.GetState() == RMAppState.Finished || app.GetState() == RMAppState.Killed ||
app.GetState() == RMAppState.Failed)
{
appMonitor.FinishApplication(app.GetApplicationId());
}
}
}
public virtual void TestRMAppSubmit()
{
appMonitor.SubmitApplication(asContext, "test");
RMApp app = rmContext.GetRMApps()[appId];
NUnit.Framework.Assert.IsNotNull("app is null", app);
NUnit.Framework.Assert.AreEqual("app id doesn't match", appId, app.GetApplicationId
());
NUnit.Framework.Assert.AreEqual("app state doesn't match", RMAppState.New, app.GetState
());
Org.Mockito.Mockito.Verify(metricsPublisher).AppACLsUpdated(Matchers.Any <RMApp>()
, Matchers.Any <string>(), Matchers.AnyLong());
// wait for event to be processed
int timeoutSecs = 0;
while ((GetAppEventType() == RMAppEventType.Kill) && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.AreEqual("app event type sent is wrong", RMAppEventType.Start
, GetAppEventType());
}
public virtual void TestRMAppRetireNullApp()
{
long now = Runtime.CurrentTimeMillis();
RMContext rmContext = MockRMContext(10, now - 20000);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, new Configuration());
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext.
GetRMApps().Count);
appMonitor.FinishApplication(null);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
0, appMonitor.GetCompletedAppsListSize());
}
public virtual void TestRMAppRetireZeroSetting()
{
long now = Runtime.CurrentTimeMillis();
RMContext rmContext = MockRMContext(10, now - 20000);
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 0);
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 0);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext.
GetRMApps().Count);
AddToCompletedApps(appMonitor, rmContext);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect", 10, appMonitor
.GetCompletedAppsListSize());
appMonitor.CheckAppNumCompletedLimit();
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, 0, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
0, appMonitor.GetCompletedAppsListSize());
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(10
)).RemoveApplication(Matchers.IsA <RMApp>());
}
public virtual void TestRMAppRetireNone()
{
long now = Runtime.CurrentTimeMillis();
// Create such that none of the applications will retire since
// haven't hit max #
RMContext rmContext = MockRMContext(10, now - 10);
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 10);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before checkAppTimeLimit"
, 10, rmContext.GetRMApps().Count);
// add them to completed apps list
AddToCompletedApps(appMonitor, rmContext);
// shouldn't have to many apps
appMonitor.CheckAppNumCompletedLimit();
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, 10, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
10, appMonitor.GetCompletedAppsListSize());
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Never()
).RemoveApplication(Matchers.IsA <RMApp>());
}
public virtual void SetUp()
{
CapacityScheduler spyCs = new CapacityScheduler();
cs = Org.Mockito.Mockito.Spy(spyCs);
scheduler = cs;
rmContext = TestUtils.GetMockRMContext();
spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Org.Mockito.Mockito.When(spyRMContext.GetScheduler()).ThenReturn(scheduler);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
ReservationSystemTestUtil.SetupQueueConfiguration(csConf);
cs.SetConf(csConf);
csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(minAlloc
);
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(maxAlloc
);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(100 * 16 * Gb, 100 * 32));
Org.Mockito.Mockito.When(scheduler.GetClusterResource()).ThenReturn(Resources.CreateResource
(125 * Gb, 125));
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(new DefaultResourceCalculator
());
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(csConf);
containerTokenSecretManager.RollMasterKey();
Org.Mockito.Mockito.When(csContext.GetContainerTokenSecretManager()).ThenReturn(containerTokenSecretManager
);
cs.SetRMContext(spyRMContext);
cs.Init(csConf);
cs.Start();
SetupPlanFollower();
}
protected internal virtual void CreateApplicationWithAMResource(ApplicationAttemptId
attId, string queue, string user, Resource amResource)
{
RMContext rmContext = resourceManager.GetRMContext();
RMApp rmApp = new RMAppImpl(attId.GetApplicationId(), rmContext, conf, null, null
, null, ApplicationSubmissionContext.NewInstance(null, null, null, null, null, false
, false, 0, amResource, null), null, null, 0, null, null, null);
rmContext.GetRMApps()[attId.GetApplicationId()] = rmApp;
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attId.GetApplicationId
(), queue, user);
scheduler.Handle(appAddedEvent);
AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent
(attId, false);
scheduler.Handle(attempAddedEvent);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual FinishApplicationMasterResponse FinishApplicationMaster(FinishApplicationMasterRequest
request)
{
ApplicationAttemptId applicationAttemptId = AuthorizeRequest().GetApplicationAttemptId
();
ApplicationId appId = applicationAttemptId.GetApplicationId();
RMApp rmApp = rmContext.GetRMApps()[applicationAttemptId.GetApplicationId()];
// checking whether the app exits in RMStateStore at first not to throw
// ApplicationDoesNotExistInCacheException before and after
// RM work-preserving restart.
if (rmApp.IsAppFinalStateStored())
{
Log.Info(rmApp.GetApplicationId() + " unregistered successfully. ");
return(FinishApplicationMasterResponse.NewInstance(true));
}
ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId
];
if (Lock == null)
{
ThrowApplicationDoesNotExistInCacheException(applicationAttemptId);
}
// Allow only one thread in AM to do finishApp at a time.
lock (Lock)
{
if (!HasApplicationMasterRegistered(applicationAttemptId))
{
string message = "Application Master is trying to unregister before registering for: "
+ appId;
Log.Error(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appId].GetUser(), RMAuditLogger.AuditConstants
.UnregisterAm, string.Empty, "ApplicationMasterService", message, appId, applicationAttemptId
);
throw new ApplicationMasterNotRegisteredException(message);
}
this.amLivelinessMonitor.ReceivedPing(applicationAttemptId);
rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptUnregistrationEvent
(applicationAttemptId, request.GetTrackingUrl(), request.GetFinalApplicationStatus
(), request.GetDiagnostics()));
// For UnmanagedAMs, return true so they don't retry
return(FinishApplicationMasterResponse.NewInstance(rmApp.GetApplicationSubmissionContext
().GetUnmanagedAM()));
}
}
private RMAppImpl CreateMockRMApp(ApplicationAttemptId attemptId, RMContext context
)
{
RMAppImpl app = Org.Mockito.Mockito.Mock <RMAppImpl>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(attemptId.GetApplicationId
());
RMAppAttemptImpl attempt = Org.Mockito.Mockito.Mock <RMAppAttemptImpl>();
Org.Mockito.Mockito.When(attempt.GetAppAttemptId()).ThenReturn(attemptId);
RMAppAttemptMetrics attemptMetric = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics>
();
Org.Mockito.Mockito.When(attempt.GetRMAppAttemptMetrics()).ThenReturn(attemptMetric
);
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(attempt);
context.GetRMApps().PutIfAbsent(attemptId.GetApplicationId(), app);
return(app);
}
public virtual AggregateAppResourceUsage GetAggregateAppResourceUsage()
{
long memorySeconds = finishedMemorySeconds.Get();
long vcoreSeconds = finishedVcoreSeconds.Get();
// Only add in the running containers if this is the active attempt.
RMAppAttempt currentAttempt = rmContext.GetRMApps()[attemptId.GetApplicationId()]
.GetCurrentAppAttempt();
if (currentAttempt.GetAppAttemptId().Equals(attemptId))
{
ApplicationResourceUsageReport appResUsageReport = rmContext.GetScheduler().GetAppResourceUsageReport
(attemptId);
if (appResUsageReport != null)
{
memorySeconds += appResUsageReport.GetMemorySeconds();
vcoreSeconds += appResUsageReport.GetVcoreSeconds();
}
}
return(new AggregateAppResourceUsage(memorySeconds, vcoreSeconds));
}
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, string user, Queue
queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
: base(applicationAttemptId, user, queue, activeUsersManager, rmContext)
{
RMApp rmApp = rmContext.GetRMApps()[GetApplicationId()];
Resource amResource;
if (rmApp == null || rmApp.GetAMResourceRequest() == null)
{
//the rmApp may be undefined (the resource manager checks for this too)
//and unmanaged applications do not provide an amResource request
//in these cases, provide a default using the scheduler
amResource = rmContext.GetScheduler().GetMinimumResourceCapability();
}
else
{
amResource = rmApp.GetAMResourceRequest().GetCapability();
}
SetAMResource(amResource);
}
public virtual void TestAppBlockRenderWithNullCurrentAppAttempt()
{
ApplicationId AppId = ApplicationId.NewInstance(1234L, 0);
Injector injector;
// init app
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetTrackingUrl()).ThenReturn("http://host:123");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Failed);
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(AppId);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("Type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("user");
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Name");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("queue");
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder());
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.CreateApplicationState()).ThenReturn(YarnApplicationState
.Failed);
RMAppMetrics appMetrics = new RMAppMetrics(Resource.NewInstance(0, 0), 0, 0, 0, 0
);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
RMContext rmContext = TestRMWebApp.MockRMContext(15, 1, 2, 8);
rmContext.GetRMApps()[AppId] = app;
injector = WebAppTests.CreateMockInjector <RMContext>(rmContext, new _Module_76(rmContext
));
AppBlock instance = injector.GetInstance <AppBlock>();
instance.Set(YarnWebParams.ApplicationId, AppId.ToString());
instance.Render();
}
public virtual void SetUp()
{
conf = CreateConfiguration();
ReservationSystemTestUtil.SetupFSAllocationFile(AllocFile);
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
// Setup
rmContext = TestUtils.GetMockRMContext();
spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
fs = ReservationSystemTestUtil.SetupFairScheduler(testUtil, spyRMContext, conf, 125
);
scheduler = fs;
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
ReservationSystemTestUtil.SetupFSAllocationFile(AllocFile);
SetupPlanFollower();
}
internal virtual void HandleNMContainerStatus(NMContainerStatus containerStatus,
NodeId nodeId)
{
ApplicationAttemptId appAttemptId = containerStatus.GetContainerId().GetApplicationAttemptId
();
RMApp rmApp = rmContext.GetRMApps()[appAttemptId.GetApplicationId()];
if (rmApp == null)
{
Log.Error("Received finished container : " + containerStatus.GetContainerId() + " for unknown application "
+ appAttemptId.GetApplicationId() + " Skipping.");
return;
}
if (rmApp.GetApplicationSubmissionContext().GetUnmanagedAM())
{
if (Log.IsDebugEnabled())
{
Log.Debug("Ignoring container completion status for unmanaged AM " + rmApp.GetApplicationId
());
}
return;
}
RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptId);
Container masterContainer = rmAppAttempt.GetMasterContainer();
if (masterContainer.GetId().Equals(containerStatus.GetContainerId()) && containerStatus
.GetContainerState() == ContainerState.Complete)
{
ContainerStatus status = ContainerStatus.NewInstance(containerStatus.GetContainerId
(), containerStatus.GetContainerState(), containerStatus.GetDiagnostics(), containerStatus
.GetContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt = new RMAppAttemptContainerFinishedEvent(appAttemptId
, status, nodeId);
rmContext.GetDispatcher().GetEventHandler().Handle(evt);
}
}
public virtual void TestRMAppRetireSomeDifferentStates()
{
long now = Runtime.CurrentTimeMillis();
// these parameters don't matter, override applications below
RMContext rmContext = MockRMContext(10, now - 20000);
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 2);
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 2);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
// clear out applications map
rmContext.GetRMApps().Clear();
NUnit.Framework.Assert.AreEqual("map isn't empty", 0, rmContext.GetRMApps().Count
);
// 6 applications are in final state, 4 are not in final state.
// / set with various finished states
RMApp app = new MockRMApp(0, now - 20000, RMAppState.Killed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(1, now - 200000, RMAppState.Failed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(2, now - 30000, RMAppState.Finished);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(3, now - 20000, RMAppState.Running);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(4, now - 20000, RMAppState.New);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
// make sure it doesn't expire these since still running
app = new MockRMApp(5, now - 10001, RMAppState.Killed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(6, now - 30000, RMAppState.Accepted);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(7, now - 20000, RMAppState.Submitted);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(8, now - 10001, RMAppState.Failed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(9, now - 20000, RMAppState.Failed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext.
GetRMApps().Count);
// add them to completed apps list
AddToCompletedApps(appMonitor, rmContext);
// shouldn't have to many apps
appMonitor.CheckAppNumCompletedLimit();
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, 6, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
2, appMonitor.GetCompletedAppsListSize());
// 6 applications in final state, 4 of them are removed
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(4
)).RemoveApplication(Matchers.IsA <RMApp>());
}
public virtual void TestHeadroom()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25);
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
// Say cluster has 100 nodes of 16G each
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(100 * 16 * Gb);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils
.spyHook);
// Manipulate queue 'a'
LeafQueue queue = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]);
string host_0 = "host_0";
string rack_0 = "rack_0";
FiCaSchedulerNode node_0 = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb);
string user_0 = "user_0";
string user_1 = "user_1";
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
RMContext rmContext = TestUtils.GetMockRMContext();
RMContext spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>();
Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource
(0, 0);
Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource
);
Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest
);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Priority priority_1 = TestUtils.CreateMockPriority(1);
// Submit first application with some resource-requests from user_0,
// and check headroom
ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0,
0);
FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_0, user_0);
IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>();
app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_0.UpdateResourceRequests(app_0_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource
(10 * 16 * Gb, 1);
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
// Submit second application from user_0, check headroom
ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1,
0);
FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_1, user_0);
IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>();
app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_1.UpdateResourceRequests(app_0_1_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
// no change
// Submit first application from user_1, check for new headroom
ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2,
0);
FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_1_0, user_1);
IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>();
app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_1_0.UpdateResourceRequests(app_1_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.CreateResource(90 * 16 * Gb);
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
}
public virtual void LogApplicationSummary(ApplicationId appId)
{
RMAppManager.ApplicationSummary.LogAppSummary(rmContext.GetRMApps()[appId]);
}
public virtual void TestReleaseWhileRunning()
{
DrainDispatcher drainDispatcher = new DrainDispatcher();
EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
<EventHandler>();
EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();
drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
drainDispatcher.Register(typeof(RMNodeEventType), generic);
drainDispatcher.Init(new YarnConfiguration());
drainDispatcher.Start();
NodeId nodeId = BuilderUtils.NewNodeId("host", 3425);
ApplicationId appId = BuilderUtils.NewApplicationId(1, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
Resource resource = BuilderUtils.NewResource(512, 1);
Priority priority = BuilderUtils.NewPriority(5);
Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
resource, priority, null);
ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()];
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
, rmContext);
NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
), Matchers.AnyLong());
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
));
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
);
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
, rmContainer.GetLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
Org.Mockito.Mockito.Reset(appAttemptEventHandler);
ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
, SchedulerUtils.ReleasedContainer);
rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
.Released));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
);
NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState
());
Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer>
(), Matchers.AnyLong());
ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass
<RMAppAttemptContainerFinishedEvent>();
Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture());
RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue();
NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId());
NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus());
NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent.
GetType());
// In RELEASED state. A FINIHSED event may come in.
rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus
(containerId, "FinishedContainer"), RMContainerEventType.Finished));
NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
);
}