public virtual void SetUp()
{
Configuration conf = new Configuration();
// Dispatcher that processes events inline
Dispatcher dispatcher = new InlineDispatcher();
RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null
, null, null, null);
dispatcher.Register(typeof(SchedulerEventType), new InlineDispatcher.EmptyEventHandler
());
dispatcher.Register(typeof(RMNodeEventType), new ResourceManager.NodeEventDispatcher
(context));
NMLivelinessMonitor nmLivelinessMonitor = new TestNMExpiry.TestNmLivelinessMonitor
(this, dispatcher);
nmLivelinessMonitor.Init(conf);
nmLivelinessMonitor.Start();
NodesListManager nodesListManager = new NodesListManager(context);
nodesListManager.Init(conf);
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.Start();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.Start();
resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor
, containerTokenSecretManager, nmTokenSecretManager);
resourceTrackerService.Init(conf);
resourceTrackerService.Start();
}
/// <exception cref="System.Exception"/>
public virtual void TestAppAttemptMetrics()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
FifoScheduler scheduler = new FifoScheduler();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
null, null, null, scheduler);
((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
Configuration conf = new Configuration();
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(conf, rmContext);
QueueMetrics metrics = scheduler.GetRootQueueMetrics();
int beforeAppsSubmitted = metrics.GetAppsSubmitted();
ApplicationId appId = BuilderUtils.NewApplicationId(200, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
scheduler.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
scheduler.Handle(attemptEvent);
appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2);
SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
scheduler.Handle(attemptEvent2);
int afterAppsSubmitted = metrics.GetAppsSubmitted();
NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted);
scheduler.Stop();
}
public virtual void Setup()
{
dispatcher = new InlineDispatcher();
++startCount;
conf = new JobConf();
taskAttemptListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
jobToken = (Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier>)Org.Mockito.Mockito.Mock
<Org.Apache.Hadoop.Security.Token.Token>();
remoteJobConfFile = Org.Mockito.Mockito.Mock <Path>();
credentials = null;
clock = new SystemClock();
metrics = Org.Mockito.Mockito.Mock <MRAppMetrics>();
dataLocations = new string[1];
appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
jobId = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobId>();
jobId.SetId(1);
jobId.SetAppId(appId);
appContext = Org.Mockito.Mockito.Mock <AppContext>();
taskSplitMetaInfo = Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>();
Org.Mockito.Mockito.When(taskSplitMetaInfo.GetLocations()).ThenReturn(dataLocations
);
taskAttempts = new AList <TestTaskImpl.MockTaskAttemptImpl>();
}
public virtual void SetUp()
{
InlineDispatcher rmDispatcher = new InlineDispatcher();
rmContext = new RMContextImpl(rmDispatcher, null, null, null, Org.Mockito.Mockito.Mock
<DelegationTokenRenewer>(), null, null, null, null, null);
NodesListManager nodesListManager = Org.Mockito.Mockito.Mock <NodesListManager>();
HostsFileReader reader = Org.Mockito.Mockito.Mock <HostsFileReader>();
Org.Mockito.Mockito.When(nodesListManager.GetHostsReader()).ThenReturn(reader);
((RMContextImpl)rmContext).SetNodesListManager(nodesListManager);
scheduler = Org.Mockito.Mockito.Mock <YarnScheduler>();
Org.Mockito.Mockito.DoAnswer(new _Answer_115(this)).When(scheduler).Handle(Matchers.Any
<SchedulerEvent>());
rmDispatcher.Register(typeof(SchedulerEventType), new TestRMNodeTransitions.TestSchedulerEventDispatcher
(this));
rmDispatcher.Register(typeof(NodesListManagerEventType), new TestRMNodeTransitions.TestNodeListManagerEventDispatcher
(this));
NodeId nodeId = BuilderUtils.NewNodeId("localhost", 0);
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
nodesListManagerEvent = null;
}
public virtual void SetUp()
{
Configuration conf = new Configuration();
// Dispatcher that processes events inline
Dispatcher dispatcher = new InlineDispatcher();
dispatcher.Register(typeof(SchedulerEventType), new _EventHandler_65());
// ignore
RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, new
RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null, null
);
dispatcher.Register(typeof(RMNodeEventType), new ResourceManager.NodeEventDispatcher
(context));
NodesListManager nodesListManager = new NodesListManager(context);
nodesListManager.Init(conf);
context.GetContainerTokenSecretManager().RollMasterKey();
context.GetNMTokenSecretManager().RollMasterKey();
resourceTrackerService = new ResourceTrackerService(context, nodesListManager, new
NMLivelinessMonitor(dispatcher), context.GetContainerTokenSecretManager(), context
.GetNMTokenSecretManager());
resourceTrackerService.Init(conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestUpdateResourceOnNode()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new _FifoScheduler_275(this);
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(2048, 4), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(scheduler.GetType(), "getNodes"
);
IDictionary <NodeId, FiCaSchedulerNode> schedulerNodes = (IDictionary <NodeId, FiCaSchedulerNode
>)method.Invoke(scheduler);
NUnit.Framework.Assert.AreEqual(schedulerNodes.Values.Count, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = Resources.CreateResource
(1024, 4);
NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent
(node0, ResourceOption.NewInstance(newResource, RMNode.OverCommitTimeoutMillisDefault
));
scheduler.Handle(node0ResourceUpdate);
// SchedulerNode's total resource and available resource are changed.
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetTotalResource
().GetMemory(), 1024);
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetAvailableResource
().GetMemory(), 1024);
QueueInfo queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(0.0f, queueInfo.GetCurrentCapacity(), 0.0f);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 1024;
int priority = 1;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, 1);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, 1);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, 1);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
// Before the node update event, there are one local request
NUnit.Framework.Assert.AreEqual(1, nodeLocal.GetNumContainers());
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Now schedule.
scheduler.Handle(node0Update);
// After the node update event, check no local request
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
// Also check that one container was scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(1, info.GetLiveContainers().Count);
// And check the default Queue now is full.
queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(1.0f, queueInfo.GetCurrentCapacity(), 0.0f);
}
/// <exception cref="System.Exception"/>
public virtual void TestNodeLocalAssignment()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024 * 64), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 64;
int nConts = 3;
int priority = 20;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, nConts);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, nConts);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, nConts);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Before the node update event, there are 3 local requests outstanding
NUnit.Framework.Assert.AreEqual(3, nodeLocal.GetNumContainers());
scheduler.Handle(node0Update);
// After the node update event, check that there are no more local requests
// outstanding
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
//Also check that the containers were scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(3, info.GetLiveContainers().Count);
scheduler.Stop();
}
