public virtual void SetUp() { Configuration conf = new Configuration(); // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null , null, null, null); dispatcher.Register(typeof(SchedulerEventType), new InlineDispatcher.EmptyEventHandler ()); dispatcher.Register(typeof(RMNodeEventType), new ResourceManager.NodeEventDispatcher (context)); NMLivelinessMonitor nmLivelinessMonitor = new TestNMExpiry.TestNmLivelinessMonitor (this, dispatcher); nmLivelinessMonitor.Init(conf); nmLivelinessMonitor.Start(); NodesListManager nodesListManager = new NodesListManager(context); nodesListManager.Init(conf); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager (conf); containerTokenSecretManager.Start(); NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf ); nmTokenSecretManager.Start(); resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor , containerTokenSecretManager, nmTokenSecretManager); resourceTrackerService.Init(conf); resourceTrackerService.Start(); }
/// <exception cref="System.Exception"/> public virtual void TestAppAttemptMetrics() { AsyncDispatcher dispatcher = new InlineDispatcher(); FifoScheduler scheduler = new FifoScheduler(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, scheduler); ((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); Configuration conf = new Configuration(); scheduler.SetRMContext(rmContext); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(conf, rmContext); QueueMetrics metrics = scheduler.GetRootQueueMetrics(); int beforeAppsSubmitted = metrics.GetAppsSubmitted(); ApplicationId appId = BuilderUtils.NewApplicationId(200, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user"); scheduler.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); scheduler.Handle(attemptEvent); appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); scheduler.Handle(attemptEvent2); int afterAppsSubmitted = metrics.GetAppsSubmitted(); NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted); scheduler.Stop(); }
public virtual void Setup() { dispatcher = new InlineDispatcher(); ++startCount; conf = new JobConf(); taskAttemptListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>(); jobToken = (Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier>)Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token>(); remoteJobConfFile = Org.Mockito.Mockito.Mock <Path>(); credentials = null; clock = new SystemClock(); metrics = Org.Mockito.Mockito.Mock <MRAppMetrics>(); dataLocations = new string[1]; appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1); jobId = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobId>(); jobId.SetId(1); jobId.SetAppId(appId); appContext = Org.Mockito.Mockito.Mock <AppContext>(); taskSplitMetaInfo = Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(); Org.Mockito.Mockito.When(taskSplitMetaInfo.GetLocations()).ThenReturn(dataLocations ); taskAttempts = new AList <TestTaskImpl.MockTaskAttemptImpl>(); }
public virtual void SetUp() { InlineDispatcher rmDispatcher = new InlineDispatcher(); rmContext = new RMContextImpl(rmDispatcher, null, null, null, Org.Mockito.Mockito.Mock <DelegationTokenRenewer>(), null, null, null, null, null); NodesListManager nodesListManager = Org.Mockito.Mockito.Mock <NodesListManager>(); HostsFileReader reader = Org.Mockito.Mockito.Mock <HostsFileReader>(); Org.Mockito.Mockito.When(nodesListManager.GetHostsReader()).ThenReturn(reader); ((RMContextImpl)rmContext).SetNodesListManager(nodesListManager); scheduler = Org.Mockito.Mockito.Mock <YarnScheduler>(); Org.Mockito.Mockito.DoAnswer(new _Answer_115(this)).When(scheduler).Handle(Matchers.Any <SchedulerEvent>()); rmDispatcher.Register(typeof(SchedulerEventType), new TestRMNodeTransitions.TestSchedulerEventDispatcher (this)); rmDispatcher.Register(typeof(NodesListManagerEventType), new TestRMNodeTransitions.TestNodeListManagerEventDispatcher (this)); NodeId nodeId = BuilderUtils.NewNodeId("localhost", 0); node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null); nodesListManagerEvent = null; }
public virtual void SetUp() { Configuration conf = new Configuration(); // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); dispatcher.Register(typeof(SchedulerEventType), new _EventHandler_65()); // ignore RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null, null ); dispatcher.Register(typeof(RMNodeEventType), new ResourceManager.NodeEventDispatcher (context)); NodesListManager nodesListManager = new NodesListManager(context); nodesListManager.Init(conf); context.GetContainerTokenSecretManager().RollMasterKey(); context.GetNMTokenSecretManager().RollMasterKey(); resourceTrackerService = new ResourceTrackerService(context, nodesListManager, new NMLivelinessMonitor(dispatcher), context.GetContainerTokenSecretManager(), context .GetNMTokenSecretManager()); resourceTrackerService.Init(conf); }
/// <exception cref="System.Exception"/> public virtual void TestUpdateResourceOnNode() { AsyncDispatcher dispatcher = new InlineDispatcher(); Configuration conf = new Configuration(); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager (conf); containerTokenSecretManager.RollMasterKey(); NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf ); nmTokenSecretManager.RollMasterKey(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); FifoScheduler scheduler = new _FifoScheduler_275(this); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler); rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >()); ((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration()); scheduler.SetRMContext(rmContext); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(2048, 4), 1, "127.0.0.1" ); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0); scheduler.Handle(nodeEvent1); MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(scheduler.GetType(), "getNodes" ); IDictionary <NodeId, FiCaSchedulerNode> schedulerNodes = (IDictionary <NodeId, FiCaSchedulerNode >)method.Invoke(scheduler); NUnit.Framework.Assert.AreEqual(schedulerNodes.Values.Count, 1); Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = Resources.CreateResource (1024, 4); NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent (node0, ResourceOption.NewInstance(newResource, RMNode.OverCommitTimeoutMillisDefault )); scheduler.Handle(node0ResourceUpdate); // SchedulerNode's total resource and available resource are changed. NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetTotalResource ().GetMemory(), 1024); NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetAvailableResource ().GetMemory(), 1024); QueueInfo queueInfo = scheduler.GetQueueInfo(null, false, false); NUnit.Framework.Assert.AreEqual(0.0f, queueInfo.GetCurrentCapacity(), 0.0f); int _appId = 1; int _appAttemptId = 1; ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId); CreateMockRMApp(appAttemptId, rmContext); AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId (), "queue1", "user1"); scheduler.Handle(appEvent); AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId , false); scheduler.Handle(attemptEvent); int memory = 1024; int priority = 1; IList <ResourceRequest> ask = new AList <ResourceRequest>(); ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority , 1); ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority , 1); ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority , 1); ask.AddItem(nodeLocal); ask.AddItem(rackLocal); ask.AddItem(any); scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null); // Before the node update event, there are one local request NUnit.Framework.Assert.AreEqual(1, nodeLocal.GetNumContainers()); NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0); // Now schedule. scheduler.Handle(node0Update); // After the node update event, check no local request NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers()); // Also check that one container was scheduled SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId); NUnit.Framework.Assert.AreEqual(1, info.GetLiveContainers().Count); // And check the default Queue now is full. queueInfo = scheduler.GetQueueInfo(null, false, false); NUnit.Framework.Assert.AreEqual(1.0f, queueInfo.GetCurrentCapacity(), 0.0f); }
/// <exception cref="System.Exception"/> public virtual void TestNodeLocalAssignment() { AsyncDispatcher dispatcher = new InlineDispatcher(); Configuration conf = new Configuration(); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager (conf); containerTokenSecretManager.RollMasterKey(); NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf ); nmTokenSecretManager.RollMasterKey(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); FifoScheduler scheduler = new FifoScheduler(); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler); rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >()); ((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration()); scheduler.SetRMContext(rmContext); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024 * 64), 1, "127.0.0.1" ); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0); scheduler.Handle(nodeEvent1); int _appId = 1; int _appAttemptId = 1; ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId); CreateMockRMApp(appAttemptId, rmContext); AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId (), "queue1", "user1"); scheduler.Handle(appEvent); AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId , false); scheduler.Handle(attemptEvent); int memory = 64; int nConts = 3; int priority = 20; IList <ResourceRequest> ask = new AList <ResourceRequest>(); ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority , nConts); ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority , nConts); ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority , nConts); ask.AddItem(nodeLocal); ask.AddItem(rackLocal); ask.AddItem(any); scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null); NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0); // Before the node update event, there are 3 local requests outstanding NUnit.Framework.Assert.AreEqual(3, nodeLocal.GetNumContainers()); scheduler.Handle(node0Update); // After the node update event, check that there are no more local requests // outstanding NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers()); //Also check that the containers were scheduled SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId); NUnit.Framework.Assert.AreEqual(3, info.GetLiveContainers().Count); scheduler.Stop(); }