public virtual void TestDelegationTokenAuth() { string token = GetDelegationToken("test"); ApplicationSubmissionContextInfo app = new ApplicationSubmissionContextInfo(); string appid = "application_123_0"; app.SetApplicationId(appid); string requestBody = GetMarshalledAppInfo(app); Uri url = new Uri("http://localhost:8088/ws/v1/cluster/apps"); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); SetupConn(conn, "POST", "application/xml", requestBody); // this should fail with unauthorized because only // auth is kerberos or delegation token try { conn.GetInputStream(); NUnit.Framework.Assert.Fail("we should not be here"); } catch (IOException) { NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Unauthorized.GetStatusCode( ), conn.GetResponseCode()); } conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestProperty(delegationTokenHeader, token); SetupConn(conn, "POST", MediaType.ApplicationXml, requestBody); // this should not fail try { conn.GetInputStream(); } catch (IOException) { InputStream errorStream = conn.GetErrorStream(); string error = string.Empty; BufferedReader reader = null; reader = new BufferedReader(new InputStreamReader(errorStream, "UTF8")); for (string line; (line = reader.ReadLine()) != null;) { error += line; } reader.Close(); errorStream.Close(); NUnit.Framework.Assert.Fail("Response " + conn.GetResponseCode() + "; " + error); } bool appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId (appid)); NUnit.Framework.Assert.IsTrue(appExists); RMApp actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid )]; string owner = actualApp.GetUser(); NUnit.Framework.Assert.AreEqual("client", owner); }
public virtual void TestZKRootPathAcls() { HAServiceProtocol.StateChangeRequestInfo req = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); string rootPath = YarnConfiguration.DefaultZkRmStateStoreParentPath + "/" + ZKRMStateStore .RootZnodeName; // Start RM with HA enabled Configuration conf = CreateHARMConf("rm1,rm2", "rm1", 1234); conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); ResourceManager rm = new MockRM(conf); rm.Start(); rm.GetRMContext().GetRMAdminService().TransitionToActive(req); Stat stat = new Stat(); IList <ACL> acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries (rootPath, stat); NUnit.Framework.Assert.AreEqual(acls.Count, 2); // CREATE and DELETE permissions for root node based on RM ID VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls ); VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms .Delete), acls); rm.Close(); // Now start RM with HA disabled. NoAuth Exception should not be thrown. conf.SetBoolean(YarnConfiguration.RmHaEnabled, false); rm = new MockRM(conf); rm.Start(); rm.GetRMContext().GetRMAdminService().TransitionToActive(req); acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath , stat); NUnit.Framework.Assert.AreEqual(acls.Count, 1); VerifyZKACL("world", "anyone", ZooDefs.Perms.All, acls); rm.Close(); // Start RM with HA enabled. conf.SetBoolean(YarnConfiguration.RmHaEnabled, true); rm = new MockRM(conf); rm.Start(); rm.GetRMContext().GetRMAdminService().TransitionToActive(req); acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath , stat); NUnit.Framework.Assert.AreEqual(acls.Count, 2); VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls ); VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms .Delete), acls); rm.Close(); }
// Test regular RM restart/failover, new RM should not count // AM failure towards the max-retry-account and should be able to // re-launch the AM. /// <exception cref="System.Exception"/> public virtual void TestRMRestartOrFailoverNotCountedForAMFailures() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler )); conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true); conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false); conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName); // explicitly set max-am-retry count as 1. conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService()); nm1.RegisterNode(); RMApp app1 = rm1.SubmitApp(200); // AM should be restarted even though max-am-attempt is 1. MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt()); // Restart rm. MockRM rm2 = new MockRM(conf, memStore); rm2.Start(); ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId ()]; // re-register the NM nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus >(); status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager); status.SetContainerId(attempt1.GetMasterContainer().GetId()); status.SetContainerState(ContainerState.Complete); status.SetDiagnostics(string.Empty); nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null); rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState .GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus()); // Will automatically start a new AppAttempt in rm2 rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1); MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2); RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt (); NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt( am2.GetApplicationAttemptId()).GetAMContainerExitStatus()); rm1.Stop(); rm2.Stop(); }
// Test RM restarts after AM container is preempted, new RM should not count // AM preemption failure towards the max-retry-account and should be able to // re-launch the AM. /// <exception cref="System.Exception"/> public virtual void TestPreemptedAMRestartOnRMRestart() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler )); conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true); conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false); conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName); // explicitly set max-am-retry count as 1. conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService()); nm1.RegisterNode(); RMApp app1 = rm1.SubmitApp(200); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler(); ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId( ), 1); // Forcibly preempt the am container; scheduler.KillContainer(scheduler.GetRMContainer(amContainer)); am1.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry()); rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); // state store has 1 attempt stored. ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId ()]; NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount()); // attempt stored has the preempted container exit status. NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt (am1.GetApplicationAttemptId()).GetAMContainerExitStatus()); // Restart rm. MockRM rm2 = new MockRM(conf, memStore); nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); nm1.RegisterNode(); rm2.Start(); // Restarted RM should re-launch the am. MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1); MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2); RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt (); NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt( am2.GetApplicationAttemptId()).GetAMContainerExitStatus()); rm1.Stop(); rm2.Stop(); }
/// <exception cref="System.Exception"/> private void TestAnonymousSimpleUser() { ApplicationSubmissionContextInfo app = new ApplicationSubmissionContextInfo(); string appid = "application_123_0"; app.SetApplicationId(appid); string requestBody = TestRMWebServicesDelegationTokenAuthentication.GetMarshalledAppInfo (app); Uri url = new Uri("http://localhost:8088/ws/v1/cluster/apps"); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); TestRMWebServicesDelegationTokenAuthentication.SetupConn(conn, "POST", "application/xml" , requestBody); conn.GetInputStream(); NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Accepted.GetStatusCode(), conn .GetResponseCode()); bool appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId (appid)); NUnit.Framework.Assert.IsTrue(appExists); RMApp actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid )]; string owner = actualApp.GetUser(); NUnit.Framework.Assert.AreEqual(rm.GetConfig().Get(CommonConfigurationKeys.HadoopHttpStaticUser , CommonConfigurationKeys.DefaultHadoopHttpStaticUser), owner); appid = "application_123_1"; app.SetApplicationId(appid); requestBody = TestRMWebServicesDelegationTokenAuthentication.GetMarshalledAppInfo (app); url = new Uri("http://localhost:8088/ws/v1/cluster/apps?user.name=client"); conn = (HttpURLConnection)url.OpenConnection(); TestRMWebServicesDelegationTokenAuthentication.SetupConn(conn, "POST", MediaType. ApplicationXml, requestBody); conn.GetInputStream(); appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId (appid)); NUnit.Framework.Assert.IsTrue(appExists); actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid)]; owner = actualApp.GetUser(); NUnit.Framework.Assert.AreEqual("client", owner); }
public virtual void TestConcurrentAccess() { conf.Set(FairSchedulerConfiguration.AssignMultiple, "false"); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); string queueName = "root.queue1"; FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true ); ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1); RMContext rmContext = resourceManager.GetRMContext(); FSAppAttempt app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable , null, rmContext); // this needs to be in sync with the number of runnables declared below int testThreads = 2; IList <Runnable> runnables = new AList <Runnable>(); // add applications to modify the list runnables.AddItem(new _Runnable_257(schedulable, app)); // iterate over the list a couple of times in a different thread runnables.AddItem(new _Runnable_267(schedulable)); IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception >()); ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads); try { CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads); CountDownLatch startBlocker = new CountDownLatch(1); CountDownLatch allDone = new CountDownLatch(testThreads); foreach (Runnable submittedTestRunnable in runnables) { threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable , exceptions, allDone)); } // wait until all threads are ready allExecutorThreadsReady.Await(); // start all test runners startBlocker.CountDown(); int testTimeout = 2; NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds" , allDone.Await(testTimeout, TimeUnit.Seconds)); } catch (Exception ie) { exceptions.AddItem(ie); } finally { threadPool.ShutdownNow(); } NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions .IsEmpty()); }
/// <exception cref="System.Exception"/> private AllocateResponse Allocate(ApplicationAttemptId attemptId, AllocateRequest req) { UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm.GetRMContext ().GetRMApps()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken (); ugi.AddTokenIdentifier(token.DecodeIdentifier()); return(ugi.DoAs(new _PrivilegedExceptionAction_67(this, req))); }
public virtual void TestNodesDefaultWithUnHealthyNode() { WebResource r = Resource(); MockNM nm1 = rm.RegisterNode("h1:1234", 5120); MockNM nm2 = rm.RegisterNode("h2:1235", 5121); rm.SendNodeStarted(nm1); rm.NMwaitForState(nm1.GetNodeId(), NodeState.Running); rm.NMwaitForState(nm2.GetNodeId(), NodeState.New); MockNM nm3 = rm.RegisterNode("h3:1236", 5122); rm.NMwaitForState(nm3.GetNodeId(), NodeState.New); rm.SendNodeStarted(nm3); rm.NMwaitForState(nm3.GetNodeId(), NodeState.Running); RMNodeImpl node = (RMNodeImpl)rm.GetRMContext().GetRMNodes()[nm3.GetNodeId()]; NodeHealthStatus nodeHealth = NodeHealthStatus.NewInstance(false, "test health report" , Runtime.CurrentTimeMillis()); node.Handle(new RMNodeStatusEvent(nm3.GetNodeId(), nodeHealth, new AList <ContainerStatus >(), null, null)); rm.NMwaitForState(nm3.GetNodeId(), NodeState.Unhealthy); ClientResponse response = r.Path("ws").Path("v1").Path("cluster").Path("nodes").Accept (MediaType.ApplicationJson).Get <ClientResponse>(); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType() ); JSONObject json = response.GetEntity <JSONObject>(); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length()); JSONObject nodes = json.GetJSONObject("nodes"); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.Length() ); JSONArray nodeArray = nodes.GetJSONArray("node"); // 3 nodes, including the unhealthy node and the new node. NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, nodeArray.Length ()); }
/// <exception cref="System.Exception"/> public virtual void TestStoreOnlyAMContainerMetrics() { Configuration conf = new Configuration(); conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); conf.SetBoolean(YarnConfiguration.ApplicationHistorySaveNonAmContainerMetaInfo, false ); MockRM rm1 = new MockRM(conf); SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher >(); rm1.GetRMContext().SetSystemMetricsPublisher(publisher); rm1.Start(); MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000); RMApp app1 = rm1.SubmitApp(1024); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Running); // request a container. am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>()); ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 2); rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated); am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers (); rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired); nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running); nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Complete); nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete); rm1.WaitForState(nm1, containerId2, RMContainerState.Completed); rm1.Stop(); // RMContainer should be publishing system metrics only for AM container. Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Times(1)).ContainerCreated (Matchers.Any <RMContainer>(), Matchers.AnyLong()); Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Times(1)).ContainerFinished (Matchers.Any <RMContainer>(), Matchers.AnyLong()); }
public virtual void TestBlackListNodes() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); string host = "127.0.0.1"; RMNode node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host); fs.Handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); CreateMockRMApp(appAttemptId, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); fs.Handle(attemptEvent); // Verify the blacklist can be updated independent of requesting containers fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null); NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host)); NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); rm.Stop(); }
public virtual void TestRefreshQueuesWithReservations() { CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler(); // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a1); a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify // capacity ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a2); cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f)); // Verify all allocations match tcs.CheckQueueCapacities(cs, ACapacity, BCapacity); // Reinitialize and verify all dynamic queued survived CapacitySchedulerConfiguration conf = cs.GetConfiguration(); conf.SetCapacity(A, 80f); conf.SetCapacity(B, 20f); cs.Reinitialize(conf, rm.GetRMContext()); tcs.CheckQueueCapacities(cs, 80f, 20f); }
/// <exception cref="System.Exception"/> public virtual void TestExcessReservationThanNodeManagerCapacity() { MockRM rm = new MockRM(conf); rm.Start(); // Register node1 MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 2 * Gb, 4); MockNM nm2 = rm.RegisterNode("127.0.0.1:2234", 3 * Gb, 4); nm1.NodeHeartbeat(true); nm2.NodeHeartbeat(true); // wait.. int waitCount = 20; int size = rm.GetRMContext().GetRMNodes().Count; while ((size = rm.GetRMContext().GetRMNodes().Count) != 2 && waitCount-- > 0) { Log.Info("Waiting for node managers to register : " + size); Sharpen.Thread.Sleep(100); } NUnit.Framework.Assert.AreEqual(2, rm.GetRMContext().GetRMNodes().Count); // Submit an application RMApp app1 = rm.SubmitApp(128); // kick the scheduling nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); Log.Info("sending container requests "); am1.AddRequests(new string[] { "*" }, 2 * Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler nm1.NodeHeartbeat(true); int waitCounter = 20; Log.Info("heartbeating nm1"); while (alloc1Response.GetAllocatedContainers().Count < 1 && waitCounter-- > 0) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(500); alloc1Response = am1.Schedule(); } Log.Info("received container : " + alloc1Response.GetAllocatedContainers().Count); // No container should be allocated. // Internally it should not been reserved. NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count == 0); Log.Info("heartbeating nm2"); waitCounter = 20; nm2.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1 && waitCounter-- > 0) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(500); alloc1Response = am1.Schedule(); } Log.Info("received container : " + alloc1Response.GetAllocatedContainers().Count); NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count == 1); rm.Stop(); }