/// <exception cref="System.Exception"/> public virtual void TestGetLabelsOnNodesWhenNodeActiveDeactive() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p2"))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1"))); // Active/Deactive a node directly assigned label, should not remove from // node->label map mgr.ActivateNode(ToNodeId("n1:1"), SmallResource); AssertCollectionEquals(mgr.GetNodeLabels()[ToNodeId("n1:1")], ToSet("p1")); mgr.DeactivateNode(ToNodeId("n1:1")); AssertCollectionEquals(mgr.GetNodeLabels()[ToNodeId("n1:1")], ToSet("p1")); // Host will not affected AssertCollectionEquals(mgr.GetNodeLabels()[ToNodeId("n1")], ToSet("p2")); // Active/Deactive a node doesn't directly assigned label, should remove // from node->label map mgr.ActivateNode(ToNodeId("n1:2"), SmallResource); AssertCollectionEquals(mgr.GetNodeLabels()[ToNodeId("n1:2")], ToSet("p2")); mgr.DeactivateNode(ToNodeId("n1:2")); NUnit.Framework.Assert.IsNull(mgr.GetNodeLabels()[ToNodeId("n1:2")]); // Host will not affected too AssertCollectionEquals(mgr.GetNodeLabels()[ToNodeId("n1")], ToSet("p2")); // When we change label on the host after active a node without directly // assigned label, such node will still be removed after deactive // Active/Deactive a node doesn't directly assigned label, should remove // from node->label map mgr.ActivateNode(ToNodeId("n1:2"), SmallResource); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p3"))); AssertCollectionEquals(mgr.GetNodeLabels()[ToNodeId("n1:2")], ToSet("p3")); mgr.DeactivateNode(ToNodeId("n1:2")); NUnit.Framework.Assert.IsNull(mgr.GetNodeLabels()[ToNodeId("n1:2")]); // Host will not affected too AssertCollectionEquals(mgr.GetNodeLabels()[ToNodeId("n1")], ToSet("p3")); }
/// <exception cref="System.Exception"/> public virtual void TestLabelsToNodesOnNodeActiveDeactive() { // Activate a node without assigning any labels mgr.ActivateNode(NodeId.NewInstance("n1", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10, 0)); NUnit.Framework.Assert.IsTrue(mgr.GetLabelsToNodes().IsEmpty()); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels ())); // Add labels and replace labels on node mgr.AddToCluserNodeLabels(ToSet("p1")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); // p1 -> n1, n1:1 NUnit.Framework.Assert.AreEqual(2, mgr.GetLabelsToNodes()["p1"].Count); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels ())); // Activate a node for which host to label mapping exists mgr.ActivateNode(NodeId.NewInstance("n1", 2), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10, 0)); // p1 -> n1, n1:1, n1:2 NUnit.Framework.Assert.AreEqual(3, mgr.GetLabelsToNodes()["p1"].Count); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels ())); // Deactivate a node. n1:1 will be removed from the map mgr.DeactivateNode(NodeId.NewInstance("n1", 1)); // p1 -> n1, n1:2 NUnit.Framework.Assert.AreEqual(2, mgr.GetLabelsToNodes()["p1"].Count); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels ())); }
/// <exception cref="System.Exception"/> public virtual void TestRemoveLabelsFromNode() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2" ), ToSet("p2"), ToNodeId("n3"), ToSet("p3"))); // active one NM to n1:1 mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource); try { mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1"))); NUnit.Framework.Assert.Fail("removeLabelsFromNode should trigger IOException"); } catch (IOException) { } mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1"))); try { mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1"))); } catch (IOException e) { NUnit.Framework.Assert.Fail("IOException from removeLabelsFromNode " + e); } }
/// <exception cref="System.Exception"/> public virtual void TestQuorums() { IDictionary <string, SettableFuture <string> > futures = ImmutableMap.Of("f1", SettableFuture .Create <string>(), "f2", SettableFuture.Create <string>(), "f3", SettableFuture.Create <string>()); QuorumCall <string, string> q = QuorumCall.Create(futures); NUnit.Framework.Assert.AreEqual(0, q.CountResponses()); futures["f1"].Set("first future"); q.WaitFor(1, 0, 0, 100000, "test"); // wait for 1 response q.WaitFor(0, 1, 0, 100000, "test"); // wait for 1 success NUnit.Framework.Assert.AreEqual(1, q.CountResponses()); futures["f2"].SetException(new Exception("error")); NUnit.Framework.Assert.AreEqual(2, q.CountResponses()); futures["f3"].Set("second future"); q.WaitFor(3, 0, 100, 100000, "test"); // wait for 3 responses q.WaitFor(0, 2, 100, 100000, "test"); // 2 successes NUnit.Framework.Assert.AreEqual(3, q.CountResponses()); NUnit.Framework.Assert.AreEqual("f1=first future,f3=second future", Joiner.On("," ).WithKeyValueSeparator("=").Join(new SortedDictionary <string, string>(q.GetResults ()))); try { q.WaitFor(0, 4, 100, 10, "test"); NUnit.Framework.Assert.Fail("Didn't time out waiting for more responses than came back" ); } catch (TimeoutException) { } }
/// <exception cref="System.IO.IOException"/> private void InitializeNodeLabels() { RMContext rmContext = distShellTest.yarnCluster.GetResourceManager(0).GetRMContext (); // Setup node labels RMNodeLabelsManager labelsMgr = rmContext.GetNodeLabelManager(); ICollection <string> labels = new HashSet <string>(); labels.AddItem("x"); labelsMgr.AddToCluserNodeLabels(labels); // Setup queue access to node labels distShellTest.conf.Set("yarn.scheduler.capacity.root.accessible-node-labels", "x" ); distShellTest.conf.Set("yarn.scheduler.capacity.root.accessible-node-labels.x.capacity" , "100"); distShellTest.conf.Set("yarn.scheduler.capacity.root.default.accessible-node-labels" , "x"); distShellTest.conf.Set("yarn.scheduler.capacity.root.default.accessible-node-labels.x.capacity" , "100"); rmContext.GetScheduler().Reinitialize(distShellTest.conf, rmContext); // Fetch node-ids from yarn cluster NodeId[] nodeIds = new NodeId[NumNms]; for (int i = 0; i < NumNms; i++) { NodeManager mgr = distShellTest.yarnCluster.GetNodeManager(i); nodeIds[i] = mgr.GetNMContext().GetNodeId(); } // Set label x to NM[1] labelsMgr.AddLabelsToNode(ImmutableMap.Of(nodeIds[1], labels)); }
/// <exception cref="System.Exception"/> public virtual void TestContainerAllocateWithLabels() { // set node -> label mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y")); mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId .NewInstance("h2", 0), ToSet("y"))); // inject node label manager MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf)); rm1.GetRMContext().SetNodeLabelManager(mgr); rm1.Start(); MockNM nm1 = rm1.RegisterNode("h1:1234", 8000); // label = x MockNM nm2 = rm1.RegisterNode("h2:1234", 8000); // label = y MockNM nm3 = rm1.RegisterNode("h3:1234", 8000); // label = <empty> ContainerId containerId; // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1"); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm3); // request a container. am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x"); containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1"); // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1"); MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm3); // request a container. am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y"); containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2"); // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1"); MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3); // request a container. am3.Allocate("*", 1024, 1, new AList <ContainerId>()); containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3"); rm1.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual void TestTrimLabelsWhenModifyLabelsOnNodes() { mgr.AddToCluserNodeLabels(ToSet(" p1", "p2")); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1 "))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p1")) ); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet(" p2"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p2")) ); mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1"), ToSet(" p2 "))); NUnit.Framework.Assert.IsTrue(mgr.GetNodeLabels().IsEmpty()); }
public CredentialsLogicHandler(TestDelegationTokenRemoteFetcher _enclosing, Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token, string serviceUrl) { this._enclosing = _enclosing; routes = ImmutableMap.Of("/exception", new TestDelegationTokenRemoteFetcher.ExceptionHandler (this), "/cancelDelegationToken", new TestDelegationTokenRemoteFetcher.CancelHandler (this), "/getDelegationToken", new TestDelegationTokenRemoteFetcher.FetchHandler (this), "/renewDelegationToken", new TestDelegationTokenRemoteFetcher.RenewHandler (this)); this.token = token; this.serviceUrl = serviceUrl; }
/// <exception cref="System.Exception"/> public virtual void TestRecoverWithMirror() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.AddToCluserNodeLabels(ToSet("p4")); mgr.AddToCluserNodeLabels(ToSet("p5", "p6")); mgr.ReplaceLabelsOnNode((IDictionary)ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2"), ToSet("p2"))); mgr.ReplaceLabelsOnNode((IDictionary)ImmutableMap.Of(ToNodeId("n3"), ToSet("p3"), ToNodeId("n4"), ToSet("p4"), ToNodeId("n5"), ToSet("p5"), ToNodeId("n6"), ToSet( "p6"), ToNodeId("n7"), ToSet("p6"))); /* * node -> partition p1: n1 p2: n2 p3: n3 p4: n4 p5: n5 p6: n6, n7 */ mgr.RemoveFromClusterNodeLabels(ToSet("p1")); mgr.RemoveFromClusterNodeLabels(Arrays.AsList("p3", "p5")); /* * After removed p2: n2 p4: n4 p6: n6, n7 */ // shutdown mgr and start a new mgr mgr.Stop(); mgr = new TestFileSystemNodeLabelsStore.MockNodeLabelManager(); mgr.Init(conf); mgr.Start(); // check variables NUnit.Framework.Assert.AreEqual(3, mgr.GetClusterNodeLabels().Count); NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().ContainsAll(Arrays.AsList ("p2", "p4", "p6"))); AssertMapContains(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n2"), ToSet("p2" ), ToNodeId("n4"), ToSet("p4"), ToNodeId("n6"), ToSet("p6"), ToNodeId("n7"), ToSet ("p6"))); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), ImmutableMap.Of("p6", ToSet(ToNodeId ("n6"), ToNodeId("n7")), "p4", ToSet(ToNodeId("n4")), "p2", ToSet(ToNodeId("n2") ))); // stutdown mgr and start a new mgr mgr.Stop(); mgr = new TestFileSystemNodeLabelsStore.MockNodeLabelManager(); mgr.Init(conf); mgr.Start(); // check variables NUnit.Framework.Assert.AreEqual(3, mgr.GetClusterNodeLabels().Count); NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().ContainsAll(Arrays.AsList ("p2", "p4", "p6"))); AssertMapContains(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n2"), ToSet("p2" ), ToNodeId("n4"), ToSet("p4"), ToNodeId("n6"), ToSet("p6"), ToNodeId("n7"), ToSet ("p6"))); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), ImmutableMap.Of("p6", ToSet(ToNodeId ("n6"), ToNodeId("n7")), "p4", ToSet(ToNodeId("n4")), "p2", ToSet(ToNodeId("n2") ))); mgr.Stop(); }
public virtual void TestChangeNsIDFails() { try { DoIt(ImmutableMap.Of("namespaceID", "2")); NUnit.Framework.Assert.Fail("Should throw InconsistentFSStateException"); } catch (IOException e) { GenericTestUtils.AssertExceptionContains("Inconsistent checkpoint fields", e); System.Console.Out.WriteLine("Correctly failed with inconsistent namespaceID: " + e); } }
/// <exception cref="System.Exception"/> public virtual void TestRemovelabelWithNodes() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n2"), ToSet("p2"))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n3"), ToSet("p3"))); mgr.RemoveFromClusterNodeLabels(ImmutableSet.Of("p1")); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n2"), ToSet("p2"), ToNodeId("n3"), ToSet("p3"))); AssertCollectionEquals(mgr.lastRemovedlabels, Arrays.AsList("p1")); mgr.RemoveFromClusterNodeLabels(ImmutableSet.Of("p2", "p3")); NUnit.Framework.Assert.IsTrue(mgr.GetNodeLabels().IsEmpty()); NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().IsEmpty()); AssertCollectionEquals(mgr.lastRemovedlabels, Arrays.AsList("p2", "p3")); }
/// <exception cref="System.Exception"/> public virtual void TestContainerAllocationWithSingleUserLimits() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.Init(conf); // set node -> label mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y")); mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId .NewInstance("h2", 0), ToSet("y"))); // inject node label manager MockRM rm1 = new _MockRM_471(mgr, GetConfigurationWithDefaultQueueLabels(conf)); rm1.GetRMContext().SetNodeLabelManager(mgr); rm1.Start(); MockNM nm1 = rm1.RegisterNode("h1:1234", 8000); // label = x rm1.RegisterNode("h2:1234", 8000); // label = y MockNM nm3 = rm1.RegisterNode("h3:1234", 8000); // label = <empty> // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1"); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); // A has only 10% of x, so it can only allocate one container in label=empty ContainerId containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId( ), 2); am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState .Allocated, 10 * 1000)); // Cannot allocate 2nd label=empty container containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3); am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState .Allocated, 10 * 1000)); // A has default user limit = 100, so it can use all resource in label = x // We can allocate floor(8000 / 1024) = 7 containers for (int id = 3; id <= 8; id++) { containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), id); am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x"); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState .Allocated, 10 * 1000)); } rm1.Close(); }
/// <exception cref="System.Exception"/> public virtual void TestGetLabelResourceWhenNodeActiveDeactive() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2" ), ToSet("p2"), ToNodeId("n3"), ToSet("p3"))); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), EmptyResource ); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p2", null), EmptyResource ); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p3", null), EmptyResource ); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(RMNodeLabelsManager.NoLabel , null), EmptyResource); // active two NM to n1, one large and one small mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n1", 2), LargeNode); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Add (SmallResource, LargeNode)); // check add labels multiple times shouldn't overwrite // original attributes on labels like resource mgr.AddToCluserNodeLabels(ToSet("p1", "p4")); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Add (SmallResource, LargeNode)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p4", null), EmptyResource ); // change the large NM to small, check if resource updated mgr.UpdateNodeResource(NodeId.NewInstance("n1", 2), SmallResource); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Multiply (SmallResource, 2)); // deactive one NM, and check if resource updated mgr.DeactivateNode(NodeId.NewInstance("n1", 1)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), SmallResource ); // continus deactive, check if resource updated mgr.DeactivateNode(NodeId.NewInstance("n1", 2)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), EmptyResource ); // Add two NM to n1 back mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n1", 2), LargeNode); // And remove p1, now the two NM should come to default label, mgr.RemoveFromClusterNodeLabels(ImmutableSet.Of("p1")); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(RMNodeLabelsManager.NoLabel , null), Resources.Add(SmallResource, LargeNode)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestReplaceLabelsOnHostsShouldUpdateNodesBelongTo() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p1")) ); // Replace labels on n1:1 to P2 mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p2"), ToNodeId("n1:2" ), ToSet("p2"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n1:1"), ToSet("p2"), ToNodeId("n1:2"), ToSet("p2"))); // Replace labels on n1 to P1, both n1:1/n1 will be P1 now mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n1:1"), ToSet("p1"), ToNodeId("n1:2"), ToSet("p1"))); // Set labels on n1:1 to P2 again to verify if add/remove works mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p2"))); }
/// <exception cref="System.IO.IOException"/> public virtual void TestLabelsToNodes() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); IDictionary <string, ICollection <NodeId> > labelsToNodes = mgr.GetLabelsToNodes(); AssertLabelsToNodesEquals(labelsToNodes, ImmutableMap.Of("p1", ToSet(ToNodeId("n1" )))); AssertLabelsToNodesEquals(labelsToNodes, TransposeNodeToLabels(mgr.GetNodeLabels( ))); // Replace labels on n1:1 to P2 mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p2"), ToNodeId("n1:2" ), ToSet("p2"))); labelsToNodes = mgr.GetLabelsToNodes(); AssertLabelsToNodesEquals(labelsToNodes, ImmutableMap.Of("p1", ToSet(ToNodeId("n1" )), "p2", ToSet(ToNodeId("n1:1"), ToNodeId("n1:2")))); AssertLabelsToNodesEquals(labelsToNodes, TransposeNodeToLabels(mgr.GetNodeLabels( ))); // Replace labels on n1 to P1, both n1:1/n1 will be P1 now mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); labelsToNodes = mgr.GetLabelsToNodes(); AssertLabelsToNodesEquals(labelsToNodes, ImmutableMap.Of("p1", ToSet(ToNodeId("n1" ), ToNodeId("n1:1"), ToNodeId("n1:2")))); AssertLabelsToNodesEquals(labelsToNodes, TransposeNodeToLabels(mgr.GetNodeLabels( ))); // Set labels on n1:1 to P2 again to verify if add/remove works mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p2"))); // Add p3 to n1, should makes n1:1 to be p2/p3, and n1:2 to be p1/p3 mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n2"), ToSet("p3"))); labelsToNodes = mgr.GetLabelsToNodes(); AssertLabelsToNodesEquals(labelsToNodes, ImmutableMap.Of("p1", ToSet(ToNodeId("n1" ), ToNodeId("n1:2")), "p2", ToSet(ToNodeId("n1:1")), "p3", ToSet(ToNodeId("n2")) )); AssertLabelsToNodesEquals(labelsToNodes, TransposeNodeToLabels(mgr.GetNodeLabels( ))); // Remove P3 from n1, should makes n1:1 to be p2, and n1:2 to be p1 mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n2"), ToSet("p3"))); labelsToNodes = mgr.GetLabelsToNodes(); AssertLabelsToNodesEquals(labelsToNodes, ImmutableMap.Of("p1", ToSet(ToNodeId("n1" ), ToNodeId("n1:2")), "p2", ToSet(ToNodeId("n1:1")))); AssertLabelsToNodesEquals(labelsToNodes, TransposeNodeToLabels(mgr.GetNodeLabels( ))); }
/// <exception cref="System.IO.IOException"/> public virtual void TestNoMoreThanOneLabelExistedInOneHost() { bool failed = false; // As in YARN-2694, we temporarily disable no more than one label existed in // one host mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); try { mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1", "p2"))); } catch (IOException) { failed = true; } NUnit.Framework.Assert.IsTrue("Should failed when set > 1 labels on a host", failed ); try { mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1", "p2"))); } catch (IOException) { failed = true; } NUnit.Framework.Assert.IsTrue("Should failed when add > 1 labels on a host", failed ); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); // add a same label to a node, #labels in this node is still 1, shouldn't // fail mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); try { mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p2"))); } catch (IOException) { failed = true; } NUnit.Framework.Assert.IsTrue("Should failed when #labels > 1 on a host after add" , failed); }
/// <exception cref="System.IO.IOException"/> public virtual void TestGetLabelResourceWhenMultipleNMsExistingInSameHost() { // active two NM to n1, one large and one small mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n1", 2), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n1", 3), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n1", 4), SmallResource); // check resource of no label, it should be small * 4 NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(CommonNodeLabelsManager.NoLabel , null), Resources.Multiply(SmallResource, 4)); // change two of these nodes to p1, check resource of no_label and P1 mgr.AddToCluserNodeLabels(ToSet("p1")); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1"), ToNodeId("n1:2" ), ToSet("p1"))); // check resource NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(CommonNodeLabelsManager.NoLabel , null), Resources.Multiply(SmallResource, 2)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Multiply (SmallResource, 2)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestPullRMNodeLabelsInfo() { mgr.AddToCluserNodeLabels(ToSet("x", "y", "z")); mgr.ActivateNode(NodeId.NewInstance("n1", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10, 0)); mgr.ActivateNode(NodeId.NewInstance("n2", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10, 0)); mgr.ActivateNode(NodeId.NewInstance("n3", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10, 0)); mgr.ActivateNode(NodeId.NewInstance("n4", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10, 0)); mgr.ActivateNode(NodeId.NewInstance("n5", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10, 0)); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("x"), ToNodeId("n2" ), ToSet("x"), ToNodeId("n3"), ToSet("y"))); // x, y, z and "" IList <NodeLabel> infos = mgr.PullRMNodeLabelsInfo(); NUnit.Framework.Assert.AreEqual(4, infos.Count); CheckNodeLabelInfo(infos, RMNodeLabelsManager.NoLabel, 2, 20); CheckNodeLabelInfo(infos, "x", 2, 20); CheckNodeLabelInfo(infos, "y", 1, 10); CheckNodeLabelInfo(infos, "z", 0, 0); }
/// <exception cref="System.IO.IOException"/> public virtual void TestLabelsToNodesForSelectedLabels() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1"), ToNodeId("n1:2" ), ToSet("p2"))); ICollection <string> setlabels = new HashSet <string>(Arrays.AsList(new string[] { "p1" })); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(setlabels), ImmutableMap.Of("p1", ToSet(ToNodeId("n1:1")))); // Replace labels on n1:1 to P3 mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p3"))); NUnit.Framework.Assert.IsTrue(mgr.GetLabelsToNodes(setlabels).IsEmpty()); setlabels = new HashSet <string>(Arrays.AsList(new string[] { "p2", "p3" })); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(setlabels), ImmutableMap.Of("p3", ToSet(ToNodeId("n1"), ToNodeId("n1:1"), ToNodeId("n1:2")))); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n2"), ToSet("p2"))); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(setlabels), ImmutableMap.Of("p2", ToSet(ToNodeId("n2")), "p3", ToSet(ToNodeId("n1"), ToNodeId("n1:1"), ToNodeId("n1:2" )))); mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p3"))); setlabels = new HashSet <string>(Arrays.AsList(new string[] { "p1", "p2", "p3" })); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(setlabels), ImmutableMap.Of("p2", ToSet(ToNodeId("n2")))); mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n3"), ToSet("p1"))); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(setlabels), ImmutableMap.Of("p1", ToSet(ToNodeId("n3")), "p2", ToSet(ToNodeId("n2")))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n2:2"), ToSet("p3"))); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(setlabels), ImmutableMap.Of("p1", ToSet(ToNodeId("n3")), "p2", ToSet(ToNodeId("n2")), "p3", ToSet(ToNodeId("n2:2") ))); setlabels = new HashSet <string>(Arrays.AsList(new string[] { "p1" })); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(setlabels), ImmutableMap.Of("p1", ToSet(ToNodeId("n3")))); }
/// <exception cref="System.Exception"/> public virtual void TestAddReplaceRemoveLabelsOnNodes() { // set a label on a node, but label doesn't exist bool caught = false; try { mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("node"), ToSet("label"))); } catch (IOException) { caught = true; } NUnit.Framework.Assert.IsTrue("trying to set a label to a node but " + "label doesn't exist in repository should fail" , caught); // set a label on a node, but node is null or empty try { mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId(CommonNodeLabelsManager.NoLabel) , ToSet("label"))); } catch (IOException) { caught = true; } NUnit.Framework.Assert.IsTrue("trying to add a empty node but succeeded", caught); // set node->label one by one mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p2"))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n2"), ToSet("p3"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p2"), ToNodeId("n2"), ToSet("p3"))); AssertMapEquals(mgr.lastNodeToLabels, ImmutableMap.Of(ToNodeId("n2"), ToSet("p3") )); // set bunch of node->label mgr.ReplaceLabelsOnNode((IDictionary)ImmutableMap.Of(ToNodeId("n3"), ToSet("p3"), ToNodeId("n1"), ToSet("p1"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2"), ToSet("p3"), ToNodeId("n3"), ToSet("p3"))); AssertMapEquals(mgr.lastNodeToLabels, ImmutableMap.Of(ToNodeId("n3"), ToSet("p3") , ToNodeId("n1"), ToSet("p1"))); /* * n1: p1 * n2: p3 * n3: p3 */ // remove label on node mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n2"), ToSet("p3"), ToNodeId("n3"), ToSet("p3"))); AssertMapEquals(mgr.lastNodeToLabels, ImmutableMap.Of(ToNodeId("n1"), CommonNodeLabelsManager .EmptyStringSet)); // add label on node mgr.AddLabelsToNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"))); AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2"), ToSet("p3"), ToNodeId("n3"), ToSet("p3"))); AssertMapEquals(mgr.lastNodeToLabels, ImmutableMap.Of(ToNodeId("n1"), ToSet("p1") )); // remove labels on node mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2" ), ToSet("p3"), ToNodeId("n3"), ToSet("p3"))); NUnit.Framework.Assert.AreEqual(0, mgr.GetNodeLabels().Count); AssertMapEquals(mgr.lastNodeToLabels, ImmutableMap.Of(ToNodeId("n1"), CommonNodeLabelsManager .EmptyStringSet, ToNodeId("n2"), CommonNodeLabelsManager.EmptyStringSet, ToNodeId ("n3"), CommonNodeLabelsManager.EmptyStringSet)); }
/// <exception cref="System.Exception"/> public virtual void TestGetQueueResource() { Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(9999, 1); /* * Node->Labels: * host1 : red * host2 : blue * host3 : yellow * host4 : */ mgr.AddToCluserNodeLabels(ToSet("red", "blue", "yellow")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("host1"), ToSet("red"))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("host2"), ToSet("blue"))); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("host3"), ToSet("yellow"))); // active two NM to n1, one large and one small mgr.ActivateNode(NodeId.NewInstance("host1", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("host2", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("host3", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("host4", 1), SmallResource); // reinitialize queue ICollection <string> q1Label = ToSet("red", "blue"); ICollection <string> q2Label = ToSet("blue", "yellow"); ICollection <string> q3Label = ToSet("yellow"); ICollection <string> q4Label = RMNodeLabelsManager.EmptyStringSet; ICollection <string> q5Label = ToSet(RMNodeLabelsManager.Any); IDictionary <string, ICollection <string> > queueToLabels = new Dictionary <string, ICollection <string> >(); queueToLabels["Q1"] = q1Label; queueToLabels["Q2"] = q2Label; queueToLabels["Q3"] = q3Label; queueToLabels["Q4"] = q4Label; queueToLabels["Q5"] = q5Label; mgr.ReinitializeQueueLabels(queueToLabels); // check resource NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q1", q1Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q2", q2Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q3", q3Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 1), mgr.GetQueueResource ("Q4", q4Label, clusterResource)); NUnit.Framework.Assert.AreEqual(clusterResource, mgr.GetQueueResource("Q5", q5Label , clusterResource)); mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("host2"), ToSet("blue"))); /* * Check resource after changes some labels * Node->Labels: * host1 : red * host2 : (was: blue) * host3 : yellow * host4 : */ // check resource NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q1", q1Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q2", q2Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q3", q3Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q4", q4Label, clusterResource)); NUnit.Framework.Assert.AreEqual(clusterResource, mgr.GetQueueResource("Q5", q5Label , clusterResource)); /* * Check resource after deactive/active some nodes * Node->Labels: * (deactived) host1 : red * host2 : * (deactived and then actived) host3 : yellow * host4 : */ mgr.DeactivateNode(NodeId.NewInstance("host1", 1)); mgr.DeactivateNode(NodeId.NewInstance("host3", 1)); mgr.ActivateNode(NodeId.NewInstance("host3", 1), SmallResource); // check resource NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q1", q1Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q2", q2Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q3", q3Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q4", q4Label, clusterResource)); NUnit.Framework.Assert.AreEqual(clusterResource, mgr.GetQueueResource("Q5", q5Label , clusterResource)); /* * Check resource after refresh queue: * Q1: blue * Q2: red, blue * Q3: red * Q4: * Q5: ANY */ q1Label = ToSet("blue"); q2Label = ToSet("blue", "red"); q3Label = ToSet("red"); q4Label = RMNodeLabelsManager.EmptyStringSet; q5Label = ToSet(RMNodeLabelsManager.Any); queueToLabels.Clear(); queueToLabels["Q1"] = q1Label; queueToLabels["Q2"] = q2Label; queueToLabels["Q3"] = q3Label; queueToLabels["Q4"] = q4Label; queueToLabels["Q5"] = q5Label; mgr.ReinitializeQueueLabels(queueToLabels); // check resource NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q1", q1Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q2", q2Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q3", q3Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 2), mgr.GetQueueResource ("Q4", q4Label, clusterResource)); NUnit.Framework.Assert.AreEqual(clusterResource, mgr.GetQueueResource("Q5", q5Label , clusterResource)); /* * Active NMs in nodes already have NM * Node->Labels: * host2 : * host3 : yellow (3 NMs) * host4 : (2 NMs) */ mgr.ActivateNode(NodeId.NewInstance("host3", 2), SmallResource); mgr.ActivateNode(NodeId.NewInstance("host3", 3), SmallResource); mgr.ActivateNode(NodeId.NewInstance("host4", 2), SmallResource); // check resource NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q1", q1Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q2", q2Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q3", q3Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 3), mgr.GetQueueResource ("Q4", q4Label, clusterResource)); NUnit.Framework.Assert.AreEqual(clusterResource, mgr.GetQueueResource("Q5", q5Label , clusterResource)); /* * Deactive NMs in nodes already have NMs * Node->Labels: * host2 : * host3 : yellow (2 NMs) * host4 : (0 NMs) */ mgr.DeactivateNode(NodeId.NewInstance("host3", 3)); mgr.DeactivateNode(NodeId.NewInstance("host4", 2)); mgr.DeactivateNode(NodeId.NewInstance("host4", 1)); // check resource NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 1), mgr.GetQueueResource ("Q1", q1Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 1), mgr.GetQueueResource ("Q2", q2Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 1), mgr.GetQueueResource ("Q3", q3Label, clusterResource)); NUnit.Framework.Assert.AreEqual(Resources.Multiply(SmallResource, 1), mgr.GetQueueResource ("Q4", q4Label, clusterResource)); NUnit.Framework.Assert.AreEqual(clusterResource, mgr.GetQueueResource("Q5", q5Label , clusterResource)); }
public virtual void TestUpgradePreFedSucceeds() { DoIt(ImmutableMap.Of("layoutVersion", "-19", "clusterID", string.Empty, "blockpoolID" , string.Empty)); }
public virtual void TestUpgradeLayoutVersionSucceeds() { DoIt(ImmutableMap.Of("layoutVersion", "-39")); }
/// <exception cref="System.IO.IOException"/> public virtual void TestNodeLabelsDisabled() { DummyCommonNodeLabelsManager mgr = new DummyCommonNodeLabelsManager(); Configuration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, false); mgr.Init(conf); mgr.Start(); bool caught = false; // add labels try { mgr.AddToCluserNodeLabels(ImmutableSet.Of("x")); } catch (IOException e) { AssertNodeLabelsDisabledErrorMessage(e); caught = true; } // check exception caught NUnit.Framework.Assert.IsTrue(caught); caught = false; // remove labels try { mgr.RemoveFromClusterNodeLabels(ImmutableSet.Of("x")); } catch (IOException e) { AssertNodeLabelsDisabledErrorMessage(e); caught = true; } // check exception caught NUnit.Framework.Assert.IsTrue(caught); caught = false; // add labels to node try { mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("host", 0), CommonNodeLabelsManager .EmptyStringSet)); } catch (IOException e) { AssertNodeLabelsDisabledErrorMessage(e); caught = true; } // check exception caught NUnit.Framework.Assert.IsTrue(caught); caught = false; // remove labels from node try { mgr.RemoveLabelsFromNode(ImmutableMap.Of(NodeId.NewInstance("host", 0), CommonNodeLabelsManager .EmptyStringSet)); } catch (IOException e) { AssertNodeLabelsDisabledErrorMessage(e); caught = true; } // check exception caught NUnit.Framework.Assert.IsTrue(caught); caught = false; // replace labels on node try { mgr.ReplaceLabelsOnNode(ImmutableMap.Of(NodeId.NewInstance("host", 0), CommonNodeLabelsManager .EmptyStringSet)); } catch (IOException e) { AssertNodeLabelsDisabledErrorMessage(e); caught = true; } // check exception caught NUnit.Framework.Assert.IsTrue(caught); caught = false; mgr.Close(); }
public _TesterCompressionStrategy_334() { this.joiner = Joiner.On("- "); this.emptySize = ImmutableMap.Of(typeof(Lz4Compressor), 4, typeof(ZlibCompressor) , 16, typeof(SnappyCompressor), 4, typeof(BuiltInZlibDeflater), 16); }
/// <exception cref="System.Exception"/> public virtual void TestContainerAllocateWithComplexLabels() { /* * Queue structure: * root (*) * ________________ * / \ * a x(100%), y(50%) b y(50%), z(100%) * ________________ ______________ * / / \ * a1 (x,y) b1(no) b2(y,z) * 100% y = 100%, z = 100% * * Node structure: * h1 : x * h2 : y * h3 : y * h4 : z * h5 : NO * * Total resource: * x: 4G * y: 6G * z: 2G * *: 2G * * Resource of * a1: x=4G, y=3G, NO=0.2G * b1: NO=0.9G (max=1G) * b2: y=3, z=2G, NO=0.9G (max=1G) * * Each node can only allocate two containers */ // set node -> label mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "z")); mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId .NewInstance("h2", 0), ToSet("y"), NodeId.NewInstance("h3", 0), ToSet("y"), NodeId .NewInstance("h4", 0), ToSet("z"), NodeId.NewInstance("h5", 0), RMNodeLabelsManager .EmptyStringSet)); // inject node label manager MockRM rm1 = new _MockRM_557(this, GetComplexConfigurationWithQueueLabels(conf)); rm1.GetRMContext().SetNodeLabelManager(mgr); rm1.Start(); MockNM nm1 = rm1.RegisterNode("h1:1234", 2048); MockNM nm2 = rm1.RegisterNode("h2:1234", 2048); MockNM nm3 = rm1.RegisterNode("h3:1234", 2048); MockNM nm4 = rm1.RegisterNode("h4:1234", 2048); MockNM nm5 = rm1.RegisterNode("h5:1234", 2048); ContainerId containerId; // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm1.SubmitApp(1024, "app", "user", null, "a1"); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); // request a container (label = y). can be allocated on nm2 am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "y"); containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2L); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h2"); // launch an app to queue b1 (label = y), and check all container will // be allocated in h5 RMApp app2 = rm1.SubmitApp(1024, "app", "user", null, "b1"); MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm5); // request a container for AM, will succeed // and now b1's queue capacity will be used, cannot allocate more containers // (Maximum capacity reached) am2.Allocate("*", 1024, 1, new AList <ContainerId>()); containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm4, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm5, containerId, RMContainerState .Allocated, 10 * 1000)); // launch an app to queue b2 RMApp app3 = rm1.SubmitApp(1024, "app", "user", null, "b2"); MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm5); // request a container. try to allocate on nm1 (label = x) and nm3 (label = // y,z). Will successfully allocate on nm3 am3.Allocate("*", 1024, 1, new AList <ContainerId>(), "y"); containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3"); // try to allocate container (request label = z) on nm4 (label = y,z). // Will successfully allocate on nm4 only. am3.Allocate("*", 1024, 1, new AList <ContainerId>(), "z"); containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 3L); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm4, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h4"); rm1.Close(); }
/// <exception cref="System.Exception"/> public virtual void TestNodeUpdate() { // set node -> label mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "z")); // set mapping: // h1 -> x // h2 -> y mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"))); mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h2", 0), ToSet("y"))); // inject node label manager MockRM rm = new _MockRM_110(this, GetConfigurationWithQueueLabels(conf)); rm.GetRMContext().SetNodeLabelManager(mgr); rm.Start(); MockNM nm1 = rm.RegisterNode("h1:1234", 8000); MockNM nm2 = rm.RegisterNode("h2:1234", 8000); MockNM nm3 = rm.RegisterNode("h3:1234", 8000); ContainerId containerId; // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm.SubmitApp(Gb, "app", "user", null, "a"); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm, nm3); // request a container. am1.Allocate("*", Gb, 1, new AList <ContainerId>(), "x"); containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState. Allocated, 10 * 1000)); // check used resource: // queue-a used x=1G, ""=1G CheckUsedResource(rm, "a", 1024, "x"); CheckUsedResource(rm, "a", 1024); // change h1's label to z, container should be killed mgr.ReplaceLabelsOnNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("z"))); NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState. Killed, 10 * 1000)); // check used resource: // queue-a used x=0G, ""=1G ("" not changed) CheckUsedResource(rm, "a", 0, "x"); CheckUsedResource(rm, "a", 1024); // request a container with label = y am1.Allocate("*", Gb, 1, new AList <ContainerId>(), "y"); containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3); NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm2, containerId, RMContainerState. Allocated, 10 * 1000)); // check used resource: // queue-a used y=1G, ""=1G CheckUsedResource(rm, "a", 1024, "y"); CheckUsedResource(rm, "a", 1024); // change h2's label to no label, container should be killed mgr.ReplaceLabelsOnNode(ImmutableMap.Of(NodeId.NewInstance("h2", 0), CommonNodeLabelsManager .EmptyStringSet)); NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState. Killed, 10 * 1000)); // check used resource: // queue-a used x=0G, y=0G, ""=1G ("" not changed) CheckUsedResource(rm, "a", 0, "x"); CheckUsedResource(rm, "a", 0, "y"); CheckUsedResource(rm, "a", 1024); containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 1); // change h3's label to z, AM container should be killed mgr.ReplaceLabelsOnNode(ImmutableMap.Of(NodeId.NewInstance("h3", 0), ToSet("z"))); NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState. Killed, 10 * 1000)); // check used resource: // queue-a used x=0G, y=0G, ""=1G ("" not changed) CheckUsedResource(rm, "a", 0, "x"); CheckUsedResource(rm, "a", 0, "y"); CheckUsedResource(rm, "a", 0); rm.Close(); }
/// <exception cref="System.Exception"/> public virtual void TestGetLabelResource() { mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2" ), ToSet("p2"), ToNodeId("n3"), ToSet("p3"))); // active two NM to n1, one large and one small mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n2", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n3", 1), SmallResource); // change label of n1 to p2 mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p2"))); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), EmptyResource ); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p2", null), Resources.Multiply (SmallResource, 2)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p3", null), SmallResource ); // add more labels mgr.AddToCluserNodeLabels(ToSet("p4", "p5", "p6")); mgr.ReplaceLabelsOnNode((IDictionary)ImmutableMap.Of(ToNodeId("n4"), ToSet("p1"), ToNodeId("n5"), ToSet("p2"), ToNodeId("n6"), ToSet("p3"), ToNodeId("n7"), ToSet( "p4"), ToNodeId("n8"), ToSet("p5"))); // now node -> label is, // p1 : n4 // p2 : n1, n2, n5 // p3 : n3, n6 // p4 : n7 // p5 : n8 // no-label : n9 // active these nodes mgr.ActivateNode(NodeId.NewInstance("n4", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n5", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n6", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n7", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n8", 1), SmallResource); mgr.ActivateNode(NodeId.NewInstance("n9", 1), SmallResource); // check varibles NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), SmallResource ); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p2", null), Resources.Multiply (SmallResource, 3)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p3", null), Resources.Multiply (SmallResource, 2)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p4", null), Resources.Multiply (SmallResource, 1)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p5", null), Resources.Multiply (SmallResource, 1)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(RMNodeLabelsManager.NoLabel , null), Resources.Multiply(SmallResource, 1)); // change a bunch of nodes -> labels // n4 -> p2 // n7 -> empty // n5 -> p1 // n8 -> empty // n9 -> p1 // // now become: // p1 : n5, n9 // p2 : n1, n2, n4 // p3 : n3, n6 // p4 : [ ] // p5 : [ ] // no label: n8, n7 mgr.ReplaceLabelsOnNode((IDictionary)ImmutableMap.Of(ToNodeId("n4"), ToSet("p2"), ToNodeId("n7"), RMNodeLabelsManager.EmptyStringSet, ToNodeId("n5"), ToSet("p1"), ToNodeId("n8"), RMNodeLabelsManager.EmptyStringSet, ToNodeId("n9"), ToSet("p1")) ); // check varibles NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Multiply (SmallResource, 2)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p2", null), Resources.Multiply (SmallResource, 3)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p3", null), Resources.Multiply (SmallResource, 2)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p4", null), Resources.Multiply (SmallResource, 0)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p5", null), Resources.Multiply (SmallResource, 0)); NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(string.Empty, null), Resources .Multiply(SmallResource, 2)); }
public virtual void TestSerilizationAfterRecovery() { //(timeout = 10000) mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3")); mgr.AddToCluserNodeLabels(ToSet("p4")); mgr.AddToCluserNodeLabels(ToSet("p5", "p6")); mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2" ), ToSet("p2"))); mgr.ReplaceLabelsOnNode((IDictionary)ImmutableMap.Of(ToNodeId("n3"), ToSet("p3"), ToNodeId("n4"), ToSet("p4"), ToNodeId("n5"), ToSet("p5"), ToNodeId("n6"), ToSet( "p6"), ToNodeId("n7"), ToSet("p6"))); /* * node -> labels * p1: n1 * p2: n2 * p3: n3 * p4: n4 * p5: n5 * p6: n6, n7 */ mgr.RemoveFromClusterNodeLabels(ToSet("p1")); mgr.RemoveFromClusterNodeLabels(Arrays.AsList("p3", "p5")); /* * After removed * p2: n2 * p4: n4 * p6: n6, n7 */ // shutdown mgr and start a new mgr mgr.Stop(); mgr = new TestFileSystemNodeLabelsStore.MockNodeLabelManager(); mgr.Init(conf); mgr.Start(); // check variables NUnit.Framework.Assert.AreEqual(3, mgr.GetClusterNodeLabels().Count); NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().ContainsAll(Arrays.AsList ("p2", "p4", "p6"))); AssertMapContains(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n2"), ToSet("p2" ), ToNodeId("n4"), ToSet("p4"), ToNodeId("n6"), ToSet("p6"), ToNodeId("n7"), ToSet ("p6"))); AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), ImmutableMap.Of("p6", ToSet(ToNodeId ("n6"), ToNodeId("n7")), "p4", ToSet(ToNodeId("n4")), "p2", ToSet(ToNodeId("n2") ))); /* * Add label p7,p8 then shutdown */ mgr = new TestFileSystemNodeLabelsStore.MockNodeLabelManager(); mgr.Init(conf); mgr.Start(); mgr.AddToCluserNodeLabels(ToSet("p7", "p8")); mgr.Stop(); /* * Restart, add label p9 and shutdown */ mgr = new TestFileSystemNodeLabelsStore.MockNodeLabelManager(); mgr.Init(conf); mgr.Start(); mgr.AddToCluserNodeLabels(ToSet("p9")); mgr.Stop(); /* * Recovery, and see if p9 added */ mgr = new TestFileSystemNodeLabelsStore.MockNodeLabelManager(); mgr.Init(conf); mgr.Start(); // check variables NUnit.Framework.Assert.AreEqual(6, mgr.GetClusterNodeLabels().Count); NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().ContainsAll(Arrays.AsList ("p2", "p4", "p6", "p7", "p8", "p9"))); mgr.Stop(); }