public virtual void TestReplaceMultipleLabelsOnSingleNode()
{
// Successfully replace labels
dummyNodeLabelsManager.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
string[] args = new string[] { "-replaceLabelsOnNode", "node1,x,y", "-directlyAccessNodeLabelStore" };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
}
public virtual void TestAddToClusterNodeLabels()
{
// successfully add labels
string[] args = new string[] { "-addToClusterNodeLabels", "x", "-directlyAccessNodeLabelStore" };
NUnit.Framework.Assert.AreEqual(0, rmAdminCLI.Run(args));
NUnit.Framework.Assert.IsTrue(dummyNodeLabelsManager.GetClusterNodeLabels().ContainsAll
(ImmutableSet.Of("x")));
// no labels, should fail
args = new string[] { "-addToClusterNodeLabels" };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
// no labels, should fail
args = new string[] { "-addToClusterNodeLabels", "-directlyAccessNodeLabelStore" };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
// no labels, should fail at client validation
args = new string[] { "-addToClusterNodeLabels", " " };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
// no labels, should fail at client validation
args = new string[] { "-addToClusterNodeLabels", " , " };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
// successfully add labels
args = new string[] { "-addToClusterNodeLabels", ",x,,", "-directlyAccessNodeLabelStore" };
NUnit.Framework.Assert.AreEqual(0, rmAdminCLI.Run(args));
NUnit.Framework.Assert.IsTrue(dummyNodeLabelsManager.GetClusterNodeLabels().ContainsAll
(ImmutableSet.Of("x")));
}
private void SetupQueueConfigurationWithLabelsInherit(CapacitySchedulerConfiguration
conf)
{
// Define top-level queues
conf.SetQueues(CapacitySchedulerConfiguration.Root, new string[] { "a", "b" });
conf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root, "red", 100);
conf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root, "blue", 100);
// Set A configuration
string A = CapacitySchedulerConfiguration.Root + ".a";
conf.SetCapacity(A, 10);
conf.SetMaximumCapacity(A, 15);
conf.SetQueues(A, new string[] { "a1", "a2" });
conf.SetAccessibleNodeLabels(A, ImmutableSet.Of("red", "blue"));
conf.SetCapacityByLabel(A, "red", 100);
conf.SetCapacityByLabel(A, "blue", 100);
// Set B configuraiton
string B = CapacitySchedulerConfiguration.Root + ".b";
conf.SetCapacity(B, 90);
conf.SetAccessibleNodeLabels(B, CommonNodeLabelsManager.EmptyStringSet);
// Define 2nd-level queues
string A1 = A + ".a1";
string A2 = A + ".a2";
conf.SetCapacity(A1, 30);
conf.SetMaximumCapacity(A1, 45);
conf.SetCapacityByLabel(A1, "red", 50);
conf.SetCapacityByLabel(A1, "blue", 100);
conf.SetCapacity(A2, 70);
conf.SetMaximumCapacity(A2, 85);
conf.SetAccessibleNodeLabels(A2, ImmutableSet.Of("red"));
conf.SetCapacityByLabel(A2, "red", 50);
}
public virtual void TestGetClusterNodeLabelsWithLocalAccess()
{
YarnClient client = Org.Mockito.Mockito.Mock <YarnClient>();
Org.Mockito.Mockito.When(client.GetClusterNodeLabels()).ThenReturn(ImmutableSet.Of
("remote1", "remote2"));
ClusterCLI cli = new ClusterCLI();
cli.SetClient(client);
cli.SetSysOutPrintStream(sysOut);
cli.SetSysErrPrintStream(sysErr);
ClusterCLI.localNodeLabelsManager = Org.Mockito.Mockito.Mock <CommonNodeLabelsManager
>();
Org.Mockito.Mockito.When(ClusterCLI.localNodeLabelsManager.GetClusterNodeLabels()
).ThenReturn(ImmutableSet.Of("local1", "local2"));
int rc = cli.Run(new string[] { ClusterCLI.Cmd, "-" + ClusterCLI.ListLabelsCmd, "-"
+ ClusterCLI.DirectlyAccessNodeLabelStore });
NUnit.Framework.Assert.AreEqual(0, rc);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
// it should return local* instead of remote*
pw.Write("Node Labels: local1,local2");
pw.Close();
Org.Mockito.Mockito.Verify(sysOut).WriteLine(baos.ToString("UTF-8"));
}
public static IList <RMNode> NewNodes(int racks, int nodesPerRack, Resource perNode
)
{
IList <RMNode> list = Lists.NewArrayList();
for (int i = 0; i < racks; ++i)
{
for (int j = 0; j < nodesPerRack; ++j)
{
if (j == (nodesPerRack - 1))
{
// One unhealthy node per rack.
list.AddItem(NodeInfo(i, perNode, NodeState.Unhealthy));
}
if (j == 0)
{
// One node with label
list.AddItem(NodeInfo(i, perNode, NodeState.Running, ImmutableSet.Of("x")));
}
else
{
list.AddItem(NewNodeInfo(i, perNode));
}
}
}
return(list);
}
public virtual void TestCountingBloomFilter()
{
int hashId = Org.Apache.Hadoop.Util.Hash.Hash.JenkinsHash;
CountingBloomFilter filter = new CountingBloomFilter(bitSize, hashFunctionNumber,
hashId);
Key key = new Key(new byte[] { 48, 48 });
filter.Add(key);
Assert.True("CountingBloomFilter.membership error ", filter.MembershipTest
(key));
Assert.True("CountingBloomFilter.approximateCount error", filter
.ApproximateCount(key) == 1);
filter.Add(key);
Assert.True("CountingBloomFilter.approximateCount error", filter
.ApproximateCount(key) == 2);
filter.Delete(key);
Assert.True("CountingBloomFilter.membership error ", filter.MembershipTest
(key));
filter.Delete(key);
NUnit.Framework.Assert.IsFalse("CountingBloomFilter.membership error ", filter.MembershipTest
(key));
Assert.True("CountingBloomFilter.approximateCount error", filter
.ApproximateCount(key) == 0);
BloomFilterCommonTester.Of(hashId, numInsertions).WithFilterInstance(filter).WithTestCases
(ImmutableSet.Of(BloomFilterCommonTester.BloomFilterTestStrategy.KeyTestStrategy
, BloomFilterCommonTester.BloomFilterTestStrategy.AddKeysStrategy, BloomFilterCommonTester.BloomFilterTestStrategy
.ExceptionsCheckStrategy, BloomFilterCommonTester.BloomFilterTestStrategy.OddEvenAbsentStrategy
, BloomFilterCommonTester.BloomFilterTestStrategy.WriteReadStrategy, BloomFilterCommonTester.BloomFilterTestStrategy
.FilterOrStrategy, BloomFilterCommonTester.BloomFilterTestStrategy.FilterXorStrategy
)).Test();
}
public virtual void TestRetouchedBloomFilterSpecific()
{
int numInsertions = 1000;
int hashFunctionNumber = 5;
ImmutableSet <int> hashes = ImmutableSet.Of(Org.Apache.Hadoop.Util.Hash.Hash.MurmurHash
, Org.Apache.Hadoop.Util.Hash.Hash.JenkinsHash);
foreach (int hashId in hashes)
{
RetouchedBloomFilter filter = new RetouchedBloomFilter(bitSize, hashFunctionNumber
, hashId);
CheckOnAbsentFalsePositive(hashId, numInsertions, filter, TestBloomFilters.Digits
.Odd, RemoveScheme.MaximumFp);
filter.And(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId));
CheckOnAbsentFalsePositive(hashId, numInsertions, filter, TestBloomFilters.Digits
.Even, RemoveScheme.MaximumFp);
filter.And(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId));
CheckOnAbsentFalsePositive(hashId, numInsertions, filter, TestBloomFilters.Digits
.Odd, RemoveScheme.MinimumFn);
filter.And(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId));
CheckOnAbsentFalsePositive(hashId, numInsertions, filter, TestBloomFilters.Digits
.Even, RemoveScheme.MinimumFn);
filter.And(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId));
CheckOnAbsentFalsePositive(hashId, numInsertions, filter, TestBloomFilters.Digits
.Odd, RemoveScheme.Ratio);
filter.And(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId));
CheckOnAbsentFalsePositive(hashId, numInsertions, filter, TestBloomFilters.Digits
.Even, RemoveScheme.Ratio);
filter.And(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAddRemovelabel()
{
// Add some label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("hello"));
AssertCollectionEquals(mgr.lastAddedlabels, Arrays.AsList("hello"));
mgr.AddToCluserNodeLabels(ImmutableSet.Of("world"));
mgr.AddToCluserNodeLabels(ToSet("hello1", "world1"));
AssertCollectionEquals(mgr.lastAddedlabels, Sets.NewHashSet("hello1", "world1"));
NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().ContainsAll(Sets.NewHashSet
("hello", "world", "hello1", "world1")));
// try to remove null, empty and non-existed label, should fail
foreach (string p in Arrays.AsList(null, CommonNodeLabelsManager.NoLabel, "xx"))
{
bool caught = false;
try
{
mgr.RemoveFromClusterNodeLabels(Arrays.AsList(p));
}
catch (IOException)
{
caught = true;
}
NUnit.Framework.Assert.IsTrue("remove label should fail " + "when label is null/empty/non-existed"
, caught);
}
// Remove some label
mgr.RemoveFromClusterNodeLabels(Arrays.AsList("hello"));
AssertCollectionEquals(mgr.lastRemovedlabels, Arrays.AsList("hello"));
NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().ContainsAll(Arrays.AsList
("world", "hello1", "world1")));
mgr.RemoveFromClusterNodeLabels(Arrays.AsList("hello1", "world1", "world"));
NUnit.Framework.Assert.IsTrue(mgr.lastRemovedlabels.ContainsAll(Sets.NewHashSet("hello1"
, "world1", "world")));
NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().IsEmpty());
}
public virtual void TestDFSAdminInvalidUsageHelp()
{
ImmutableSet <string> args = ImmutableSet.Of("-report", "-saveNamespace", "-rollEdits"
, "-restoreFailedStorage", "-refreshNodes", "-finalizeUpgrade", "-metasave", "-refreshUserToGroupsMappings"
, "-printTopology", "-refreshNamenodes", "-deleteBlockPool", "-setBalancerBandwidth"
, "-fetchImage");
try
{
foreach (string arg in args)
{
NUnit.Framework.Assert.IsTrue(ToolRunner.Run(new DFSAdmin(), FillArgs(arg)) == -1
);
}
NUnit.Framework.Assert.IsTrue(ToolRunner.Run(new DFSAdmin(), new string[] { "-help"
, "-some" }) == 0);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("testDFSAdminHelp error" + e);
}
string pattern = "Usage: hdfs dfsadmin";
CheckOutput(new string[] { "-cancel", "-renew" }, pattern, System.Console.Error,
typeof(DFSAdmin));
}
public virtual void TestReplaceLabelsOnNode()
{
// Successfully replace labels
dummyNodeLabelsManager.AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "Y"));
string[] args = new string[] { "-replaceLabelsOnNode", "node1:8000,x node2:8000=y node3,x node4=Y"
, "-directlyAccessNodeLabelStore" };
NUnit.Framework.Assert.AreEqual(0, rmAdminCLI.Run(args));
NUnit.Framework.Assert.IsTrue(dummyNodeLabelsManager.GetNodeLabels().Contains(NodeId
.NewInstance("node1", 8000)));
NUnit.Framework.Assert.IsTrue(dummyNodeLabelsManager.GetNodeLabels().Contains(NodeId
.NewInstance("node2", 8000)));
NUnit.Framework.Assert.IsTrue(dummyNodeLabelsManager.GetNodeLabels().Contains(NodeId
.NewInstance("node3", 0)));
NUnit.Framework.Assert.IsTrue(dummyNodeLabelsManager.GetNodeLabels().Contains(NodeId
.NewInstance("node4", 0)));
// no labels, should fail
args = new string[] { "-replaceLabelsOnNode" };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
// no labels, should fail
args = new string[] { "-replaceLabelsOnNode", "-directlyAccessNodeLabelStore" };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
// no labels, should fail
args = new string[] { "-replaceLabelsOnNode", " " };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
args = new string[] { "-replaceLabelsOnNode", ", " };
NUnit.Framework.Assert.IsTrue(0 != rmAdminCLI.Run(args));
}
/// <summary>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%.
/// </summary>
/// <remarks>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%. This expect IllegalArgumentException thrown.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard
()
{
nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue"));
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
SetupQueueConfigurationWithLabels(csConf);
csConf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root + ".b.b3", "red", 24
);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, ImmutableSet.
Of(RMNodeLabelsManager.Any));
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root + ".b", ImmutableSet
.Of(RMNodeLabelsManager.Any));
CapacityScheduler capacityScheduler = new CapacityScheduler();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetConf(csConf);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(csConf);
capacityScheduler.Start();
ServiceOperations.StopQuietly(capacityScheduler);
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithLabels()
{
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
MockNM nm2 = rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm3);
// request a container.
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h2
RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm3);
// request a container.
am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue c1 (label = ""), and check all container will
// be allocated in h3
RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3);
// request a container.
am3.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
rm1.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestAddlabelWithCase()
{
// Add some label, case will not ignore here
mgr.AddToCluserNodeLabels(ImmutableSet.Of("HeLlO"));
AssertCollectionEquals(mgr.lastAddedlabels, Arrays.AsList("HeLlO"));
NUnit.Framework.Assert.IsFalse(mgr.GetClusterNodeLabels().ContainsAll(Arrays.AsList
("hello")));
}
/// <exception cref="System.Exception"/>
private void AssertStorageDirsMatch(NameNode nn, BackupNode backup)
{
// Check that the stored files in the name dirs are identical
IList <FilePath> dirs = Lists.NewArrayList(FSImageTestUtil.GetCurrentDirs(nn.GetFSImage
().GetStorage(), null));
Sharpen.Collections.AddAll(dirs, FSImageTestUtil.GetCurrentDirs(backup.GetFSImage
().GetStorage(), null));
FSImageTestUtil.AssertParallelFilesAreIdentical(dirs, ImmutableSet.Of("VERSION"));
}
private void SetupQueueConfigurationWithLabels(CapacitySchedulerConfiguration conf
)
{
// Define top-level queues
conf.SetQueues(CapacitySchedulerConfiguration.Root, new string[] { "a", "b" });
conf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root, "red", 100);
conf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root, "blue", 100);
string A = CapacitySchedulerConfiguration.Root + ".a";
conf.SetCapacity(A, 10);
conf.SetMaximumCapacity(A, 15);
string B = CapacitySchedulerConfiguration.Root + ".b";
conf.SetCapacity(B, 90);
Log.Info("Setup top-level queues");
// Define 2nd-level queues
string A1 = A + ".a1";
string A2 = A + ".a2";
conf.SetQueues(A, new string[] { "a1", "a2" });
conf.SetAccessibleNodeLabels(A, ImmutableSet.Of("red", "blue"));
conf.SetCapacityByLabel(A, "red", 50);
conf.SetMaximumCapacityByLabel(A, "red", 50);
conf.SetCapacityByLabel(A, "blue", 50);
conf.SetCapacity(A1, 30);
conf.SetMaximumCapacity(A1, 45);
conf.SetCapacityByLabel(A1, "red", 50);
conf.SetCapacityByLabel(A1, "blue", 100);
conf.SetCapacity(A2, 70);
conf.SetMaximumCapacity(A2, 85);
conf.SetAccessibleNodeLabels(A2, ImmutableSet.Of("red"));
conf.SetCapacityByLabel(A2, "red", 50);
conf.SetMaximumCapacityByLabel(A2, "red", 60);
string B1 = B + ".b1";
string B2 = B + ".b2";
string B3 = B + ".b3";
conf.SetQueues(B, new string[] { "b1", "b2", "b3" });
conf.SetAccessibleNodeLabels(B, ImmutableSet.Of("red", "blue"));
conf.SetCapacityByLabel(B, "red", 50);
conf.SetCapacityByLabel(B, "blue", 50);
conf.SetCapacity(B1, 50);
conf.SetMaximumCapacity(B1, 85);
conf.SetCapacityByLabel(B1, "red", 50);
conf.SetCapacityByLabel(B1, "blue", 50);
conf.SetCapacity(B2, 30);
conf.SetMaximumCapacity(B2, 35);
conf.SetCapacityByLabel(B2, "red", 25);
conf.SetCapacityByLabel(B2, "blue", 25);
conf.SetCapacity(B3, 20);
conf.SetMaximumCapacity(B3, 35);
conf.SetCapacityByLabel(B3, "red", 25);
conf.SetCapacityByLabel(B3, "blue", 25);
}
public virtual void TestDynamicBloomFilter()
{
int hashId = Org.Apache.Hadoop.Util.Hash.Hash.JenkinsHash;
Filter filter = new DynamicBloomFilter(bitSize, hashFunctionNumber, Org.Apache.Hadoop.Util.Hash.Hash
.JenkinsHash, 3);
BloomFilterCommonTester.Of(hashId, numInsertions).WithFilterInstance(filter).WithTestCases
(ImmutableSet.Of(BloomFilterCommonTester.BloomFilterTestStrategy.KeyTestStrategy
, BloomFilterCommonTester.BloomFilterTestStrategy.AddKeysStrategy, BloomFilterCommonTester.BloomFilterTestStrategy
.ExceptionsCheckStrategy, BloomFilterCommonTester.BloomFilterTestStrategy.WriteReadStrategy
, BloomFilterCommonTester.BloomFilterTestStrategy.OddEvenAbsentStrategy)).Test();
NUnit.Framework.Assert.IsNotNull("testDynamicBloomFilter error ", filter.ToString
());
}
/// <exception cref="System.Exception"/>
public static void AssertNNFilesMatch(MiniDFSCluster cluster)
{
IList <FilePath> curDirs = Lists.NewArrayList();
Sharpen.Collections.AddAll(curDirs, FSImageTestUtil.GetNameNodeCurrentDirs(cluster
, 0));
Sharpen.Collections.AddAll(curDirs, FSImageTestUtil.GetNameNodeCurrentDirs(cluster
, 1));
// Ignore seen_txid file, since the newly bootstrapped standby
// will have a higher seen_txid than the one it bootstrapped from.
ICollection <string> ignoredFiles = ImmutableSet.Of("seen_txid");
FSImageTestUtil.AssertParallelFilesAreIdentical(curDirs, ignoredFiles);
}
/// <exception cref="System.Exception"/>
public virtual void TestRemovelabelWithNodes()
{
mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3"));
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1")));
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n2"), ToSet("p2")));
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n3"), ToSet("p3")));
mgr.RemoveFromClusterNodeLabels(ImmutableSet.Of("p1"));
AssertMapEquals(mgr.GetNodeLabels(), ImmutableMap.Of(ToNodeId("n2"), ToSet("p2"),
ToNodeId("n3"), ToSet("p3")));
AssertCollectionEquals(mgr.lastRemovedlabels, Arrays.AsList("p1"));
mgr.RemoveFromClusterNodeLabels(ImmutableSet.Of("p2", "p3"));
NUnit.Framework.Assert.IsTrue(mgr.GetNodeLabels().IsEmpty());
NUnit.Framework.Assert.IsTrue(mgr.GetClusterNodeLabels().IsEmpty());
AssertCollectionEquals(mgr.lastRemovedlabels, Arrays.AsList("p2", "p3"));
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocationWithSingleUserLimits()
{
RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
mgr.Init(conf);
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_471(mgr, GetConfigurationWithDefaultQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// A has only 10% of x, so it can only allocate one container in label=empty
ContainerId containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 2);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// Cannot allocate 2nd label=empty container
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// A has default user limit = 100, so it can use all resource in label = x
// We can allocate floor(8000 / 1024) = 7 containers
for (int id = 3; id <= 8; id++)
{
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), id);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
}
rm1.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestNormalizeNodeLabelExpression()
{
// mock queue and scheduler
YarnScheduler scheduler = Org.Mockito.Mockito.Mock <YarnScheduler>();
ICollection <string> queueAccessibleNodeLabels = Sets.NewHashSet();
QueueInfo queueInfo = Org.Mockito.Mockito.Mock <QueueInfo>();
Org.Mockito.Mockito.When(queueInfo.GetQueueName()).ThenReturn("queue");
Org.Mockito.Mockito.When(queueInfo.GetAccessibleNodeLabels()).ThenReturn(queueAccessibleNodeLabels
);
Org.Mockito.Mockito.When(queueInfo.GetDefaultNodeLabelExpression()).ThenReturn(" x "
);
Org.Mockito.Mockito.When(scheduler.GetQueueInfo(Matchers.Any <string>(), Matchers.AnyBoolean
(), Matchers.AnyBoolean())).ThenReturn(queueInfo);
Org.Apache.Hadoop.Yarn.Api.Records.Resource maxResource = Resources.CreateResource
(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb, YarnConfiguration.DefaultRmSchedulerMaximumAllocationVcores
);
// queue has labels, success cases
try
{
// set queue accessible node labels to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.IsTrue(resReq.GetNodeLabelExpression().Equals("x"));
resReq.SetNodeLabelExpression(" y ");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.IsTrue(resReq.GetNodeLabelExpression().Equals("y"));
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when request labels is a subset of queue labels"
);
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
}
public virtual void TestQueueParsingWithUnusedLabels()
{
ImmutableSet <string> labels = ImmutableSet.Of("red", "blue");
// Initialize a cluster with labels, but doesn't use them, reinitialize
// shouldn't fail
nodeLabelManager.AddToCluserNodeLabels(labels);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, rmContext);
// check root queue's capacity by label -- they should be all zero
CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"),
Delta);
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
capacityScheduler.Stop();
}
public virtual void TestMultipleSecondaryCheckpoint()
{
SecondaryNameNode secondary = null;
try
{
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).ManageNameDfsDirs(false
).Build();
cluster.WaitActive();
secondary = new SecondaryNameNode(config);
FSImage fsImage = cluster.GetNameNode().GetFSImage();
PrintStorages(fsImage);
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/", "test");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(testPath));
PrintStorages(fsImage);
// Take name1 offline
InvalidateStorage(fsImage, ImmutableSet.Of(path1));
// Simulate a 2NN beginning a checkpoint, but not finishing. This will
// cause name1 to be restored.
cluster.GetNameNodeRpc().RollEditLog();
PrintStorages(fsImage);
// Now another 2NN comes along to do a full checkpoint.
secondary.DoCheckpoint();
PrintStorages(fsImage);
// The created file should still exist in the in-memory FS state after the
// checkpoint.
NUnit.Framework.Assert.IsTrue("path exists before restart", fs.Exists(testPath));
secondary.Shutdown();
// Restart the NN so it reloads the edits from on-disk.
cluster.RestartNameNode();
// The created file should still exist after the restart.
NUnit.Framework.Assert.IsTrue("path should still exist after restart", fs.Exists(
testPath));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
if (secondary != null)
{
secondary.Shutdown();
}
}
}
private void CheckQueueLabelsInheritConfig(CapacityScheduler capacityScheduler)
{
// queue-A is red, blue
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A1 inherits A's configuration
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A2 is "red"
NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Count);
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Contains("red"));
// queue-B is "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels
().IsEmpty());
}
public virtual ICollection <string> GetAccessibleNodeLabels(string queue)
{
string accessibleLabelStr = Get(GetQueuePrefix(queue) + AccessibleNodeLabels);
// When accessible-label is null,
if (accessibleLabelStr == null)
{
// Only return null when queue is not ROOT
if (!queue.Equals(Root))
{
return(null);
}
}
else
{
// print a warning when accessibleNodeLabel specified in config and queue
// is ROOT
if (queue.Equals(Root))
{
Log.Warn("Accessible node labels for root queue will be ignored," + " it will be automatically set to \"*\"."
);
}
}
// always return ANY for queue root
if (queue.Equals(Root))
{
return(ImmutableSet.Of(RMNodeLabelsManager.Any));
}
// In other cases, split the accessibleLabelStr by ","
ICollection <string> set = new HashSet <string>();
foreach (string str in accessibleLabelStr.Split(","))
{
if (!str.Trim().IsEmpty())
{
set.AddItem(str.Trim());
}
}
// if labels contains "*", only keep ANY behind
if (set.Contains(RMNodeLabelsManager.Any))
{
set.Clear();
set.AddItem(RMNodeLabelsManager.Any);
}
return(Sharpen.Collections.UnmodifiableSet(set));
}
/// <exception cref="System.Exception"/>
public virtual void TestGetLabelResourceWhenNodeActiveDeactive()
{
mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3"));
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2"
), ToSet("p2"), ToNodeId("n3"), ToSet("p3")));
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), EmptyResource
);
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p2", null), EmptyResource
);
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p3", null), EmptyResource
);
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(RMNodeLabelsManager.NoLabel
, null), EmptyResource);
// active two NM to n1, one large and one small
mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource);
mgr.ActivateNode(NodeId.NewInstance("n1", 2), LargeNode);
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Add
(SmallResource, LargeNode));
// check add labels multiple times shouldn't overwrite
// original attributes on labels like resource
mgr.AddToCluserNodeLabels(ToSet("p1", "p4"));
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Add
(SmallResource, LargeNode));
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p4", null), EmptyResource
);
// change the large NM to small, check if resource updated
mgr.UpdateNodeResource(NodeId.NewInstance("n1", 2), SmallResource);
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), Resources.Multiply
(SmallResource, 2));
// deactive one NM, and check if resource updated
mgr.DeactivateNode(NodeId.NewInstance("n1", 1));
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), SmallResource
);
// continus deactive, check if resource updated
mgr.DeactivateNode(NodeId.NewInstance("n1", 2));
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel("p1", null), EmptyResource
);
// Add two NM to n1 back
mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource);
mgr.ActivateNode(NodeId.NewInstance("n1", 2), LargeNode);
// And remove p1, now the two NM should come to default label,
mgr.RemoveFromClusterNodeLabels(ImmutableSet.Of("p1"));
NUnit.Framework.Assert.AreEqual(mgr.GetResourceByLabel(RMNodeLabelsManager.NoLabel
, null), Resources.Add(SmallResource, LargeNode));
}
private Configuration GetConfigurationWithQueueLabels(Configuration config)
{
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config);
// Define top-level queues
conf.SetQueues(CapacitySchedulerConfiguration.Root, new string[] { "a" });
conf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root, "x", 100);
conf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root, "y", 100);
conf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root, "z", 100);
string A = CapacitySchedulerConfiguration.Root + ".a";
conf.SetCapacity(A, 100);
conf.SetAccessibleNodeLabels(A, ImmutableSet.Of("x", "y", "z"));
conf.SetCapacityByLabel(A, "x", 100);
conf.SetCapacityByLabel(A, "y", 100);
conf.SetCapacityByLabel(A, "z", 100);
return(conf);
}
public virtual void TestQueueParsingWithLabelsInherit()
{
nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue"));
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
SetupQueueConfigurationWithLabelsInherit(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetConf(csConf);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(csConf);
capacityScheduler.Start();
CheckQueueLabelsInheritConfig(capacityScheduler);
ServiceOperations.StopQuietly(capacityScheduler);
}
private void CheckQueueLabels(CapacityScheduler capacityScheduler)
{
// queue-A is red, blue
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A1 inherits A's configuration
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A2 is "red"
NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Count);
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Contains("red"));
// queue-B is "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-B2 inherits "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b2").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// check capacity of A2
CSQueue qA2 = capacityScheduler.GetQueue("a2");
NUnit.Framework.Assert.AreEqual(0.7, qA2.GetCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.5, qA2.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0.07, qA2.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.25, qA2.GetQueueCapacities().GetAbsoluteCapacity
("red"), Delta);
NUnit.Framework.Assert.AreEqual(0.1275, qA2.GetAbsoluteMaximumCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.3, qA2.GetQueueCapacities().GetAbsoluteMaximumCapacity
("red"), Delta);
// check capacity of B3
CSQueue qB3 = capacityScheduler.GetQueue("b3");
NUnit.Framework.Assert.AreEqual(0.18, qB3.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.125, qB3.GetQueueCapacities().GetAbsoluteCapacity
("red"), Delta);
NUnit.Framework.Assert.AreEqual(0.35, qB3.GetAbsoluteMaximumCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(1, qB3.GetQueueCapacities().GetAbsoluteMaximumCapacity
("red"), Delta);
}
private void SetupQueueConfigurationWithSingleLevel(CapacitySchedulerConfiguration
conf)
{
// Define top-level queues
conf.SetQueues(CapacitySchedulerConfiguration.Root, new string[] { "a", "b" });
// Set A configuration
string A = CapacitySchedulerConfiguration.Root + ".a";
conf.SetCapacity(A, 10);
conf.SetMaximumCapacity(A, 15);
conf.SetAccessibleNodeLabels(A, ImmutableSet.Of("red", "blue"));
conf.SetCapacityByLabel(A, "red", 90);
conf.SetCapacityByLabel(A, "blue", 90);
// Set B configuraiton
string B = CapacitySchedulerConfiguration.Root + ".b";
conf.SetCapacity(B, 90);
conf.SetAccessibleNodeLabels(B, ImmutableSet.Of("red", "blue"));
conf.SetCapacityByLabel(B, "red", 10);
conf.SetCapacityByLabel(B, "blue", 10);
}
/// <summary>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%.
/// </summary>
/// <remarks>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%. This expect IllegalArgumentException thrown.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent
()
{
nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue"));
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
SetupQueueConfiguration(csConf);
csConf.SetCapacity(CapacitySchedulerConfiguration.Root + ".c.c2", 5);
CapacityScheduler capacityScheduler = new CapacityScheduler();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetConf(csConf);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(csConf);
capacityScheduler.Start();
ServiceOperations.StopQuietly(capacityScheduler);
}