/// <summary> /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2) /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. /// </summary> /// <remarks> /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2) /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. Fill the cluster /// to 60% and 3 replicas, so n2 and n3 will have replica for all blocks according /// to replica placement policy with NodeGroup. As a result, n2 and n3 will be /// filled with 80% (60% x 4 / 3), and no blocks can be migrated from n2 and n3 /// to n0 or n1 as balancer policy with node group. Thus, we expect the balancer /// to end in 5 iterations without move block process. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestBalancerEndInNoMoveProgress() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 60% full long totalUsedSpace = totalCapacity * 6 / 10; TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / 3, (short)(3), 0); // run balancer which can finish in 5 iterations with no block movement. RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); } finally { cluster.Shutdown(); } }
/// <summary> /// Test a cluster with even distribution, then a new empty node is added to /// the cluster. /// </summary> /// <remarks> /// Test a cluster with even distribution, then a new empty node is added to /// the cluster. Test start a cluster with specified number of nodes, and fills /// it to be 30% full (with a single file replicated identically to all /// datanodes); It then adds one new empty node and starts balancing. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithHANameNodes() { Configuration conf = new HdfsConfiguration(); TestBalancer.InitConf(conf); long newNodeCapacity = TestBalancer.Capacity; // new node's capacity string newNodeRack = TestBalancer.Rack2; // new node's rack // array of racks for original nodes in cluster string[] racks = new string[] { TestBalancer.Rack0, TestBalancer.Rack1 }; // array of capacities of original nodes in cluster long[] capacities = new long[] { TestBalancer.Capacity, TestBalancer.Capacity }; NUnit.Framework.Assert.AreEqual(capacities.Length, racks.Length); int numOfDatanodes = capacities.Length; MiniDFSNNTopology.NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); nn1Conf.SetIpcPort(NameNode.DefaultPort); Configuration copiedConf = new Configuration(conf); cluster = new MiniDFSCluster.Builder(copiedConf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(capacities.Length).Racks(racks).SimulatedCapacities(capacities) .Build(); HATestUtil.SetFailoverConfigurations(cluster, conf); try { cluster.WaitActive(); cluster.TransitionToActive(1); Sharpen.Thread.Sleep(500); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, FileSystem.GetDefaultUri (conf)).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; TestBalancer.CreateFile(cluster, TestBalancer.filePath, totalUsedSpace / numOfDatanodes , (short)numOfDatanodes, 1); // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newNodeRack }, new long [] { newNodeCapacity }); totalCapacity += newNodeCapacity; TestBalancer.WaitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster); ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf); NUnit.Framework.Assert.AreEqual(1, namenodes.Count); NUnit.Framework.Assert.IsTrue(namenodes.Contains(HATestUtil.GetLogicalUri(cluster ))); int r = Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer.Run(namenodes, Balancer.Parameters .Default, conf); NUnit.Framework.Assert.AreEqual(ExitStatus.Success.GetExitCode(), r); TestBalancer.WaitForBalancer(totalUsedSpace, totalCapacity, client, cluster, Balancer.Parameters .Default); } finally { cluster.Shutdown(); } }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test rack locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithRackLocality() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; long length = totalUsedSpace / numOfDatanodes; TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0); LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length ); ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0 ); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length); ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0); NUnit.Framework.Assert.AreEqual(before, after); } finally { cluster.Shutdown(); } }
/// <summary> /// This test start a cluster, fill the DataNodes to be 30% full; /// It then adds an empty node and start balancing. /// </summary> /// <param name="nNameNodes">Number of NameNodes</param> /// <param name="capacities">Capacities of the datanodes</param> /// <param name="racks">Rack names</param> /// <param name="newCapacity">the capacity of the new DataNode</param> /// <param name="newRack">the rack for the new DataNode</param> /// <param name="conf">Configuration</param> /// <exception cref="System.Exception"/> private void RunTest(int nNameNodes, long[] capacities, string[] racks, long newCapacity , string newRack, Configuration conf) { int nDataNodes = capacities.Length; Log.Info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes); NUnit.Framework.Assert.AreEqual(nDataNodes, racks.Length); Log.Info("RUN_TEST -1"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration(conf)).NnTopology (MiniDFSNNTopology.SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes) .Racks(racks).SimulatedCapacities(capacities).Build(); Log.Info("RUN_TEST 0"); DFSTestUtil.SetFederatedConfiguration(cluster, conf); try { cluster.WaitActive(); Log.Info("RUN_TEST 1"); TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite (cluster, nNameNodes, nDataNodes, conf); long totalCapacity = TestBalancer.Sum(capacities); Log.Info("RUN_TEST 2"); // fill up the cluster to be 30% full long totalUsed = totalCapacity * 3 / 10; long size = (totalUsed / nNameNodes) / s.replication; for (int n = 0; n < nNameNodes; n++) { CreateFile(s, n, size); } Log.Info("RUN_TEST 3"); // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }); totalCapacity += newCapacity; Log.Info("RUN_TEST 4"); // run RUN_TEST and validate results RunBalancer(s, totalUsed, totalCapacity); Log.Info("RUN_TEST 5"); } finally { cluster.Shutdown(); } Log.Info("RUN_TEST 6"); }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test node-group locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithNodeGroup() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 20% full long totalUsedSpace = totalCapacity * 2 / 10; TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / (numOfDatanodes / 2), (short)(numOfDatanodes / 2), 0); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on NODEGROUP2 cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancer(conf, totalUsedSpace, totalCapacity); } finally { cluster.Shutdown(); } }
/// <summary>First start a cluster and fill the cluster up to a certain size.</summary> /// <remarks> /// First start a cluster and fill the cluster up to a certain size. /// Then redistribute blocks according the required distribution. /// Finally, balance the cluster. /// </remarks> /// <param name="nNameNodes">Number of NameNodes</param> /// <param name="distributionPerNN">The distribution for each NameNode.</param> /// <param name="capacities">Capacities of the datanodes</param> /// <param name="racks">Rack names</param> /// <param name="conf">Configuration</param> /// <exception cref="System.Exception"/> private void UnevenDistribution(int nNameNodes, long[] distributionPerNN, long[] capacities, string[] racks, Configuration conf) { Log.Info("UNEVEN 0"); int nDataNodes = distributionPerNN.Length; if (capacities.Length != nDataNodes || racks.Length != nDataNodes) { throw new ArgumentException("Array length is not the same"); } // calculate total space that need to be filled long usedSpacePerNN = TestBalancer.Sum(distributionPerNN); // fill the cluster ExtendedBlock[][] blocks; { Log.Info("UNEVEN 1"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration(conf)).NnTopology (MiniDFSNNTopology.SimpleFederatedTopology(2)).NumDataNodes(nDataNodes).Racks(racks ).SimulatedCapacities(capacities).Build(); Log.Info("UNEVEN 2"); try { cluster.WaitActive(); DFSTestUtil.SetFederatedConfiguration(cluster, conf); Log.Info("UNEVEN 3"); TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite (cluster, nNameNodes, nDataNodes, conf); blocks = GenerateBlocks(s, usedSpacePerNN); Log.Info("UNEVEN 4"); } finally { cluster.Shutdown(); } } conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.0f"); { Log.Info("UNEVEN 10"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes).Racks(racks).SimulatedCapacities (capacities).Format(false).Build(); Log.Info("UNEVEN 11"); try { cluster.WaitActive(); Log.Info("UNEVEN 12"); TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite (cluster, nNameNodes, nDataNodes, conf); for (int n = 0; n < nNameNodes; n++) { // redistribute blocks Block[][] blocksDN = TestBalancer.DistributeBlocks(blocks[n], s.replication, distributionPerNN ); for (int d = 0; d < blocksDN.Length; d++) { cluster.InjectBlocks(n, d, Arrays.AsList(blocksDN[d])); } Log.Info("UNEVEN 13: n=" + n); } long totalCapacity = TestBalancer.Sum(capacities); long totalUsed = nNameNodes * usedSpacePerNN; Log.Info("UNEVEN 14"); RunBalancer(s, totalUsed, totalCapacity); Log.Info("UNEVEN 15"); } finally { cluster.Shutdown(); } Log.Info("UNEVEN 16"); } }
/// <exception cref="System.Exception"/> internal static void RunBalancer(TestBalancerWithMultipleNameNodes.Suite s, long totalUsed, long totalCapacity) { double avg = totalUsed * 100.0 / totalCapacity; Log.Info("BALANCER 0: totalUsed=" + totalUsed + ", totalCapacity=" + totalCapacity + ", avg=" + avg); Wait(s.clients, totalUsed, totalCapacity); Log.Info("BALANCER 1"); // start rebalancing ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(s.conf); int r = Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer.Run(namenodes, Balancer.Parameters .Default, s.conf); NUnit.Framework.Assert.AreEqual(ExitStatus.Success.GetExitCode(), r); Log.Info("BALANCER 2"); Wait(s.clients, totalUsed, totalCapacity); Log.Info("BALANCER 3"); int i = 0; for (bool balanced = false; !balanced; i++) { long[] used = new long[s.cluster.GetDataNodes().Count]; long[] cap = new long[used.Length]; for (int n = 0; n < s.clients.Length; n++) { DatanodeInfo[] datanodes = s.clients[n].GetDatanodeReport(HdfsConstants.DatanodeReportType .All); NUnit.Framework.Assert.AreEqual(datanodes.Length, used.Length); for (int d = 0; d < datanodes.Length; d++) { if (n == 0) { used[d] = datanodes[d].GetDfsUsed(); cap[d] = datanodes[d].GetCapacity(); if (i % 100 == 0) { Log.Warn("datanodes[" + d + "]: getDfsUsed()=" + datanodes[d].GetDfsUsed() + ", getCapacity()=" + datanodes[d].GetCapacity()); } } else { NUnit.Framework.Assert.AreEqual(used[d], datanodes[d].GetDfsUsed()); NUnit.Framework.Assert.AreEqual(cap[d], datanodes[d].GetCapacity()); } } } balanced = true; for (int d_1 = 0; d_1 < used.Length; d_1++) { double p = used[d_1] * 100.0 / cap[d_1]; balanced = p <= avg + Balancer.Parameters.Default.threshold; if (!balanced) { if (i % 100 == 0) { Log.Warn("datanodes " + d_1 + " is not yet balanced: " + "used=" + used[d_1] + ", cap=" + cap[d_1] + ", avg=" + avg); Log.Warn("TestBalancer.sum(used)=" + TestBalancer.Sum(used) + ", TestBalancer.sum(cap)=" + TestBalancer.Sum(cap)); } Sleep(100); break; } } } Log.Info("BALANCER 6"); }