public virtual void TestMoverCliWithFederation() { MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NnTopology (MiniDFSNNTopology.SimpleFederatedTopology(3)).NumDataNodes(0).Build(); Configuration conf = new HdfsConfiguration(); DFSTestUtil.SetFederatedConfiguration(cluster, conf); try { ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf); NUnit.Framework.Assert.AreEqual(3, namenodes.Count); try { Mover.Cli.GetNameNodePathsToMove(conf, "-p", "/foo"); NUnit.Framework.Assert.Fail("Expect exception for missing authority information"); } catch (ArgumentException e) { GenericTestUtils.AssertExceptionContains("does not contain scheme and authority", e); } try { Mover.Cli.GetNameNodePathsToMove(conf, "-p", "hdfs:///foo"); NUnit.Framework.Assert.Fail("Expect exception for missing authority information"); } catch (ArgumentException e) { GenericTestUtils.AssertExceptionContains("does not contain scheme and authority", e); } try { Mover.Cli.GetNameNodePathsToMove(conf, "-p", "wrong-hdfs://ns1/foo"); NUnit.Framework.Assert.Fail("Expect exception for wrong scheme"); } catch (ArgumentException e) { GenericTestUtils.AssertExceptionContains("Cannot resolve the path", e); } IEnumerator <URI> iter = namenodes.GetEnumerator(); URI nn1 = iter.Next(); URI nn2 = iter.Next(); IDictionary <URI, IList <Path> > movePaths = Mover.Cli.GetNameNodePathsToMove(conf, "-p", nn1 + "/foo", nn1 + "/bar", nn2 + "/foo/bar"); NUnit.Framework.Assert.AreEqual(2, movePaths.Count); CheckMovePaths(movePaths[nn1], new Path("/foo"), new Path("/bar")); CheckMovePaths(movePaths[nn2], new Path("/foo/bar")); } finally { cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> private void RunTest(int nNameNodes, int nDataNodes, Configuration conf) { Log.Info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes); Log.Info("RUN_TEST -1"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes).Build(); Log.Info("RUN_TEST 0"); DFSTestUtil.SetFederatedConfiguration(cluster, conf); try { cluster.WaitActive(); Log.Info("RUN_TEST 1"); TestFsckWithMultipleNameNodes.Suite s = new TestFsckWithMultipleNameNodes.Suite(cluster , nNameNodes, nDataNodes); for (int i = 0; i < nNameNodes; i++) { s.CreateFile(i, 1024); } Log.Info("RUN_TEST 2"); string[] urls = new string[nNameNodes]; for (int i_1 = 0; i_1 < urls.Length; i_1++) { urls[i_1] = cluster.GetFileSystem(i_1).GetUri() + FileName; Log.Info("urls[" + i_1 + "]=" + urls[i_1]); string result = TestFsck.RunFsck(conf, 0, false, urls[i_1]); Log.Info("result=" + result); NUnit.Framework.Assert.IsTrue(result.Contains("Status: HEALTHY")); } // Test viewfs // Log.Info("RUN_TEST 3"); string[] vurls = new string[nNameNodes]; for (int i_2 = 0; i_2 < vurls.Length; i_2++) { string link = "/mount/nn_" + i_2 + FileName; ConfigUtil.AddLink(conf, link, new URI(urls[i_2])); vurls[i_2] = "viewfs:" + link; } for (int i_3 = 0; i_3 < vurls.Length; i_3++) { Log.Info("vurls[" + i_3 + "]=" + vurls[i_3]); string result = TestFsck.RunFsck(conf, 0, false, vurls[i_3]); Log.Info("result=" + result); NUnit.Framework.Assert.IsTrue(result.Contains("Status: HEALTHY")); } } finally { cluster.Shutdown(); } Log.Info("RUN_TEST 6"); }
/// <summary> /// This test start a cluster, fill the DataNodes to be 30% full; /// It then adds an empty node and start balancing. /// </summary> /// <param name="nNameNodes">Number of NameNodes</param> /// <param name="capacities">Capacities of the datanodes</param> /// <param name="racks">Rack names</param> /// <param name="newCapacity">the capacity of the new DataNode</param> /// <param name="newRack">the rack for the new DataNode</param> /// <param name="conf">Configuration</param> /// <exception cref="System.Exception"/> private void RunTest(int nNameNodes, long[] capacities, string[] racks, long newCapacity , string newRack, Configuration conf) { int nDataNodes = capacities.Length; Log.Info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes); NUnit.Framework.Assert.AreEqual(nDataNodes, racks.Length); Log.Info("RUN_TEST -1"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration(conf)).NnTopology (MiniDFSNNTopology.SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes) .Racks(racks).SimulatedCapacities(capacities).Build(); Log.Info("RUN_TEST 0"); DFSTestUtil.SetFederatedConfiguration(cluster, conf); try { cluster.WaitActive(); Log.Info("RUN_TEST 1"); TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite (cluster, nNameNodes, nDataNodes, conf); long totalCapacity = TestBalancer.Sum(capacities); Log.Info("RUN_TEST 2"); // fill up the cluster to be 30% full long totalUsed = totalCapacity * 3 / 10; long size = (totalUsed / nNameNodes) / s.replication; for (int n = 0; n < nNameNodes; n++) { CreateFile(s, n, size); } Log.Info("RUN_TEST 3"); // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }); totalCapacity += newCapacity; Log.Info("RUN_TEST 4"); // run RUN_TEST and validate results RunBalancer(s, totalUsed, totalCapacity); Log.Info("RUN_TEST 5"); } finally { cluster.Shutdown(); } Log.Info("RUN_TEST 6"); }
/// <summary>First start a cluster and fill the cluster up to a certain size.</summary> /// <remarks> /// First start a cluster and fill the cluster up to a certain size. /// Then redistribute blocks according the required distribution. /// Finally, balance the cluster. /// </remarks> /// <param name="nNameNodes">Number of NameNodes</param> /// <param name="distributionPerNN">The distribution for each NameNode.</param> /// <param name="capacities">Capacities of the datanodes</param> /// <param name="racks">Rack names</param> /// <param name="conf">Configuration</param> /// <exception cref="System.Exception"/> private void UnevenDistribution(int nNameNodes, long[] distributionPerNN, long[] capacities, string[] racks, Configuration conf) { Log.Info("UNEVEN 0"); int nDataNodes = distributionPerNN.Length; if (capacities.Length != nDataNodes || racks.Length != nDataNodes) { throw new ArgumentException("Array length is not the same"); } // calculate total space that need to be filled long usedSpacePerNN = TestBalancer.Sum(distributionPerNN); // fill the cluster ExtendedBlock[][] blocks; { Log.Info("UNEVEN 1"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration(conf)).NnTopology (MiniDFSNNTopology.SimpleFederatedTopology(2)).NumDataNodes(nDataNodes).Racks(racks ).SimulatedCapacities(capacities).Build(); Log.Info("UNEVEN 2"); try { cluster.WaitActive(); DFSTestUtil.SetFederatedConfiguration(cluster, conf); Log.Info("UNEVEN 3"); TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite (cluster, nNameNodes, nDataNodes, conf); blocks = GenerateBlocks(s, usedSpacePerNN); Log.Info("UNEVEN 4"); } finally { cluster.Shutdown(); } } conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.0f"); { Log.Info("UNEVEN 10"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes).Racks(racks).SimulatedCapacities (capacities).Format(false).Build(); Log.Info("UNEVEN 11"); try { cluster.WaitActive(); Log.Info("UNEVEN 12"); TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite (cluster, nNameNodes, nDataNodes, conf); for (int n = 0; n < nNameNodes; n++) { // redistribute blocks Block[][] blocksDN = TestBalancer.DistributeBlocks(blocks[n], s.replication, distributionPerNN ); for (int d = 0; d < blocksDN.Length; d++) { cluster.InjectBlocks(n, d, Arrays.AsList(blocksDN[d])); } Log.Info("UNEVEN 13: n=" + n); } long totalCapacity = TestBalancer.Sum(capacities); long totalUsed = nNameNodes * usedSpacePerNN; Log.Info("UNEVEN 14"); RunBalancer(s, totalUsed, totalCapacity); Log.Info("UNEVEN 15"); } finally { cluster.Shutdown(); } Log.Info("UNEVEN 16"); } }