public virtual void TestMoverCliWithFederation()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NnTopology
(MiniDFSNNTopology.SimpleFederatedTopology(3)).NumDataNodes(0).Build();
Configuration conf = new HdfsConfiguration();
DFSTestUtil.SetFederatedConfiguration(cluster, conf);
try
{
ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf);
NUnit.Framework.Assert.AreEqual(3, namenodes.Count);
try
{
Mover.Cli.GetNameNodePathsToMove(conf, "-p", "/foo");
NUnit.Framework.Assert.Fail("Expect exception for missing authority information");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("does not contain scheme and authority",
e);
}
try
{
Mover.Cli.GetNameNodePathsToMove(conf, "-p", "hdfs:///foo");
NUnit.Framework.Assert.Fail("Expect exception for missing authority information");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("does not contain scheme and authority",
e);
}
try
{
Mover.Cli.GetNameNodePathsToMove(conf, "-p", "wrong-hdfs://ns1/foo");
NUnit.Framework.Assert.Fail("Expect exception for wrong scheme");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Cannot resolve the path", e);
}
IEnumerator <URI> iter = namenodes.GetEnumerator();
URI nn1 = iter.Next();
URI nn2 = iter.Next();
IDictionary <URI, IList <Path> > movePaths = Mover.Cli.GetNameNodePathsToMove(conf,
"-p", nn1 + "/foo", nn1 + "/bar", nn2 + "/foo/bar");
NUnit.Framework.Assert.AreEqual(2, movePaths.Count);
CheckMovePaths(movePaths[nn1], new Path("/foo"), new Path("/bar"));
CheckMovePaths(movePaths[nn2], new Path("/foo/bar"));
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
private void RunTest(int nNameNodes, int nDataNodes, Configuration conf)
{
Log.Info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes);
Log.Info("RUN_TEST -1");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes).Build();
Log.Info("RUN_TEST 0");
DFSTestUtil.SetFederatedConfiguration(cluster, conf);
try
{
cluster.WaitActive();
Log.Info("RUN_TEST 1");
TestFsckWithMultipleNameNodes.Suite s = new TestFsckWithMultipleNameNodes.Suite(cluster
, nNameNodes, nDataNodes);
for (int i = 0; i < nNameNodes; i++)
{
s.CreateFile(i, 1024);
}
Log.Info("RUN_TEST 2");
string[] urls = new string[nNameNodes];
for (int i_1 = 0; i_1 < urls.Length; i_1++)
{
urls[i_1] = cluster.GetFileSystem(i_1).GetUri() + FileName;
Log.Info("urls[" + i_1 + "]=" + urls[i_1]);
string result = TestFsck.RunFsck(conf, 0, false, urls[i_1]);
Log.Info("result=" + result);
NUnit.Framework.Assert.IsTrue(result.Contains("Status: HEALTHY"));
}
// Test viewfs
//
Log.Info("RUN_TEST 3");
string[] vurls = new string[nNameNodes];
for (int i_2 = 0; i_2 < vurls.Length; i_2++)
{
string link = "/mount/nn_" + i_2 + FileName;
ConfigUtil.AddLink(conf, link, new URI(urls[i_2]));
vurls[i_2] = "viewfs:" + link;
}
for (int i_3 = 0; i_3 < vurls.Length; i_3++)
{
Log.Info("vurls[" + i_3 + "]=" + vurls[i_3]);
string result = TestFsck.RunFsck(conf, 0, false, vurls[i_3]);
Log.Info("result=" + result);
NUnit.Framework.Assert.IsTrue(result.Contains("Status: HEALTHY"));
}
}
finally
{
cluster.Shutdown();
}
Log.Info("RUN_TEST 6");
}
/// <summary>
/// This test start a cluster, fill the DataNodes to be 30% full;
/// It then adds an empty node and start balancing.
/// </summary>
/// <param name="nNameNodes">Number of NameNodes</param>
/// <param name="capacities">Capacities of the datanodes</param>
/// <param name="racks">Rack names</param>
/// <param name="newCapacity">the capacity of the new DataNode</param>
/// <param name="newRack">the rack for the new DataNode</param>
/// <param name="conf">Configuration</param>
/// <exception cref="System.Exception"/>
private void RunTest(int nNameNodes, long[] capacities, string[] racks, long newCapacity
, string newRack, Configuration conf)
{
int nDataNodes = capacities.Length;
Log.Info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes);
NUnit.Framework.Assert.AreEqual(nDataNodes, racks.Length);
Log.Info("RUN_TEST -1");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration(conf)).NnTopology
(MiniDFSNNTopology.SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes)
.Racks(racks).SimulatedCapacities(capacities).Build();
Log.Info("RUN_TEST 0");
DFSTestUtil.SetFederatedConfiguration(cluster, conf);
try
{
cluster.WaitActive();
Log.Info("RUN_TEST 1");
TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite
(cluster, nNameNodes, nDataNodes, conf);
long totalCapacity = TestBalancer.Sum(capacities);
Log.Info("RUN_TEST 2");
// fill up the cluster to be 30% full
long totalUsed = totalCapacity * 3 / 10;
long size = (totalUsed / nNameNodes) / s.replication;
for (int n = 0; n < nNameNodes; n++)
{
CreateFile(s, n, size);
}
Log.Info("RUN_TEST 3");
// start up an empty node with the same capacity and on the same rack
cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
{ newCapacity });
totalCapacity += newCapacity;
Log.Info("RUN_TEST 4");
// run RUN_TEST and validate results
RunBalancer(s, totalUsed, totalCapacity);
Log.Info("RUN_TEST 5");
}
finally
{
cluster.Shutdown();
}
Log.Info("RUN_TEST 6");
}
/// <summary>First start a cluster and fill the cluster up to a certain size.</summary>
/// <remarks>
/// First start a cluster and fill the cluster up to a certain size.
/// Then redistribute blocks according the required distribution.
/// Finally, balance the cluster.
/// </remarks>
/// <param name="nNameNodes">Number of NameNodes</param>
/// <param name="distributionPerNN">The distribution for each NameNode.</param>
/// <param name="capacities">Capacities of the datanodes</param>
/// <param name="racks">Rack names</param>
/// <param name="conf">Configuration</param>
/// <exception cref="System.Exception"/>
private void UnevenDistribution(int nNameNodes, long[] distributionPerNN, long[]
capacities, string[] racks, Configuration conf)
{
Log.Info("UNEVEN 0");
int nDataNodes = distributionPerNN.Length;
if (capacities.Length != nDataNodes || racks.Length != nDataNodes)
{
throw new ArgumentException("Array length is not the same");
}
// calculate total space that need to be filled
long usedSpacePerNN = TestBalancer.Sum(distributionPerNN);
// fill the cluster
ExtendedBlock[][] blocks;
{
Log.Info("UNEVEN 1");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration(conf)).NnTopology
(MiniDFSNNTopology.SimpleFederatedTopology(2)).NumDataNodes(nDataNodes).Racks(racks
).SimulatedCapacities(capacities).Build();
Log.Info("UNEVEN 2");
try
{
cluster.WaitActive();
DFSTestUtil.SetFederatedConfiguration(cluster, conf);
Log.Info("UNEVEN 3");
TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite
(cluster, nNameNodes, nDataNodes, conf);
blocks = GenerateBlocks(s, usedSpacePerNN);
Log.Info("UNEVEN 4");
}
finally
{
cluster.Shutdown();
}
}
conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.0f");
{
Log.Info("UNEVEN 10");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleFederatedTopology(nNameNodes)).NumDataNodes(nDataNodes).Racks(racks).SimulatedCapacities
(capacities).Format(false).Build();
Log.Info("UNEVEN 11");
try
{
cluster.WaitActive();
Log.Info("UNEVEN 12");
TestBalancerWithMultipleNameNodes.Suite s = new TestBalancerWithMultipleNameNodes.Suite
(cluster, nNameNodes, nDataNodes, conf);
for (int n = 0; n < nNameNodes; n++)
{
// redistribute blocks
Block[][] blocksDN = TestBalancer.DistributeBlocks(blocks[n], s.replication, distributionPerNN
);
for (int d = 0; d < blocksDN.Length; d++)
{
cluster.InjectBlocks(n, d, Arrays.AsList(blocksDN[d]));
}
Log.Info("UNEVEN 13: n=" + n);
}
long totalCapacity = TestBalancer.Sum(capacities);
long totalUsed = nNameNodes * usedSpacePerNN;
Log.Info("UNEVEN 14");
RunBalancer(s, totalUsed, totalCapacity);
Log.Info("UNEVEN 15");
}
finally
{
cluster.Shutdown();
}
Log.Info("UNEVEN 16");
}
}
