/// <summary>
/// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2)
/// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2.
/// </summary>
/// <remarks>
/// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2)
/// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. Fill the cluster
/// to 60% and 3 replicas, so n2 and n3 will have replica for all blocks according
/// to replica placement policy with NodeGroup. As a result, n2 and n3 will be
/// filled with 80% (60% x 4 / 3), and no blocks can be migrated from n2 and n3
/// to n0 or n1 as balancer policy with node group. Thus, we expect the balancer
/// to end in 5 iterations without move block process.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerEndInNoMoveProgress()
{
Configuration conf = CreateConf();
long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity };
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 };
string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 };
int numOfDatanodes = capacities.Length;
NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
.Length).Racks(racks).SimulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
cluster = new MiniDFSClusterWithNodeGroup(builder);
try
{
cluster.WaitActive();
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
0).GetUri()).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 60% full
long totalUsedSpace = totalCapacity * 6 / 10;
TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / 3, (short)(3), 0);
// run balancer which can finish in 5 iterations with no block movement.
RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Create a cluster with even distribution, and a new empty node is added to
/// the cluster, then test rack locality for balancer policy.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerWithRackLocality()
{
Configuration conf = CreateConf();
long[] capacities = new long[] { Capacity, Capacity };
string[] racks = new string[] { Rack0, Rack1 };
string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 };
int numOfDatanodes = capacities.Length;
NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
.Length).Racks(racks).SimulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
cluster = new MiniDFSClusterWithNodeGroup(builder);
try
{
cluster.WaitActive();
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
0).GetUri()).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity * 3 / 10;
long length = totalUsedSpace / numOfDatanodes;
TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0);
LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length
);
ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0
);
long newCapacity = Capacity;
string newRack = Rack1;
string newNodeGroup = Nodegroup2;
// start up an empty node with the same capacity and on the same rack
cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
{ newCapacity }, new string[] { newNodeGroup });
totalCapacity += newCapacity;
// run balancer and validate results
RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length);
ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0);
NUnit.Framework.Assert.AreEqual(before, after);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Create a cluster with even distribution, and a new empty node is added to
/// the cluster, then test node-group locality for balancer policy.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerWithNodeGroup()
{
Configuration conf = CreateConf();
long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity };
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 };
string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 };
int numOfDatanodes = capacities.Length;
NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
.Length).Racks(racks).SimulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
cluster = new MiniDFSClusterWithNodeGroup(builder);
try
{
cluster.WaitActive();
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
0).GetUri()).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 20% full
long totalUsedSpace = totalCapacity * 2 / 10;
TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / (numOfDatanodes / 2),
(short)(numOfDatanodes / 2), 0);
long newCapacity = Capacity;
string newRack = Rack1;
string newNodeGroup = Nodegroup2;
// start up an empty node with the same capacity and on NODEGROUP2
cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
{ newCapacity }, new string[] { newNodeGroup });
totalCapacity += newCapacity;
// run balancer and validate results
RunBalancer(conf, totalUsedSpace, totalCapacity);
}
finally
{
cluster.Shutdown();
}
}