/// <summary> /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2) /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. /// </summary> /// <remarks> /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2) /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. Fill the cluster /// to 60% and 3 replicas, so n2 and n3 will have replica for all blocks according /// to replica placement policy with NodeGroup. As a result, n2 and n3 will be /// filled with 80% (60% x 4 / 3), and no blocks can be migrated from n2 and n3 /// to n0 or n1 as balancer policy with node group. Thus, we expect the balancer /// to end in 5 iterations without move block process. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestBalancerEndInNoMoveProgress() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 60% full long totalUsedSpace = totalCapacity * 6 / 10; TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / 3, (short)(3), 0); // run balancer which can finish in 5 iterations with no block movement. RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); } finally { cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> public NameNodeConnector(string name, URI nameNodeUri, Path idPath, IList <Path> targetPaths , Configuration conf, int maxNotChangedIterations) { this.nameNodeUri = nameNodeUri; this.idPath = idPath; this.targetPaths = targetPaths == null || targetPaths.IsEmpty() ? Arrays.AsList(new Path("/")) : targetPaths; this.maxNotChangedIterations = maxNotChangedIterations; this.namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(conf, nameNodeUri). GetProxy(); this.client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, nameNodeUri, fallbackToSimpleAuth ).GetProxy(); this.fs = (DistributedFileSystem)FileSystem.Get(nameNodeUri, conf); NamespaceInfo namespaceinfo = namenode.VersionRequest(); this.blockpoolID = namespaceinfo.GetBlockPoolID(); FsServerDefaults defaults = fs.GetServerDefaults(new Path("/")); this.keyManager = new KeyManager(blockpoolID, namenode, defaults.GetEncryptDataTransfer (), conf); // if it is for test, we do not create the id file @out = CheckAndMarkRunning(); if (@out == null) { // Exit if there is another one running. throw new IOException("Another " + name + " is running."); } }
/// <summary> /// Test a cluster with even distribution, then a new empty node is added to /// the cluster. /// </summary> /// <remarks> /// Test a cluster with even distribution, then a new empty node is added to /// the cluster. Test start a cluster with specified number of nodes, and fills /// it to be 30% full (with a single file replicated identically to all /// datanodes); It then adds one new empty node and starts balancing. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithHANameNodes() { Configuration conf = new HdfsConfiguration(); TestBalancer.InitConf(conf); long newNodeCapacity = TestBalancer.Capacity; // new node's capacity string newNodeRack = TestBalancer.Rack2; // new node's rack // array of racks for original nodes in cluster string[] racks = new string[] { TestBalancer.Rack0, TestBalancer.Rack1 }; // array of capacities of original nodes in cluster long[] capacities = new long[] { TestBalancer.Capacity, TestBalancer.Capacity }; NUnit.Framework.Assert.AreEqual(capacities.Length, racks.Length); int numOfDatanodes = capacities.Length; MiniDFSNNTopology.NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); nn1Conf.SetIpcPort(NameNode.DefaultPort); Configuration copiedConf = new Configuration(conf); cluster = new MiniDFSCluster.Builder(copiedConf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(capacities.Length).Racks(racks).SimulatedCapacities(capacities) .Build(); HATestUtil.SetFailoverConfigurations(cluster, conf); try { cluster.WaitActive(); cluster.TransitionToActive(1); Sharpen.Thread.Sleep(500); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, FileSystem.GetDefaultUri (conf)).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; TestBalancer.CreateFile(cluster, TestBalancer.filePath, totalUsedSpace / numOfDatanodes , (short)numOfDatanodes, 1); // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newNodeRack }, new long [] { newNodeCapacity }); totalCapacity += newNodeCapacity; TestBalancer.WaitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster); ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf); NUnit.Framework.Assert.AreEqual(1, namenodes.Count); NUnit.Framework.Assert.IsTrue(namenodes.Contains(HATestUtil.GetLogicalUri(cluster ))); int r = Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer.Run(namenodes, Balancer.Parameters .Default, conf); NUnit.Framework.Assert.AreEqual(ExitStatus.Success.GetExitCode(), r); TestBalancer.WaitForBalancer(totalUsedSpace, totalCapacity, client, cluster, Balancer.Parameters .Default); } finally { cluster.Shutdown(); } }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test rack locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithRackLocality() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; long length = totalUsedSpace / numOfDatanodes; TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0); LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length ); ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0 ); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length); ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0); NUnit.Framework.Assert.AreEqual(before, after); } finally { cluster.Shutdown(); } }
/// <summary> /// Create a /// <see cref="NameNode"/> /// proxy from the current /// <see cref="Javax.Servlet.ServletContext"/> /// . /// </summary> /// <exception cref="System.IO.IOException"/> protected internal virtual ClientProtocol CreateNameNodeProxy() { ServletContext context = GetServletContext(); // if we are running in the Name Node, use it directly rather than via // rpc NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context); if (nn != null) { return(nn.GetRpcServer()); } IPEndPoint nnAddr = NameNodeHttpServer.GetNameNodeAddressFromContext(context); Configuration conf = new HdfsConfiguration(NameNodeHttpServer.GetConfFromContext( context)); return(NameNodeProxies.CreateProxy <ClientProtocol>(conf, NameNode.GetUri(nnAddr)) .GetProxy()); }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test node-group locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithNodeGroup() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 20% full long totalUsedSpace = totalCapacity * 2 / 10; TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / (numOfDatanodes / 2), (short)(numOfDatanodes / 2), 0); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on NODEGROUP2 cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancer(conf, totalUsedSpace, totalCapacity); } finally { cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> protected override GetUserMappingsProtocol GetUgmProtocol() { return(NameNodeProxies.CreateProxy <GetUserMappingsProtocol>(GetConf(), FileSystem .GetDefaultUri(GetConf())).GetProxy()); }