/// <summary>
        /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2)
        /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2.
        /// </summary>
        /// <remarks>
        /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2)
        /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. Fill the cluster
        /// to 60% and 3 replicas, so n2 and n3 will have replica for all blocks according
        /// to replica placement policy with NodeGroup. As a result, n2 and n3 will be
        /// filled with 80% (60% x 4 / 3), and no blocks can be migrated from n2 and n3
        /// to n0 or n1 as balancer policy with node group. Thus, we expect the balancer
        /// to end in 5 iterations without move block process.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestBalancerEndInNoMoveProgress()
        {
            Configuration conf = CreateConf();

            long[]   capacities     = new long[] { Capacity, Capacity, Capacity, Capacity };
            string[] racks          = new string[] { Rack0, Rack0, Rack1, Rack1 };
            string[] nodeGroups     = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 };
            int      numOfDatanodes = capacities.Length;

            NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
            NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length);
            MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
                                                                                           .Length).Racks(racks).SimulatedCapacities(capacities);
            MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
            cluster = new MiniDFSClusterWithNodeGroup(builder);
            try
            {
                cluster.WaitActive();
                client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
                                                                          0).GetUri()).GetProxy();
                long totalCapacity = TestBalancer.Sum(capacities);
                // fill up the cluster to be 60% full
                long totalUsedSpace = totalCapacity * 6 / 10;
                TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / 3, (short)(3), 0);
                // run balancer which can finish in 5 iterations with no block movement.
                RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
Exemplo n.º 2
0
        /// <exception cref="System.IO.IOException"/>
        public NameNodeConnector(string name, URI nameNodeUri, Path idPath, IList <Path> targetPaths
                                 , Configuration conf, int maxNotChangedIterations)
        {
            this.nameNodeUri = nameNodeUri;
            this.idPath      = idPath;
            this.targetPaths = targetPaths == null || targetPaths.IsEmpty() ? Arrays.AsList(new
                                                                                            Path("/")) : targetPaths;
            this.maxNotChangedIterations = maxNotChangedIterations;
            this.namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(conf, nameNodeUri).
                            GetProxy();
            this.client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, nameNodeUri, fallbackToSimpleAuth
                                                                       ).GetProxy();
            this.fs = (DistributedFileSystem)FileSystem.Get(nameNodeUri, conf);
            NamespaceInfo namespaceinfo = namenode.VersionRequest();

            this.blockpoolID = namespaceinfo.GetBlockPoolID();
            FsServerDefaults defaults = fs.GetServerDefaults(new Path("/"));

            this.keyManager = new KeyManager(blockpoolID, namenode, defaults.GetEncryptDataTransfer
                                                 (), conf);
            // if it is for test, we do not create the id file
            @out = CheckAndMarkRunning();
            if (@out == null)
            {
                // Exit if there is another one running.
                throw new IOException("Another " + name + " is running.");
            }
        }
Exemplo n.º 3
0
        /// <summary>
        /// Test a cluster with even distribution, then a new empty node is added to
        /// the cluster.
        /// </summary>
        /// <remarks>
        /// Test a cluster with even distribution, then a new empty node is added to
        /// the cluster. Test start a cluster with specified number of nodes, and fills
        /// it to be 30% full (with a single file replicated identically to all
        /// datanodes); It then adds one new empty node and starts balancing.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestBalancerWithHANameNodes()
        {
            Configuration conf = new HdfsConfiguration();

            TestBalancer.InitConf(conf);
            long newNodeCapacity = TestBalancer.Capacity;
            // new node's capacity
            string newNodeRack = TestBalancer.Rack2;

            // new node's rack
            // array of racks for original nodes in cluster
            string[] racks = new string[] { TestBalancer.Rack0, TestBalancer.Rack1 };
            // array of capacities of original nodes in cluster
            long[] capacities = new long[] { TestBalancer.Capacity, TestBalancer.Capacity };
            NUnit.Framework.Assert.AreEqual(capacities.Length, racks.Length);
            int numOfDatanodes = capacities.Length;

            MiniDFSNNTopology.NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1");
            nn1Conf.SetIpcPort(NameNode.DefaultPort);
            Configuration copiedConf = new Configuration(conf);

            cluster = new MiniDFSCluster.Builder(copiedConf).NnTopology(MiniDFSNNTopology.SimpleHATopology
                                                                            ()).NumDataNodes(capacities.Length).Racks(racks).SimulatedCapacities(capacities)
                      .Build();
            HATestUtil.SetFailoverConfigurations(cluster, conf);
            try
            {
                cluster.WaitActive();
                cluster.TransitionToActive(1);
                Sharpen.Thread.Sleep(500);
                client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, FileSystem.GetDefaultUri
                                                                          (conf)).GetProxy();
                long totalCapacity = TestBalancer.Sum(capacities);
                // fill up the cluster to be 30% full
                long totalUsedSpace = totalCapacity * 3 / 10;
                TestBalancer.CreateFile(cluster, TestBalancer.filePath, totalUsedSpace / numOfDatanodes
                                        , (short)numOfDatanodes, 1);
                // start up an empty node with the same capacity and on the same rack
                cluster.StartDataNodes(conf, 1, true, null, new string[] { newNodeRack }, new long
                                       [] { newNodeCapacity });
                totalCapacity += newNodeCapacity;
                TestBalancer.WaitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
                ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf);
                NUnit.Framework.Assert.AreEqual(1, namenodes.Count);
                NUnit.Framework.Assert.IsTrue(namenodes.Contains(HATestUtil.GetLogicalUri(cluster
                                                                                          )));
                int r = Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer.Run(namenodes, Balancer.Parameters
                                                                            .Default, conf);
                NUnit.Framework.Assert.AreEqual(ExitStatus.Success.GetExitCode(), r);
                TestBalancer.WaitForBalancer(totalUsedSpace, totalCapacity, client, cluster, Balancer.Parameters
                                             .Default);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
        /// <summary>
        /// Create a cluster with even distribution, and a new empty node is added to
        /// the cluster, then test rack locality for balancer policy.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestBalancerWithRackLocality()
        {
            Configuration conf = CreateConf();

            long[]   capacities     = new long[] { Capacity, Capacity };
            string[] racks          = new string[] { Rack0, Rack1 };
            string[] nodeGroups     = new string[] { Nodegroup0, Nodegroup1 };
            int      numOfDatanodes = capacities.Length;

            NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
            MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
                                                                                           .Length).Racks(racks).SimulatedCapacities(capacities);
            MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
            cluster = new MiniDFSClusterWithNodeGroup(builder);
            try
            {
                cluster.WaitActive();
                client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
                                                                          0).GetUri()).GetProxy();
                long totalCapacity = TestBalancer.Sum(capacities);
                // fill up the cluster to be 30% full
                long totalUsedSpace = totalCapacity * 3 / 10;
                long length         = totalUsedSpace / numOfDatanodes;
                TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0);
                LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length
                                                             );
                ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0
                                                                     );
                long   newCapacity  = Capacity;
                string newRack      = Rack1;
                string newNodeGroup = Nodegroup2;
                // start up an empty node with the same capacity and on the same rack
                cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
                                       { newCapacity }, new string[] { newNodeGroup });
                totalCapacity += newCapacity;
                // run balancer and validate results
                RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
                lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length);
                ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0);
                NUnit.Framework.Assert.AreEqual(before, after);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
Exemplo n.º 5
0
        /// <summary>
        /// Create a
        /// <see cref="NameNode"/>
        /// proxy from the current
        /// <see cref="Javax.Servlet.ServletContext"/>
        /// .
        /// </summary>
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual ClientProtocol CreateNameNodeProxy()
        {
            ServletContext context = GetServletContext();
            // if we are running in the Name Node, use it directly rather than via
            // rpc
            NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);

            if (nn != null)
            {
                return(nn.GetRpcServer());
            }
            IPEndPoint    nnAddr = NameNodeHttpServer.GetNameNodeAddressFromContext(context);
            Configuration conf   = new HdfsConfiguration(NameNodeHttpServer.GetConfFromContext(
                                                             context));

            return(NameNodeProxies.CreateProxy <ClientProtocol>(conf, NameNode.GetUri(nnAddr))
                   .GetProxy());
        }
        /// <summary>
        /// Create a cluster with even distribution, and a new empty node is added to
        /// the cluster, then test node-group locality for balancer policy.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestBalancerWithNodeGroup()
        {
            Configuration conf = CreateConf();

            long[]   capacities     = new long[] { Capacity, Capacity, Capacity, Capacity };
            string[] racks          = new string[] { Rack0, Rack0, Rack1, Rack1 };
            string[] nodeGroups     = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 };
            int      numOfDatanodes = capacities.Length;

            NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
            NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length);
            MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
                                                                                           .Length).Racks(racks).SimulatedCapacities(capacities);
            MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
            cluster = new MiniDFSClusterWithNodeGroup(builder);
            try
            {
                cluster.WaitActive();
                client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
                                                                          0).GetUri()).GetProxy();
                long totalCapacity = TestBalancer.Sum(capacities);
                // fill up the cluster to be 20% full
                long totalUsedSpace = totalCapacity * 2 / 10;
                TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / (numOfDatanodes / 2),
                                        (short)(numOfDatanodes / 2), 0);
                long   newCapacity  = Capacity;
                string newRack      = Rack1;
                string newNodeGroup = Nodegroup2;
                // start up an empty node with the same capacity and on NODEGROUP2
                cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
                                       { newCapacity }, new string[] { newNodeGroup });
                totalCapacity += newCapacity;
                // run balancer and validate results
                RunBalancer(conf, totalUsedSpace, totalCapacity);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
Exemplo n.º 7
0
 /// <exception cref="System.IO.IOException"/>
 protected override GetUserMappingsProtocol GetUgmProtocol()
 {
     return(NameNodeProxies.CreateProxy <GetUserMappingsProtocol>(GetConf(), FileSystem
                                                                  .GetDefaultUri(GetConf())).GetProxy());
 }