Exemplo n.º 1
0
        /// <summary>Balance all namenodes.</summary>
        /// <remarks>
        /// Balance all namenodes.
        /// For each iteration,
        /// for each namenode,
        /// execute a
        /// <see cref="Balancer"/>
        /// to work through all datanodes once.
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        internal static int Run(ICollection <URI> namenodes, Balancer.Parameters p, Configuration
                                conf)
        {
            long sleeptime = conf.GetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DFSConfigKeys
                                          .DfsHeartbeatIntervalDefault) * 2000 + conf.GetLong(DFSConfigKeys.DfsNamenodeReplicationIntervalKey
                                                                                              , DFSConfigKeys.DfsNamenodeReplicationIntervalDefault) * 1000;

            Log.Info("namenodes  = " + namenodes);
            Log.Info("parameters = " + p);
            System.Console.Out.WriteLine("Time Stamp               Iteration#  Bytes Already Moved  Bytes Left To Move  Bytes Being Moved"
                                         );
            IList <NameNodeConnector> connectors = Sharpen.Collections.EmptyList();

            try
            {
                connectors = NameNodeConnector.NewNameNodeConnectors(namenodes, typeof(Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer
                                                                                       ).Name, BalancerIdPath, conf, p.maxIdleIteration);
                bool done = false;
                for (int iteration = 0; !done; iteration++)
                {
                    done = true;
                    Sharpen.Collections.Shuffle(connectors);
                    foreach (NameNodeConnector nnc in connectors)
                    {
                        Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer b = new Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer
                                                                                (nnc, p, conf);
                        Balancer.Result r = b.RunOneIteration();
                        r.Print(iteration, System.Console.Out);
                        // clean all lists
                        b.ResetData(conf);
                        if (r.exitStatus == ExitStatus.InProgress)
                        {
                            done = false;
                        }
                        else
                        {
                            if (r.exitStatus != ExitStatus.Success)
                            {
                                //must be an error statue, return.
                                return(r.exitStatus.GetExitCode());
                            }
                        }
                    }
                    if (!done)
                    {
                        Sharpen.Thread.Sleep(sleeptime);
                    }
                }
            }
            finally
            {
                foreach (NameNodeConnector nnc in connectors)
                {
                    IOUtils.Cleanup(Log, nnc);
                }
            }
            return(ExitStatus.Success.GetExitCode());
        }
Exemplo n.º 2
0
        /// <summary>Construct a balancer.</summary>
        /// <remarks>
        /// Construct a balancer.
        /// Initialize balancer. It sets the value of the threshold, and
        /// builds the communication proxies to
        /// namenode as a client and a secondary namenode and retry proxies
        /// when connection fails.
        /// </remarks>
        internal Balancer(NameNodeConnector theblockpool, Balancer.Parameters p, Configuration
                          conf)
        {
            long movedWinWidth = conf.GetLong(DFSConfigKeys.DfsBalancerMovedwinwidthKey, DFSConfigKeys
                                              .DfsBalancerMovedwinwidthDefault);
            int moverThreads = conf.GetInt(DFSConfigKeys.DfsBalancerMoverthreadsKey, DFSConfigKeys
                                           .DfsBalancerMoverthreadsDefault);
            int dispatcherThreads = conf.GetInt(DFSConfigKeys.DfsBalancerDispatcherthreadsKey
                                                , DFSConfigKeys.DfsBalancerDispatcherthreadsDefault);
            int maxConcurrentMovesPerNode = conf.GetInt(DFSConfigKeys.DfsDatanodeBalanceMaxNumConcurrentMovesKey
                                                        , DFSConfigKeys.DfsDatanodeBalanceMaxNumConcurrentMovesDefault);

            this.dispatcher = new Dispatcher(theblockpool, p.nodesToBeIncluded, p.nodesToBeExcluded
                                             , movedWinWidth, moverThreads, dispatcherThreads, maxConcurrentMovesPerNode, conf
                                             );
            this.threshold = p.threshold;
            this.policy    = p.policy;
        }