Beispiel #1
0
        /// <exception cref="System.IO.IOException"/>
        internal virtual void Init()
        {
            InitStoragePolicies();
            IList <DatanodeStorageReport> reports = dispatcher.Init();

            foreach (DatanodeStorageReport r in reports)
            {
                Dispatcher.DDatanode dn = dispatcher.NewDatanode(r.GetDatanodeInfo());
                foreach (StorageType t in StorageType.GetMovableTypes())
                {
                    Dispatcher.Source source = dn.AddSource(t, long.MaxValue, dispatcher);
                    long maxRemaining        = GetMaxRemaining(r, t);
                    Dispatcher.DDatanode.StorageGroup target = maxRemaining > 0L ? dn.AddTarget(t, maxRemaining
                                                                                                ) : null;
                    storages.Add(source, target);
                }
            }
        }
Beispiel #2
0
        /// <summary>
        /// Given a datanode storage set, build a network topology and decide
        /// over-utilized storages, above average utilized storages,
        /// below average utilized storages, and underutilized storages.
        /// </summary>
        /// <remarks>
        /// Given a datanode storage set, build a network topology and decide
        /// over-utilized storages, above average utilized storages,
        /// below average utilized storages, and underutilized storages.
        /// The input datanode storage set is shuffled in order to randomize
        /// to the storage matching later on.
        /// </remarks>
        /// <returns>the number of bytes needed to move in order to balance the cluster.</returns>
        private long Init(IList <DatanodeStorageReport> reports)
        {
            // compute average utilization
            foreach (DatanodeStorageReport r in reports)
            {
                policy.AccumulateSpaces(r);
            }
            policy.InitAvgUtilization();
            // create network topology and classify utilization collections:
            //   over-utilized, above-average, below-average and under-utilized.
            long overLoadedBytes  = 0L;
            long underLoadedBytes = 0L;

            foreach (DatanodeStorageReport r_1 in reports)
            {
                Dispatcher.DDatanode dn = dispatcher.NewDatanode(r_1.GetDatanodeInfo());
                foreach (StorageType t in StorageType.GetMovableTypes())
                {
                    double utilization = policy.GetUtilization(r_1, t);
                    if (utilization == null)
                    {
                        // datanode does not have such storage type
                        continue;
                    }
                    long   capacity        = GetCapacity(r_1, t);
                    double utilizationDiff = utilization - policy.GetAvgUtilization(t);
                    double thresholdDiff   = Math.Abs(utilizationDiff) - threshold;
                    long   maxSize2Move    = ComputeMaxSize2Move(capacity, GetRemaining(r_1, t), utilizationDiff
                                                                 , threshold);
                    Dispatcher.DDatanode.StorageGroup g;
                    if (utilizationDiff > 0)
                    {
                        Dispatcher.Source s = dn.AddSource(t, maxSize2Move, dispatcher);
                        if (thresholdDiff <= 0)
                        {
                            // within threshold
                            aboveAvgUtilized.AddItem(s);
                        }
                        else
                        {
                            overLoadedBytes += Precentage2bytes(thresholdDiff, capacity);
                            overUtilized.AddItem(s);
                        }
                        g = s;
                    }
                    else
                    {
                        g = dn.AddTarget(t, maxSize2Move);
                        if (thresholdDiff <= 0)
                        {
                            // within threshold
                            belowAvgUtilized.AddItem(g);
                        }
                        else
                        {
                            underLoadedBytes += Precentage2bytes(thresholdDiff, capacity);
                            underUtilized.AddItem(g);
                        }
                    }
                    dispatcher.GetStorageGroupMap().Put(g);
                }
            }
            LogUtilizationCollections();
            Preconditions.CheckState(dispatcher.GetStorageGroupMap().Size() == overUtilized.Count
                                     + underUtilized.Count + aboveAvgUtilized.Count + belowAvgUtilized.Count, "Mismatched number of storage groups"
                                     );
            // return number of bytes to be moved in order to make the cluster balanced
            return(Math.Max(overLoadedBytes, underLoadedBytes));
        }