/// <exception cref="System.IO.IOException"/> internal virtual void Init() { InitStoragePolicies(); IList <DatanodeStorageReport> reports = dispatcher.Init(); foreach (DatanodeStorageReport r in reports) { Dispatcher.DDatanode dn = dispatcher.NewDatanode(r.GetDatanodeInfo()); foreach (StorageType t in StorageType.GetMovableTypes()) { Dispatcher.Source source = dn.AddSource(t, long.MaxValue, dispatcher); long maxRemaining = GetMaxRemaining(r, t); Dispatcher.DDatanode.StorageGroup target = maxRemaining > 0L ? dn.AddTarget(t, maxRemaining ) : null; storages.Add(source, target); } } }
/// <summary> /// Given a datanode storage set, build a network topology and decide /// over-utilized storages, above average utilized storages, /// below average utilized storages, and underutilized storages. /// </summary> /// <remarks> /// Given a datanode storage set, build a network topology and decide /// over-utilized storages, above average utilized storages, /// below average utilized storages, and underutilized storages. /// The input datanode storage set is shuffled in order to randomize /// to the storage matching later on. /// </remarks> /// <returns>the number of bytes needed to move in order to balance the cluster.</returns> private long Init(IList <DatanodeStorageReport> reports) { // compute average utilization foreach (DatanodeStorageReport r in reports) { policy.AccumulateSpaces(r); } policy.InitAvgUtilization(); // create network topology and classify utilization collections: // over-utilized, above-average, below-average and under-utilized. long overLoadedBytes = 0L; long underLoadedBytes = 0L; foreach (DatanodeStorageReport r_1 in reports) { Dispatcher.DDatanode dn = dispatcher.NewDatanode(r_1.GetDatanodeInfo()); foreach (StorageType t in StorageType.GetMovableTypes()) { double utilization = policy.GetUtilization(r_1, t); if (utilization == null) { // datanode does not have such storage type continue; } long capacity = GetCapacity(r_1, t); double utilizationDiff = utilization - policy.GetAvgUtilization(t); double thresholdDiff = Math.Abs(utilizationDiff) - threshold; long maxSize2Move = ComputeMaxSize2Move(capacity, GetRemaining(r_1, t), utilizationDiff , threshold); Dispatcher.DDatanode.StorageGroup g; if (utilizationDiff > 0) { Dispatcher.Source s = dn.AddSource(t, maxSize2Move, dispatcher); if (thresholdDiff <= 0) { // within threshold aboveAvgUtilized.AddItem(s); } else { overLoadedBytes += Precentage2bytes(thresholdDiff, capacity); overUtilized.AddItem(s); } g = s; } else { g = dn.AddTarget(t, maxSize2Move); if (thresholdDiff <= 0) { // within threshold belowAvgUtilized.AddItem(g); } else { underLoadedBytes += Precentage2bytes(thresholdDiff, capacity); underUtilized.AddItem(g); } } dispatcher.GetStorageGroupMap().Put(g); } } LogUtilizationCollections(); Preconditions.CheckState(dispatcher.GetStorageGroupMap().Size() == overUtilized.Count + underUtilized.Count + aboveAvgUtilized.Count + belowAvgUtilized.Count, "Mismatched number of storage groups" ); // return number of bytes to be moved in order to make the cluster balanced return(Math.Max(overLoadedBytes, underLoadedBytes)); }