/// <exception cref="System.IO.IOException"/>
internal virtual void Init()
{
InitStoragePolicies();
IList <DatanodeStorageReport> reports = dispatcher.Init();
foreach (DatanodeStorageReport r in reports)
{
Dispatcher.DDatanode dn = dispatcher.NewDatanode(r.GetDatanodeInfo());
foreach (StorageType t in StorageType.GetMovableTypes())
{
Dispatcher.Source source = dn.AddSource(t, long.MaxValue, dispatcher);
long maxRemaining = GetMaxRemaining(r, t);
Dispatcher.DDatanode.StorageGroup target = maxRemaining > 0L ? dn.AddTarget(t, maxRemaining
) : null;
storages.Add(source, target);
}
}
}
/// <summary>
/// Given a datanode storage set, build a network topology and decide
/// over-utilized storages, above average utilized storages,
/// below average utilized storages, and underutilized storages.
/// </summary>
/// <remarks>
/// Given a datanode storage set, build a network topology and decide
/// over-utilized storages, above average utilized storages,
/// below average utilized storages, and underutilized storages.
/// The input datanode storage set is shuffled in order to randomize
/// to the storage matching later on.
/// </remarks>
/// <returns>the number of bytes needed to move in order to balance the cluster.</returns>
private long Init(IList <DatanodeStorageReport> reports)
{
// compute average utilization
foreach (DatanodeStorageReport r in reports)
{
policy.AccumulateSpaces(r);
}
policy.InitAvgUtilization();
// create network topology and classify utilization collections:
// over-utilized, above-average, below-average and under-utilized.
long overLoadedBytes = 0L;
long underLoadedBytes = 0L;
foreach (DatanodeStorageReport r_1 in reports)
{
Dispatcher.DDatanode dn = dispatcher.NewDatanode(r_1.GetDatanodeInfo());
foreach (StorageType t in StorageType.GetMovableTypes())
{
double utilization = policy.GetUtilization(r_1, t);
if (utilization == null)
{
// datanode does not have such storage type
continue;
}
long capacity = GetCapacity(r_1, t);
double utilizationDiff = utilization - policy.GetAvgUtilization(t);
double thresholdDiff = Math.Abs(utilizationDiff) - threshold;
long maxSize2Move = ComputeMaxSize2Move(capacity, GetRemaining(r_1, t), utilizationDiff
, threshold);
Dispatcher.DDatanode.StorageGroup g;
if (utilizationDiff > 0)
{
Dispatcher.Source s = dn.AddSource(t, maxSize2Move, dispatcher);
if (thresholdDiff <= 0)
{
// within threshold
aboveAvgUtilized.AddItem(s);
}
else
{
overLoadedBytes += Precentage2bytes(thresholdDiff, capacity);
overUtilized.AddItem(s);
}
g = s;
}
else
{
g = dn.AddTarget(t, maxSize2Move);
if (thresholdDiff <= 0)
{
// within threshold
belowAvgUtilized.AddItem(g);
}
else
{
underLoadedBytes += Precentage2bytes(thresholdDiff, capacity);
underUtilized.AddItem(g);
}
}
dispatcher.GetStorageGroupMap().Put(g);
}
}
LogUtilizationCollections();
Preconditions.CheckState(dispatcher.GetStorageGroupMap().Size() == overUtilized.Count
+ underUtilized.Count + aboveAvgUtilized.Count + belowAvgUtilized.Count, "Mismatched number of storage groups"
);
// return number of bytes to be moved in order to make the cluster balanced
return(Math.Max(overLoadedBytes, underLoadedBytes));
}
