internal virtual bool ScheduleMoves4Block(Mover.StorageTypeDiff diff, LocatedBlock
lb)
{
IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb);
Sharpen.Collections.Shuffle(locations);
Dispatcher.DBlock db = this._enclosing.NewDBlock(lb.GetBlock().GetLocalBlock(), locations
);
foreach (StorageType t in diff.existing)
{
foreach (Mover.MLocation ml in locations)
{
Dispatcher.Source source = this._enclosing.storages.GetSource(ml);
if (ml.storageType == t && source != null)
{
// try to schedule one replica move.
if (this.ScheduleMoveReplica(db, source, diff.expected))
{
return(true);
}
}
}
}
return(false);
}
/// <returns>true if it is necessary to run another round of migration</returns>
private bool ProcessFile(string fullPath, HdfsLocatedFileStatus status)
{
byte policyId = status.GetStoragePolicy();
// currently we ignore files with unspecified storage policy
if (policyId == BlockStoragePolicySuite.IdUnspecified)
{
return(false);
}
BlockStoragePolicy policy = this._enclosing.blockStoragePolicies[policyId];
if (policy == null)
{
Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to get the storage policy of file "
+ fullPath);
return(false);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
LocatedBlocks locatedBlocks = status.GetBlockLocations();
bool hasRemaining = false;
bool lastBlkComplete = locatedBlocks.IsLastBlockComplete();
IList <LocatedBlock> lbs = locatedBlocks.GetLocatedBlocks();
for (int i = 0; i < lbs.Count; i++)
{
if (i == lbs.Count - 1 && !lastBlkComplete)
{
// last block is incomplete, skip it
continue;
}
LocatedBlock lb = lbs[i];
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
if (!diff.RemoveOverlap(true))
{
if (this.ScheduleMoves4Block(diff, lb))
{
hasRemaining |= (diff.existing.Count > 1 && diff.expected.Count > 1);
}
}
}
return(hasRemaining);
}
/// <exception cref="System.Exception"/>
private void VerifyFile(Path parent, HdfsFileStatus status, byte expectedPolicyId
)
{
HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus)status;
byte policyId = fileStatus.GetStoragePolicy();
BlockStoragePolicy policy = this.policies.GetPolicy(policyId);
if (expectedPolicyId != null)
{
NUnit.Framework.Assert.AreEqual(unchecked ((byte)expectedPolicyId), policy.GetId()
);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
foreach (LocatedBlock lb in fileStatus.GetBlockLocations().GetLocatedBlocks())
{
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
NUnit.Framework.Assert.IsTrue(fileStatus.GetFullName(parent.ToString()) + " with policy "
+ policy + " has non-empty overlap: " + diff + ", the corresponding block is "
+ lb.GetBlock().GetLocalBlock(), diff.RemoveOverlap(true));
}
}