internal virtual bool ScheduleMoves4Block(Mover.StorageTypeDiff diff, LocatedBlock
lb)
{
IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb);
Sharpen.Collections.Shuffle(locations);
Dispatcher.DBlock db = this._enclosing.NewDBlock(lb.GetBlock().GetLocalBlock(), locations
);
foreach (StorageType t in diff.existing)
{
foreach (Mover.MLocation ml in locations)
{
Dispatcher.Source source = this._enclosing.storages.GetSource(ml);
if (ml.storageType == t && source != null)
{
// try to schedule one replica move.
if (this.ScheduleMoveReplica(db, source, diff.expected))
{
return(true);
}
}
}
}
return(false);
}
public virtual void TestScheduleSameBlock()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string file = "/testScheduleSameBlock/file";
{
FSDataOutputStream @out = dfs.Create(new Path(file));
@out.WriteChars("testScheduleSameBlock");
@out.Close();
}
Org.Apache.Hadoop.Hdfs.Server.Mover.Mover mover = NewMover(conf);
mover.Init();
Mover.Processor processor = new Mover.Processor(this);
LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb);
Mover.MLocation ml = locations[0];
Dispatcher.DBlock db = mover.NewDBlock(lb.GetBlock().GetLocalBlock(), locations);
IList <StorageType> storageTypes = new AList <StorageType>(Arrays.AsList(StorageType
.Default, StorageType.Default));
NUnit.Framework.Assert.IsTrue(processor.ScheduleMoveReplica(db, ml, storageTypes)
);
NUnit.Framework.Assert.IsFalse(processor.ScheduleMoveReplica(db, ml, storageTypes
));
}
finally
{
cluster.Shutdown();
}
}
internal virtual Dispatcher.DBlock NewDBlock(Block block, IList <Mover.MLocation>
locations)
{
Dispatcher.DBlock db = new Dispatcher.DBlock(block);
foreach (Mover.MLocation ml in locations)
{
Dispatcher.DDatanode.StorageGroup source = storages.GetSource(ml);
if (source != null)
{
db.AddLocation(source);
}
}
return(db);
}
/// <summary>Choose the target storage within same Datanode if possible.</summary>
internal virtual bool ChooseTargetInSameNode(Dispatcher.DBlock db, Dispatcher.Source
source, IList <StorageType> targetTypes)
{
foreach (StorageType t in targetTypes)
{
Dispatcher.DDatanode.StorageGroup target = this._enclosing.storages.GetTarget(source
.GetDatanodeInfo().GetDatanodeUuid(), t);
if (target == null)
{
continue;
}
Dispatcher.PendingMove pm = source.AddPendingMove(db, target);
if (pm != null)
{
this._enclosing.dispatcher.ExecutePendingMove(pm);
return(true);
}
}
return(false);
}
internal virtual bool ChooseTarget(Dispatcher.DBlock db, Dispatcher.Source source
, IList <StorageType> targetTypes, Matcher matcher)
{
NetworkTopology cluster = this._enclosing.dispatcher.GetCluster();
foreach (StorageType t in targetTypes)
{
foreach (Dispatcher.DDatanode.StorageGroup target in this._enclosing.storages.GetTargetStorages
(t))
{
if (matcher.Match(cluster, source.GetDatanodeInfo(), target.GetDatanodeInfo()))
{
Dispatcher.PendingMove pm = source.AddPendingMove(db, target);
if (pm != null)
{
this._enclosing.dispatcher.ExecutePendingMove(pm);
return(true);
}
}
}
}
return(false);
}
internal virtual bool ScheduleMoveReplica(Dispatcher.DBlock db, Dispatcher.Source
source, IList <StorageType> targetTypes)
{
// Match storage on the same node
if (this.ChooseTargetInSameNode(db, source, targetTypes))
{
return(true);
}
if (this._enclosing.dispatcher.GetCluster().IsNodeGroupAware())
{
if (this.ChooseTarget(db, source, targetTypes, Matcher.SameNodeGroup))
{
return(true);
}
}
// Then, match nodes on the same rack
if (this.ChooseTarget(db, source, targetTypes, Matcher.SameRack))
{
return(true);
}
// At last, match all remaining nodes
return(this.ChooseTarget(db, source, targetTypes, Matcher.AnyOther));
}
internal virtual bool ScheduleMoveReplica(Dispatcher.DBlock db, Mover.MLocation ml
, IList <StorageType> targetTypes)
{
Dispatcher.Source source = this._enclosing.storages.GetSource(ml);
return(source == null ? false : this.ScheduleMoveReplica(db, source, targetTypes));
}
