public virtual void TestSingleList()
{
DatanodeDescriptor dn = new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost"
, "abcd", 5000, 5001, 5002, 5003));
CachedBlock[] blocks = new CachedBlock[] { new CachedBlock(0L, (short)1, true), new
CachedBlock(1L, (short)1, true), new CachedBlock(2L, (short)1, true) };
// check that lists are empty
NUnit.Framework.Assert.IsTrue("expected pending cached list to start off empty.",
!dn.GetPendingCached().GetEnumerator().HasNext());
NUnit.Framework.Assert.IsTrue("expected cached list to start off empty.", !dn.GetCached
().GetEnumerator().HasNext());
NUnit.Framework.Assert.IsTrue("expected pending uncached list to start off empty."
, !dn.GetPendingUncached().GetEnumerator().HasNext());
// add a block to the back
NUnit.Framework.Assert.IsTrue(dn.GetCached().AddItem(blocks[0]));
NUnit.Framework.Assert.IsTrue("expected pending cached list to still be empty.",
!dn.GetPendingCached().GetEnumerator().HasNext());
NUnit.Framework.Assert.AreEqual("failed to insert blocks[0]", blocks[0], dn.GetCached
().GetEnumerator().Next());
NUnit.Framework.Assert.IsTrue("expected pending uncached list to still be empty."
, !dn.GetPendingUncached().GetEnumerator().HasNext());
// add another block to the back
NUnit.Framework.Assert.IsTrue(dn.GetCached().AddItem(blocks[1]));
IEnumerator <CachedBlock> iter = dn.GetCached().GetEnumerator();
NUnit.Framework.Assert.AreEqual(blocks[0], iter.Next());
NUnit.Framework.Assert.AreEqual(blocks[1], iter.Next());
NUnit.Framework.Assert.IsTrue(!iter.HasNext());
// add a block to the front
NUnit.Framework.Assert.IsTrue(dn.GetCached().AddFirst(blocks[2]));
iter = dn.GetCached().GetEnumerator();
NUnit.Framework.Assert.AreEqual(blocks[2], iter.Next());
NUnit.Framework.Assert.AreEqual(blocks[0], iter.Next());
NUnit.Framework.Assert.AreEqual(blocks[1], iter.Next());
NUnit.Framework.Assert.IsTrue(!iter.HasNext());
// remove a block from the middle
NUnit.Framework.Assert.IsTrue(dn.GetCached().Remove(blocks[0]));
iter = dn.GetCached().GetEnumerator();
NUnit.Framework.Assert.AreEqual(blocks[2], iter.Next());
NUnit.Framework.Assert.AreEqual(blocks[1], iter.Next());
NUnit.Framework.Assert.IsTrue(!iter.HasNext());
// remove all blocks
dn.GetCached().Clear();
NUnit.Framework.Assert.IsTrue("expected cached list to be empty after clear.", !dn
.GetPendingCached().GetEnumerator().HasNext());
}
/// <summary>Scan through the cached block map.</summary>
/// <remarks>
/// Scan through the cached block map.
/// Any blocks which are under-replicated should be assigned new Datanodes.
/// Blocks that are over-replicated should be removed from Datanodes.
/// </remarks>
private void RescanCachedBlockMap()
{
for (IEnumerator <CachedBlock> cbIter = cachedBlocks.GetEnumerator(); cbIter.HasNext
();)
{
scannedBlocks++;
CachedBlock cblock = cbIter.Next();
IList <DatanodeDescriptor> pendingCached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type
.PendingCached);
IList <DatanodeDescriptor> cached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type
.Cached);
IList <DatanodeDescriptor> pendingUncached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type
.PendingUncached);
// Remove nodes from PENDING_UNCACHED if they were actually uncached.
for (IEnumerator <DatanodeDescriptor> iter = pendingUncached.GetEnumerator(); iter
.HasNext();)
{
DatanodeDescriptor datanode = iter.Next();
if (!cblock.IsInList(datanode.GetCached()))
{
Log.Trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because the DataNode uncached it."
, cblock.GetBlockId(), datanode.GetDatanodeUuid());
datanode.GetPendingUncached().Remove(cblock);
iter.Remove();
}
}
BlockInfoContiguous blockInfo = blockManager.GetStoredBlock(new Block(cblock.GetBlockId
()));
string reason = FindReasonForNotCaching(cblock, blockInfo);
int neededCached = 0;
if (reason != null)
{
Log.Trace("Block {}: can't cache block because it is {}", cblock.GetBlockId(), reason
);
}
else
{
neededCached = cblock.GetReplication();
}
int numCached = cached.Count;
if (numCached >= neededCached)
{
// If we have enough replicas, drop all pending cached.
for (IEnumerator <DatanodeDescriptor> iter_1 = pendingCached.GetEnumerator(); iter_1
.HasNext();)
{
DatanodeDescriptor datanode = iter_1.Next();
datanode.GetPendingCached().Remove(cblock);
iter_1.Remove();
Log.Trace("Block {}: removing from PENDING_CACHED for node {}" + "because we already have {} cached replicas and we only"
+ " need {}", cblock.GetBlockId(), datanode.GetDatanodeUuid(), numCached, neededCached
);
}
}
if (numCached < neededCached)
{
// If we don't have enough replicas, drop all pending uncached.
for (IEnumerator <DatanodeDescriptor> iter_1 = pendingUncached.GetEnumerator(); iter_1
.HasNext();)
{
DatanodeDescriptor datanode = iter_1.Next();
datanode.GetPendingUncached().Remove(cblock);
iter_1.Remove();
Log.Trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because we only have {} cached replicas and we need "
+ "{}", cblock.GetBlockId(), datanode.GetDatanodeUuid(), numCached, neededCached
);
}
}
int neededUncached = numCached - (pendingUncached.Count + neededCached);
if (neededUncached > 0)
{
AddNewPendingUncached(neededUncached, cblock, cached, pendingUncached);
}
else
{
int additionalCachedNeeded = neededCached - (numCached + pendingCached.Count);
if (additionalCachedNeeded > 0)
{
AddNewPendingCached(additionalCachedNeeded, cblock, cached, pendingCached);
}
}
if ((neededCached == 0) && pendingUncached.IsEmpty() && pendingCached.IsEmpty())
{
// we have nothing more to do with this block.
Log.Trace("Block {}: removing from cachedBlocks, since neededCached " + "== 0, and pendingUncached and pendingCached are empty."
, cblock.GetBlockId());
cbIter.Remove();
}
}
}