public virtual void TestQueues()
{
DatanodeDescriptor fakeDN = DFSTestUtil.GetLocalDatanodeDescriptor();
DatanodeStorage storage = new DatanodeStorage("STORAGE_ID");
DatanodeStorageInfo storageInfo = new DatanodeStorageInfo(fakeDN, storage);
msgs.EnqueueReportedBlock(storageInfo, block1Gs1, HdfsServerConstants.ReplicaState
.Finalized);
msgs.EnqueueReportedBlock(storageInfo, block1Gs2, HdfsServerConstants.ReplicaState
.Finalized);
NUnit.Framework.Assert.AreEqual(2, msgs.Count());
// Nothing queued yet for block 2
NUnit.Framework.Assert.IsNull(msgs.TakeBlockQueue(block2Gs1));
NUnit.Framework.Assert.AreEqual(2, msgs.Count());
Queue <PendingDataNodeMessages.ReportedBlockInfo> q = msgs.TakeBlockQueue(block1Gs2DifferentInstance
);
NUnit.Framework.Assert.AreEqual("ReportedBlockInfo [block=blk_1_1, dn=127.0.0.1:50010, reportedState=FINALIZED],"
+ "ReportedBlockInfo [block=blk_1_2, dn=127.0.0.1:50010, reportedState=FINALIZED]"
, Joiner.On(",").Join(q));
NUnit.Framework.Assert.AreEqual(0, msgs.Count());
// Should be null if we pull again
NUnit.Framework.Assert.IsNull(msgs.TakeBlockQueue(block1Gs1));
NUnit.Framework.Assert.AreEqual(0, msgs.Count());
}
internal virtual DatanodeStorageInfo UpdateStorage(DatanodeStorage s)
{
lock (storageMap)
{
DatanodeStorageInfo storage = storageMap[s.GetStorageID()];
if (storage == null)
{
Log.Info("Adding new storage ID " + s.GetStorageID() + " for DN " + GetXferAddr()
);
storage = new DatanodeStorageInfo(this, s);
storageMap[s.GetStorageID()] = storage;
}
else
{
if (storage.GetState() != s.GetState() || storage.GetStorageType() != s.GetStorageType
())
{
// For backwards compatibility, make sure that the type and
// state are updated. Some reports from older datanodes do
// not include these fields so we may have assumed defaults.
storage.UpdateFromStorage(s);
storageMap[storage.GetStorageID()] = storage;
}
}
return(storage);
}
}
internal DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s)
{
// The ID of the last full block report which updated this storage.
this.dn = dn;
this.storageID = s.GetStorageID();
this.storageType = s.GetStorageType();
this.state = s.GetState();
}
private void ValidateStorageState(StorageReport[] storageReports, DatanodeStorage.State
state)
{
foreach (StorageReport storageReport in storageReports)
{
DatanodeStorage storage = storageReport.GetStorage();
Assert.AssertThat(storage.GetState(), CoreMatchers.Is(state));
}
}
private static StorageReceivedDeletedBlocks[] MakeReportForReceivedBlock(Block block
, DatanodeStorage storage)
{
ReceivedDeletedBlockInfo[] receivedBlocks = new ReceivedDeletedBlockInfo[1];
receivedBlocks[0] = new ReceivedDeletedBlockInfo(block, ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, null);
StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[1];
reports[0] = new StorageReceivedDeletedBlocks(storage, receivedBlocks);
return(reports);
}
public virtual void TestConvertDatanodeStorage()
{
DatanodeStorage dns1 = new DatanodeStorage("id1", DatanodeStorage.State.Normal, StorageType
.Ssd);
HdfsProtos.DatanodeStorageProto proto = PBHelper.Convert(dns1);
DatanodeStorage dns2 = PBHelper.Convert(proto);
Compare(dns1, dns2);
}
public static DatanodeDescriptor GetLocalDatanodeDescriptor(bool initializeStorage
)
{
DatanodeDescriptor dn = new DatanodeDescriptor(DFSTestUtil.GetLocalDatanodeID());
if (initializeStorage)
{
dn.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
}
return(dn);
}
// Generate a block report, optionally corrupting the generation
// stamp and/or length of one block.
private static StorageBlockReport[] GetBlockReports(DataNode dn, string bpid, bool
corruptOneBlockGs, bool corruptOneBlockLen)
{
IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset
().GetBlockReports(bpid);
// Send block report
StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count];
bool corruptedGs = false;
bool corruptedLen = false;
int reportIndex = 0;
foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists)
{
DatanodeStorage dnStorage = kvPair.Key;
BlockListAsLongs blockList = kvPair.Value;
// Walk the list of blocks until we find one each to corrupt the
// generation stamp and length, if so requested.
BlockListAsLongs.Builder builder = BlockListAsLongs.Builder();
foreach (BlockListAsLongs.BlockReportReplica block in blockList)
{
if (corruptOneBlockGs && !corruptedGs)
{
long gsOld = block.GetGenerationStamp();
long gsNew;
do
{
gsNew = rand.Next();
}while (gsNew == gsOld);
block.SetGenerationStamp(gsNew);
Log.Info("Corrupted the GS for block ID " + block);
corruptedGs = true;
}
else
{
if (corruptOneBlockLen && !corruptedLen)
{
long lenOld = block.GetNumBytes();
long lenNew;
do
{
lenNew = rand.Next((int)lenOld - 1);
}while (lenNew == lenOld);
block.SetNumBytes(lenNew);
Log.Info("Corrupted the length for block ID " + block);
corruptedLen = true;
}
}
builder.Add(new BlockListAsLongs.BlockReportReplica(block));
}
reports[reportIndex++] = new StorageBlockReport(dnStorage, builder.Build());
}
return(reports);
}
public static DatanodeDescriptor GetDatanodeDescriptor(string ipAddr, string rackLocation
, DatanodeStorage storage, string hostname)
{
DatanodeDescriptor dn = DFSTestUtil.GetDatanodeDescriptor(ipAddr, DFSConfigKeys.DfsDatanodeDefaultPort
, rackLocation, hostname);
if (storage != null)
{
dn.UpdateStorage(storage);
}
return(dn);
}
public static StorageReport[] GetStorageReportsForDatanode(DatanodeDescriptor dnd
)
{
AList <StorageReport> reports = new AList <StorageReport>();
foreach (DatanodeStorageInfo storage in dnd.GetStorageInfos())
{
DatanodeStorage dns = new DatanodeStorage(storage.GetStorageID(), storage.GetState
(), storage.GetStorageType());
StorageReport report = new StorageReport(dns, false, storage.GetCapacity(), storage
.GetDfsUsed(), storage.GetRemaining(), storage.GetBlockPoolUsed());
reports.AddItem(report);
}
return(Sharpen.Collections.ToArray(reports, StorageReport.EmptyArray));
}
/// <exception cref="System.Exception"/>
private void DoTestOneOfTwoRacksDecommissioned(int testIndex)
{
// Block originally on A1, A2, B1
IList <DatanodeStorageInfo> origStorages = GetStorages(0, 1, 3);
IList <DatanodeDescriptor> origNodes = GetNodes(origStorages);
BlockInfoContiguous blockInfo = AddBlockOnNodes(testIndex, origNodes);
// Decommission all of the nodes in rack A
IList <DatanodeDescriptor> decomNodes = StartDecommission(0, 1, 2);
DatanodeStorageInfo[] pipeline = ScheduleSingleReplication(blockInfo);
NUnit.Framework.Assert.IsTrue("Source of replication should be one of the nodes the block "
+ "was on. Was: " + pipeline[0], origStorages.Contains(pipeline[0]));
// Only up to two nodes can be picked per rack when there are two racks.
NUnit.Framework.Assert.AreEqual("Should have two targets", 2, pipeline.Length);
bool foundOneOnRackB = false;
for (int i = 1; i < pipeline.Length; i++)
{
DatanodeDescriptor target = pipeline[i].GetDatanodeDescriptor();
if (rackB.Contains(target))
{
foundOneOnRackB = true;
}
NUnit.Framework.Assert.IsFalse(decomNodes.Contains(target));
NUnit.Framework.Assert.IsFalse(origNodes.Contains(target));
}
NUnit.Framework.Assert.IsTrue("Should have at least one target on rack B. Pipeline: "
+ Joiner.On(",").Join(pipeline), foundOneOnRackB);
// Mark the block as received on the target nodes in the pipeline
FulfillPipeline(blockInfo, pipeline);
// the block is still under-replicated. Add a new node. This should allow
// the third off-rack replica.
DatanodeDescriptor rackCNode = DFSTestUtil.GetDatanodeDescriptor("7.7.7.7", "/rackC"
);
rackCNode.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
AddNodes(ImmutableList.Of(rackCNode));
try
{
DatanodeStorageInfo[] pipeline2 = ScheduleSingleReplication(blockInfo);
NUnit.Framework.Assert.AreEqual(2, pipeline2.Length);
NUnit.Framework.Assert.AreEqual(rackCNode, pipeline2[1].GetDatanodeDescriptor());
}
finally
{
RemoveNode(rackCNode);
}
}
/// <exception cref="System.IO.IOException"/>
public void VerifyClusterPostUpgrade(MiniDFSCluster cluster)
{
// Verify that a GUID-based storage ID was generated.
string bpid = cluster.GetNamesystem().GetBlockPoolId();
StorageReport[] reports = cluster.GetDataNodes()[0].GetFSDataset().GetStorageReports
(bpid);
Assert.AssertThat(reports.Length, IS.Is(1));
string storageID = reports[0].GetStorage().GetStorageID();
NUnit.Framework.Assert.IsTrue(DatanodeStorage.IsValidStorageId(storageID));
if (expectedStorageId != null)
{
Assert.AssertThat(storageID, IS.Is(expectedStorageId));
}
}
public virtual void TestBlockHasMultipleReplicasOnSameDN()
{
string filename = MakeFileName(GenericTestUtils.GetMethodName());
Path filePath = new Path(filename);
// Write out a file with a few blocks.
DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize,
NumDatanodes, seed);
// Get the block list for the file with the block locations.
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize
* NumBlocks);
// Generate a fake block report from one of the DataNodes, such
// that it reports one copy of each block on either storage.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid);
StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode
()];
AList <Replica> blocks = new AList <Replica>();
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
Block localBlock = locatedBlock.GetBlock().GetLocalBlock();
blocks.AddItem(new FinalizedReplica(localBlock, null, null));
}
BlockListAsLongs bll = BlockListAsLongs.Encode(blocks);
for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i)
{
FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i];
DatanodeStorage dns = new DatanodeStorage(v.GetStorageID());
reports[i] = new StorageBlockReport(dns, bll);
}
// Should not assert!
cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext
(1, 0, Runtime.NanoTime()));
// Get the block locations once again.
locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks);
// Make sure that each block has two replicas, one on each DataNode.
foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks())
{
DatanodeInfo[] locations = locatedBlock_1.GetLocations();
Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes));
Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid
()));
}
}
/// <summary>
/// Verify that the NameNode can learn about new storages from incremental
/// block reports.
/// </summary>
/// <remarks>
/// Verify that the NameNode can learn about new storages from incremental
/// block reports.
/// This tests the fix for the error condition seen in HDFS-6904.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestNnLearnsNewStorages()
{
// Generate a report for a fake block on a fake storage.
string newStorageUuid = UUID.RandomUUID().ToString();
DatanodeStorage newStorage = new DatanodeStorage(newStorageUuid);
StorageReceivedDeletedBlocks[] reports = MakeReportForReceivedBlock(GetDummyBlock
(), newStorage);
// Send the report to the NN.
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, reports);
// Make sure that the NN has learned of the new storage.
DatanodeStorageInfo storageInfo = cluster.GetNameNode().GetNamesystem().GetBlockManager
().GetDatanodeManager().GetDatanode(dn0.GetDatanodeId()).GetStorageInfo(newStorageUuid
);
NUnit.Framework.Assert.IsNotNull(storageInfo);
}
public virtual void TestDatanodeDetect()
{
AtomicReference <DatanodeProtocolProtos.BlockReportRequestProto> request = new AtomicReference
<DatanodeProtocolProtos.BlockReportRequestProto>();
// just capture the outgoing PB
DatanodeProtocolPB mockProxy = Org.Mockito.Mockito.Mock <DatanodeProtocolPB>();
Org.Mockito.Mockito.DoAnswer(new _Answer_205(request)).When(mockProxy).BlockReport
(Matchers.Any <RpcController>(), Matchers.Any <DatanodeProtocolProtos.BlockReportRequestProto
>());
DatanodeProtocolClientSideTranslatorPB nn = new DatanodeProtocolClientSideTranslatorPB
(mockProxy);
DatanodeRegistration reg = DFSTestUtil.GetLocalDatanodeRegistration();
NamespaceInfo nsInfo = new NamespaceInfo(1, "cluster", "bp", 1);
reg.SetNamespaceInfo(nsInfo);
Replica r = new FinalizedReplica(new Block(1, 2, 3), null, null);
BlockListAsLongs bbl = BlockListAsLongs.Encode(Sharpen.Collections.Singleton(r));
DatanodeStorage storage = new DatanodeStorage("s1");
StorageBlockReport[] sbr = new StorageBlockReport[] { new StorageBlockReport(storage
, bbl) };
// check DN sends new-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.StorageBlockReportBuffers.GetMask
());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
DatanodeProtocolProtos.BlockReportRequestProto proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksBuffersList().IsEmpty
());
// back up to prior version and check DN sends old-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.Unknown.GetMask());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksBuffersList().IsEmpty(
));
}
public virtual void UpdateFromStorage(DatanodeStorage storage)
{
state = storage.GetState();
storageType = storage.GetStorageType();
}
public virtual void TestVolumeFailure()
{
System.Console.Out.WriteLine("Data dir: is " + dataDir.GetPath());
// Data dir structure is dataDir/data[1-4]/[current,tmp...]
// data1,2 is for datanode 1, data2,3 - datanode2
string filename = "/test.txt";
Path filePath = new Path(filename);
// we use only small number of blocks to avoid creating subdirs in the data dir..
int filesize = block_size * blocks_num;
DFSTestUtil.CreateFile(fs, filePath, filesize, repl, 1L);
DFSTestUtil.WaitReplication(fs, filePath, repl);
System.Console.Out.WriteLine("file " + filename + "(size " + filesize + ") is created and replicated"
);
// fail the volume
// delete/make non-writable one of the directories (failed volume)
data_fail = new FilePath(dataDir, "data3");
failedDir = MiniDFSCluster.GetFinalizedDir(dataDir, cluster.GetNamesystem().GetBlockPoolId
());
if (failedDir.Exists() && !DeteteBlocks(failedDir))
{
//!FileUtil.fullyDelete(failedDir)
throw new IOException("Could not delete hdfs directory '" + failedDir + "'");
}
data_fail.SetReadOnly();
failedDir.SetReadOnly();
System.Console.Out.WriteLine("Deleteing " + failedDir.GetPath() + "; exist=" + failedDir
.Exists());
// access all the blocks on the "failed" DataNode,
// we need to make sure that the "failed" volume is being accessed -
// and that will cause failure, blocks removal, "emergency" block report
TriggerFailure(filename, filesize);
// make sure a block report is sent
DataNode dn = cluster.GetDataNodes()[1];
//corresponds to dir data3
string bpid = cluster.GetNamesystem().GetBlockPoolId();
DatanodeRegistration dnR = dn.GetDNRegistrationForBP(bpid);
IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset
().GetBlockReports(bpid);
// Send block report
StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count];
int reportIndex = 0;
foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists)
{
DatanodeStorage dnStorage = kvPair.Key;
BlockListAsLongs blockList = kvPair.Value;
reports[reportIndex++] = new StorageBlockReport(dnStorage, blockList);
}
cluster.GetNameNodeRpc().BlockReport(dnR, bpid, reports, null);
// verify number of blocks and files...
Verify(filename, filesize);
// create another file (with one volume failed).
System.Console.Out.WriteLine("creating file test1.txt");
Path fileName1 = new Path("/test1.txt");
DFSTestUtil.CreateFile(fs, fileName1, filesize, repl, 1L);
// should be able to replicate to both nodes (2 DN, repl=2)
DFSTestUtil.WaitReplication(fs, fileName1, repl);
System.Console.Out.WriteLine("file " + fileName1.GetName() + " is created and replicated"
);
}
internal virtual void Compare(DatanodeStorage dns1, DatanodeStorage dns2)
{
Assert.AssertThat(dns2.GetStorageID(), CoreMatchers.Is(dns1.GetStorageID()));
Assert.AssertThat(dns2.GetState(), CoreMatchers.Is(dns1.GetState()));
Assert.AssertThat(dns2.GetStorageType(), CoreMatchers.Is(dns1.GetStorageType()));
}
public virtual void TestHeartbeat()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager
();
string poolId = namesystem.GetBlockPoolId();
DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg);
string storageID = DatanodeStorage.GenerateUuid();
dd.UpdateStorage(new DatanodeStorage(storageID));
int RemainingBlocks = 1;
int MaxReplicateLimit = conf.GetInt(DFSConfigKeys.DfsNamenodeReplicationMaxStreamsKey
, 2);
int MaxInvalidateLimit = DFSConfigKeys.DfsBlockInvalidateLimitDefault;
int MaxInvalidateBlocks = 2 * MaxInvalidateLimit + RemainingBlocks;
int MaxReplicateBlocks = 2 * MaxReplicateLimit + RemainingBlocks;
DatanodeStorageInfo[] OneTarget = new DatanodeStorageInfo[] { dd.GetStorageInfo(storageID
) };
try
{
namesystem.WriteLock();
lock (hm)
{
for (int i = 0; i < MaxReplicateBlocks; i++)
{
dd.AddBlockToBeReplicated(new Block(i, 0, GenerationStamp.LastReservedStamp), OneTarget
);
}
DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks
().Length);
AList <Block> blockList = new AList <Block>(MaxInvalidateBlocks);
for (int i_1 = 0; i_1 < MaxInvalidateBlocks; i_1++)
{
blockList.AddItem(new Block(i_1, 0, GenerationStamp.LastReservedStamp));
}
dd.AddBlocksToBeInvalidated(blockList);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(2, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks
().Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction
());
NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(2, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks
().Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction
());
NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[0].GetAction
());
NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(0, cmds.Length);
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestHeartbeatBlockRecovery()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager
();
string poolId = namesystem.GetBlockPoolId();
DatanodeRegistration nodeReg1 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[0], poolId);
DatanodeDescriptor dd1 = NameNodeAdapter.GetDatanode(namesystem, nodeReg1);
dd1.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
DatanodeRegistration nodeReg2 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[1], poolId);
DatanodeDescriptor dd2 = NameNodeAdapter.GetDatanode(namesystem, nodeReg2);
dd2.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
DatanodeRegistration nodeReg3 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[2], poolId);
DatanodeDescriptor dd3 = NameNodeAdapter.GetDatanode(namesystem, nodeReg3);
dd3.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
try
{
namesystem.WriteLock();
lock (hm)
{
NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem);
NameNodeAdapter.SendHeartBeat(nodeReg2, dd2, namesystem);
NameNodeAdapter.SendHeartBeat(nodeReg3, dd3, namesystem);
// Test with all alive nodes.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0);
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, 0);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0);
DatanodeStorageInfo[] storages = new DatanodeStorageInfo[] { dd1.GetStorageInfos(
)[0], dd2.GetStorageInfos()[0], dd3.GetStorageInfos()[0] };
BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction
(new Block(0, 0, GenerationStamp.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState
.UnderRecovery, storages);
dd1.AddBlockToBeRecovered(blockInfo);
DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem)
.GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
BlockRecoveryCommand recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
DatanodeInfo[] recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length);
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2);
NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3);
// Test with one stale node.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0);
// More than the default stale interval of 30 seconds.
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0);
blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp
.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages
);
dd1.AddBlockToBeRecovered(blockInfo);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
NUnit.Framework.Assert.AreEqual(2, recoveringNodes.Length);
// dd2 is skipped.
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd3);
// Test with all stale node.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, -60 * 1000);
// More than the default stale interval of 30 seconds.
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, -80 * 1000);
blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp
.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages
);
dd1.AddBlockToBeRecovered(blockInfo);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
// Only dd1 is included since it heart beated and hence its not stale
// when the list of recovery blocks is constructed.
NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length);
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2);
NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3);
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
public static DatanodeStorageInfo NewDatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage
s)
{
return(new DatanodeStorageInfo(dn, s));
}
public static DatanodeDescriptor GetDatanodeDescriptor(string ipAddr, string rackLocation
, DatanodeStorage storage)
{
return(GetDatanodeDescriptor(ipAddr, rackLocation, storage, "host"));
}
public static DatanodeDescriptor GetDatanodeDescriptor(string ipAddr, string rackLocation
, bool initializeStorage)
{
return(GetDatanodeDescriptor(ipAddr, rackLocation, initializeStorage ? new DatanodeStorage
(DatanodeStorage.GenerateUuid()) : null));
}
public static DatanodeStorageInfo UpdateStorage(DatanodeDescriptor dn, DatanodeStorage
s)
{
return(dn.UpdateStorage(s));
}
