public virtual void TestDeadDatanode()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
dn.Shutdown();
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000);
DatanodeProtocol dnp = cluster.GetNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo
(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) };
StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[]
{ new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try
{
dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks);
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure blockReport from dead datanode is rejected with IOException
StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new
DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) };
try
{
dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime
()));
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage
(reg.GetDatanodeUuid()), false, 0, 0, 0, 0) };
DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmd.Length);
NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction
());
}
// Generate a block report, optionally corrupting the generation
// stamp and/or length of one block.
private static StorageBlockReport[] GetBlockReports(DataNode dn, string bpid, bool
corruptOneBlockGs, bool corruptOneBlockLen)
{
IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset
().GetBlockReports(bpid);
// Send block report
StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count];
bool corruptedGs = false;
bool corruptedLen = false;
int reportIndex = 0;
foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists)
{
DatanodeStorage dnStorage = kvPair.Key;
BlockListAsLongs blockList = kvPair.Value;
// Walk the list of blocks until we find one each to corrupt the
// generation stamp and length, if so requested.
BlockListAsLongs.Builder builder = BlockListAsLongs.Builder();
foreach (BlockListAsLongs.BlockReportReplica block in blockList)
{
if (corruptOneBlockGs && !corruptedGs)
{
long gsOld = block.GetGenerationStamp();
long gsNew;
do
{
gsNew = rand.Next();
}while (gsNew == gsOld);
block.SetGenerationStamp(gsNew);
Log.Info("Corrupted the GS for block ID " + block);
corruptedGs = true;
}
else
{
if (corruptOneBlockLen && !corruptedLen)
{
long lenOld = block.GetNumBytes();
long lenNew;
do
{
lenNew = rand.Next((int)lenOld - 1);
}while (lenNew == lenOld);
block.SetNumBytes(lenNew);
Log.Info("Corrupted the length for block ID " + block);
corruptedLen = true;
}
}
builder.Add(new BlockListAsLongs.BlockReportReplica(block));
}
reports[reportIndex++] = new StorageBlockReport(dnStorage, builder.Build());
}
return(reports);
}
/// <exception cref="System.IO.IOException"/>
protected internal override void SendBlockReports(DatanodeRegistration dnR, string
poolId, StorageBlockReport[] reports)
{
int i = 0;
foreach (StorageBlockReport report in reports)
{
Log.Info("Sending block report for storage " + report.GetStorage().GetStorageID()
);
StorageBlockReport[] singletonReport = new StorageBlockReport[] { report };
cluster.GetNameNodeRpc().BlockReport(dnR, poolId, singletonReport, new BlockReportContext
(reports.Length, i, Runtime.NanoTime()));
i++;
}
}
public virtual void TestBlockHasMultipleReplicasOnSameDN()
{
string filename = MakeFileName(GenericTestUtils.GetMethodName());
Path filePath = new Path(filename);
// Write out a file with a few blocks.
DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize,
NumDatanodes, seed);
// Get the block list for the file with the block locations.
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize
* NumBlocks);
// Generate a fake block report from one of the DataNodes, such
// that it reports one copy of each block on either storage.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid);
StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode
()];
AList <Replica> blocks = new AList <Replica>();
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
Block localBlock = locatedBlock.GetBlock().GetLocalBlock();
blocks.AddItem(new FinalizedReplica(localBlock, null, null));
}
BlockListAsLongs bll = BlockListAsLongs.Encode(blocks);
for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i)
{
FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i];
DatanodeStorage dns = new DatanodeStorage(v.GetStorageID());
reports[i] = new StorageBlockReport(dns, bll);
}
// Should not assert!
cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext
(1, 0, Runtime.NanoTime()));
// Get the block locations once again.
locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks);
// Make sure that each block has two replicas, one on each DataNode.
foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks())
{
DatanodeInfo[] locations = locatedBlock_1.GetLocations();
Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes));
Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid
()));
}
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual DatanodeProtocolProtos.BlockReportResponseProto BlockReport(RpcController
controller, DatanodeProtocolProtos.BlockReportRequestProto request)
{
DatanodeCommand cmd = null;
StorageBlockReport[] report = new StorageBlockReport[request.GetReportsCount()];
int index = 0;
foreach (DatanodeProtocolProtos.StorageBlockReportProto s in request.GetReportsList
())
{
BlockListAsLongs blocks;
if (s.HasNumberOfBlocks())
{
// new style buffer based reports
int num = (int)s.GetNumberOfBlocks();
Preconditions.CheckState(s.GetBlocksCount() == 0, "cannot send both blocks list and buffers"
);
blocks = BlockListAsLongs.DecodeBuffers(num, s.GetBlocksBuffersList());
}
else
{
blocks = BlockListAsLongs.DecodeLongs(s.GetBlocksList());
}
report[index++] = new StorageBlockReport(PBHelper.Convert(s.GetStorage()), blocks
);
}
try
{
cmd = impl.BlockReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId
(), report, request.HasContext() ? PBHelper.Convert(request.GetContext()) : null
);
}
catch (IOException e)
{
throw new ServiceException(e);
}
DatanodeProtocolProtos.BlockReportResponseProto.Builder builder = DatanodeProtocolProtos.BlockReportResponseProto
.NewBuilder();
if (cmd != null)
{
builder.SetCmd(PBHelper.Convert(cmd));
}
return((DatanodeProtocolProtos.BlockReportResponseProto)builder.Build());
}
public virtual void TestDatanodeDetect()
{
AtomicReference <DatanodeProtocolProtos.BlockReportRequestProto> request = new AtomicReference
<DatanodeProtocolProtos.BlockReportRequestProto>();
// just capture the outgoing PB
DatanodeProtocolPB mockProxy = Org.Mockito.Mockito.Mock <DatanodeProtocolPB>();
Org.Mockito.Mockito.DoAnswer(new _Answer_205(request)).When(mockProxy).BlockReport
(Matchers.Any <RpcController>(), Matchers.Any <DatanodeProtocolProtos.BlockReportRequestProto
>());
DatanodeProtocolClientSideTranslatorPB nn = new DatanodeProtocolClientSideTranslatorPB
(mockProxy);
DatanodeRegistration reg = DFSTestUtil.GetLocalDatanodeRegistration();
NamespaceInfo nsInfo = new NamespaceInfo(1, "cluster", "bp", 1);
reg.SetNamespaceInfo(nsInfo);
Replica r = new FinalizedReplica(new Block(1, 2, 3), null, null);
BlockListAsLongs bbl = BlockListAsLongs.Encode(Sharpen.Collections.Singleton(r));
DatanodeStorage storage = new DatanodeStorage("s1");
StorageBlockReport[] sbr = new StorageBlockReport[] { new StorageBlockReport(storage
, bbl) };
// check DN sends new-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.StorageBlockReportBuffers.GetMask
());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
DatanodeProtocolProtos.BlockReportRequestProto proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksBuffersList().IsEmpty
());
// back up to prior version and check DN sends old-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.Unknown.GetMask());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksBuffersList().IsEmpty(
));
}
public virtual void TestVolumeFailure()
{
System.Console.Out.WriteLine("Data dir: is " + dataDir.GetPath());
// Data dir structure is dataDir/data[1-4]/[current,tmp...]
// data1,2 is for datanode 1, data2,3 - datanode2
string filename = "/test.txt";
Path filePath = new Path(filename);
// we use only small number of blocks to avoid creating subdirs in the data dir..
int filesize = block_size * blocks_num;
DFSTestUtil.CreateFile(fs, filePath, filesize, repl, 1L);
DFSTestUtil.WaitReplication(fs, filePath, repl);
System.Console.Out.WriteLine("file " + filename + "(size " + filesize + ") is created and replicated"
);
// fail the volume
// delete/make non-writable one of the directories (failed volume)
data_fail = new FilePath(dataDir, "data3");
failedDir = MiniDFSCluster.GetFinalizedDir(dataDir, cluster.GetNamesystem().GetBlockPoolId
());
if (failedDir.Exists() && !DeteteBlocks(failedDir))
{
//!FileUtil.fullyDelete(failedDir)
throw new IOException("Could not delete hdfs directory '" + failedDir + "'");
}
data_fail.SetReadOnly();
failedDir.SetReadOnly();
System.Console.Out.WriteLine("Deleteing " + failedDir.GetPath() + "; exist=" + failedDir
.Exists());
// access all the blocks on the "failed" DataNode,
// we need to make sure that the "failed" volume is being accessed -
// and that will cause failure, blocks removal, "emergency" block report
TriggerFailure(filename, filesize);
// make sure a block report is sent
DataNode dn = cluster.GetDataNodes()[1];
//corresponds to dir data3
string bpid = cluster.GetNamesystem().GetBlockPoolId();
DatanodeRegistration dnR = dn.GetDNRegistrationForBP(bpid);
IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset
().GetBlockReports(bpid);
// Send block report
StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count];
int reportIndex = 0;
foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists)
{
DatanodeStorage dnStorage = kvPair.Key;
BlockListAsLongs blockList = kvPair.Value;
reports[reportIndex++] = new StorageBlockReport(dnStorage, blockList);
}
cluster.GetNameNodeRpc().BlockReport(dnR, bpid, reports, null);
// verify number of blocks and files...
Verify(filename, filesize);
// create another file (with one volume failed).
System.Console.Out.WriteLine("creating file test1.txt");
Path fileName1 = new Path("/test1.txt");
DFSTestUtil.CreateFile(fs, fileName1, filesize, repl, 1L);
// should be able to replicate to both nodes (2 DN, repl=2)
DFSTestUtil.WaitReplication(fs, fileName1, repl);
System.Console.Out.WriteLine("file " + fileName1.GetName() + " is created and replicated"
);
}
