/// <summary>Verify the support for decommissioning a datanode that is already dead.</summary>
/// <remarks>
/// Verify the support for decommissioning a datanode that is already dead.
/// Under this scenario the datanode should immediately be marked as
/// DECOMMISSIONED
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDecommissionDeadDN()
{
Logger log = Logger.GetLogger(typeof(DecommissionManager));
log.SetLevel(Level.Debug);
DatanodeID dnID = cluster.GetDataNodes()[0].GetDatanodeId();
string dnName = dnID.GetXferAddr();
MiniDFSCluster.DataNodeProperties stoppedDN = cluster.StopDataNode(0);
DFSTestUtil.WaitForDatanodeState(cluster, dnID.GetDatanodeUuid(), false, 30000);
FSNamesystem fsn = cluster.GetNamesystem();
DatanodeManager dm = fsn.GetBlockManager().GetDatanodeManager();
DatanodeDescriptor dnDescriptor = dm.GetDatanode(dnID);
DecommissionNode(fsn, localFileSys, dnName);
dm.RefreshNodes(conf);
BlockManagerTestUtil.RecheckDecommissionState(dm);
NUnit.Framework.Assert.IsTrue(dnDescriptor.IsDecommissioned());
// Add the node back
cluster.RestartDataNode(stoppedDN, true);
cluster.WaitActive();
// Call refreshNodes on FSNamesystem with empty exclude file to remove the
// datanode from decommissioning list and make it available again.
WriteConfigFile(localFileSys, excludeFile, null);
dm.RefreshNodes(conf);
}
private RemoteBlockReader(string file, string bpid, long blockId, DataInputStream
@in, DataChecksum checksum, bool verifyChecksum, long startOffset, long firstChunkOffset
, long bytesToRead, Peer peer, DatanodeID datanodeID, PeerCache peerCache)
: base(new Path("/" + Block.BlockFilePrefix + blockId + ":" + bpid + ":of:" + file
), 1, verifyChecksum, checksum.GetChecksumSize() > 0 ? checksum : null, checksum
.GetBytesPerChecksum(), checksum.GetChecksumSize())
{
// Path is used only for printing block and file information in debug
/*too non path-like?*/
this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr
()));
this.peer = peer;
this.datanodeID = datanodeID;
this.@in = @in;
this.checksum = checksum;
this.startOffset = Math.Max(startOffset, 0);
this.blockId = blockId;
// The total number of bytes that we need to transfer from the DN is
// the amount that the user wants (bytesToRead), plus the padding at
// the beginning in order to chunk-align. Note that the DN may elect
// to send more than this amount if the read starts/ends mid-chunk.
this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset);
this.firstChunkOffset = firstChunkOffset;
lastChunkOffset = firstChunkOffset;
lastChunkLen = -1;
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
this.peerCache = peerCache;
}
protected internal RemoteBlockReader2(string file, string bpid, long blockId, DataChecksum
checksum, bool verifyChecksum, long startOffset, long firstChunkOffset, long bytesToRead
, Peer peer, DatanodeID datanodeID, PeerCache peerCache)
{
this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr
()));
// Path is used only for printing block and file information in debug
this.peer = peer;
this.datanodeID = datanodeID;
this.@in = peer.GetInputStreamChannel();
this.checksum = checksum;
this.verifyChecksum = verifyChecksum;
this.startOffset = Math.Max(startOffset, 0);
this.filename = file;
this.peerCache = peerCache;
this.blockId = blockId;
// The total number of bytes that we need to transfer from the DN is
// the amount that the user wants (bytesToRead), plus the padding at
// the beginning in order to chunk-align. Note that the DN may elect
// to send more than this amount if the read starts/ends mid-chunk.
this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset);
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
}
public virtual void TestDataTransferProtocol()
{
Random random = new Random();
int oneMil = 1024 * 1024;
Path file = new Path("dataprotocol.dat");
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
datanode = cluster.GetFileSystem().GetDataNodeStats(HdfsConstants.DatanodeReportType
.Live)[0];
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
int fileLen = Math.Min(conf.GetInt(DFSConfigKeys.DfsBlockSizeKey, 4096), 4096);
CreateFile(fileSys, file, fileLen);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
string poolId = firstBlock.GetBlockPoolId();
long newBlockId = firstBlock.GetBlockId() + 1;
recvBuf.Reset();
sendBuf.Reset();
// bad version
recvOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
sendOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
SendRecvData("Wrong Version", true);
// bad ops
sendBuf.Reset();
sendOut.WriteShort((short)DataTransferProtocol.DataTransferVersion);
sendOut.WriteByte(OP.WriteBlock.code - 1);
SendRecvData("Wrong Op Code", true);
/* Test OP_WRITE_BLOCK */
sendBuf.Reset();
DataChecksum badChecksum = Org.Mockito.Mockito.Spy(DefaultChecksum);
Org.Mockito.Mockito.DoReturn(-1).When(badChecksum).GetBytesPerChecksum();
WriteBlock(poolId, newBlockId, badChecksum);
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, null, recvOut);
SendRecvData("wrong bytesPerChecksum while writing", true);
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
PacketHeader hdr = new PacketHeader(4, 0, 100, false, -1 - random.Next(oneMil), false
);
// size of packet
// offset in block,
// seqno
// last packet
// bad datalen
hdr.Write(sendOut);
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Error) }).Write(recvOut);
SendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true);
// test for writing a valid zero size block
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
hdr = new PacketHeader(8, 0, 100, true, 0, false);
// size of packet
// OffsetInBlock
// sequencenumber
// lastPacketInBlock
// chunk length
hdr.Write(sendOut);
sendOut.WriteInt(0);
// zero checksum
sendOut.Flush();
//ok finally write a block with 0 len
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Success) }).Write(recvOut);
SendRecvData("Writing a zero len block blockid " + newBlockId, false);
/* Test OP_READ_BLOCK */
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.GetLocalBlock());
long blkid = blk.GetBlockId();
// bad block id
sendBuf.Reset();
recvBuf.Reset();
blk.SetBlockId(blkid - 1);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong block ID " + newBlockId + " for read", false);
// negative block start offset -1L
sendBuf.Reset();
blk.SetBlockId(blkid);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", -1L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative start-offset for read for block " + firstBlock.GetBlockId(
), false);
// bad block start offset
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", fileLen, fileLen,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong start-offset for reading block " + firstBlock.GetBlockId(), false
);
// negative length is ok. Datanode assumes we want to read the whole block.
recvBuf.Reset();
((DataTransferProtos.BlockOpResponseProto)DataTransferProtos.BlockOpResponseProto
.NewBuilder().SetStatus(DataTransferProtos.Status.Success).SetReadOpChecksumInfo
(DataTransferProtos.ReadOpChecksumInfoProto.NewBuilder().SetChecksum(DataTransferProtoUtil
.ToProto(DefaultChecksum)).SetChunkOffset(0L)).Build()).WriteDelimitedTo(recvOut
);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, -1L - random.
Next(oneMil), true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative length for reading block " + firstBlock.GetBlockId(), false
);
// length is more than size of block.
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, "opReadBlock " + firstBlock +
" received exception java.io.IOException: " + "Offset 0 and length 4097 don't match block "
+ firstBlock + " ( blockLen 4096 )", recvOut);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen + 1,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong length for reading block " + firstBlock.GetBlockId(), false);
//At the end of all this, read the file to make sure that succeeds finally.
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
ReadFile(fileSys, file, fileLen);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestOpWrite()
{
int numDataNodes = 1;
long BlockIdFudge = 128;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId
);
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test PIPELINE_SETUP_CREATE on a finalized block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block"
, true);
// test PIPELINE_DATA_STREAMING on a finalized block
TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage"
, true);
// test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block
long newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Cannot recover data streaming to a finalized replica", true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_APPEND_RECOVERY on an existing block
file = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover appending to a finalized replica", false);
// test PIPELINE_CLOSE_RECOVERY on an existing block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// Test writing to a new block. Don't choose the next sequential
// block ID to avoid conflicting with IDs chosen by the NN.
long newBlockId = firstBlock.GetBlockId() + BlockIdFudge;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
// test PIPELINE_SETUP_CREATE on a new block
TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block"
, false);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a new block
newGS = newBlock.GetGenerationStamp() + 1;
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS,
"Recover a new block", true);
// test PIPELINE_SETUP_APPEND on a new block
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block"
, true);
// test PIPELINE_SETUP_APPEND_RECOVERY on a new block
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block"
, true);
/* Test writing to RBW replicas */
Path file1 = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream()
);
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = fileSys.Open(file1);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
try
{
// test PIPELINE_SETUP_CREATE on a RBW block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover append to a RBW replica", false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Recover a RBW replica", false);
}
finally
{
IOUtils.CloseStream(@in);
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestProcesOverReplicateBlock()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
Path fileName = new Path("/foo1");
DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)3);
// corrupt the block on datanode 0
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block));
MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0);
// remove block scanner log to trigger block scanning
FilePath scanCursor = new FilePath(new FilePath(MiniDFSCluster.GetFinalizedDir(cluster
.GetInstanceStorageDir(0, 0), cluster.GetNamesystem().GetBlockPoolId()).GetParent
()).GetParent(), "scanner.cursor");
//wait for one minute for deletion to succeed;
for (int i = 0; !scanCursor.Delete(); i++)
{
NUnit.Framework.Assert.IsTrue("Could not delete " + scanCursor.GetAbsolutePath()
+ " in one minute", i < 60);
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
}
// restart the datanode so the corrupt replica will be detected
cluster.RestartDataNode(dnProps);
DFSTestUtil.WaitReplication(fs, fileName, (short)2);
string blockPoolId = cluster.GetNamesystem().GetBlockPoolId();
DatanodeID corruptDataNode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[2], blockPoolId);
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
HeartbeatManager hm = bm.GetDatanodeManager().GetHeartbeatManager();
try
{
namesystem.WriteLock();
lock (hm)
{
// set live datanode's remaining space to be 0
// so they will be chosen to be deleted when over-replication occurs
string corruptMachineName = corruptDataNode.GetXferAddr();
foreach (DatanodeDescriptor datanode in hm.GetDatanodes())
{
if (!corruptMachineName.Equals(datanode.GetXferAddr()))
{
datanode.GetStorageInfos()[0].SetUtilizationForTesting(100L, 100L, 0, 100L);
datanode.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(datanode
), 0L, 0L, 0, 0, null);
}
}
// decrease the replication factor to 1;
NameNodeAdapter.SetReplication(namesystem, fileName.ToString(), (short)1);
// corrupt one won't be chosen to be excess one
// without 4910 the number of live replicas would be 0: block gets lost
NUnit.Framework.Assert.AreEqual(1, bm.CountNodes(block.GetLocalBlock()).LiveReplicas
());
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
