public virtual void TestConvertExtendedBlock()
{
ExtendedBlock b = GetExtendedBlock();
HdfsProtos.ExtendedBlockProto bProto = PBHelper.Convert(b);
ExtendedBlock b1 = PBHelper.Convert(bProto);
NUnit.Framework.Assert.AreEqual(b, b1);
b.SetBlockId(-1);
bProto = PBHelper.Convert(b);
b1 = PBHelper.Convert(bProto);
NUnit.Framework.Assert.AreEqual(b, b1);
}
public virtual void TestDataTransferProtocol()
{
Random random = new Random();
int oneMil = 1024 * 1024;
Path file = new Path("dataprotocol.dat");
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
datanode = cluster.GetFileSystem().GetDataNodeStats(HdfsConstants.DatanodeReportType
.Live)[0];
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
int fileLen = Math.Min(conf.GetInt(DFSConfigKeys.DfsBlockSizeKey, 4096), 4096);
CreateFile(fileSys, file, fileLen);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
string poolId = firstBlock.GetBlockPoolId();
long newBlockId = firstBlock.GetBlockId() + 1;
recvBuf.Reset();
sendBuf.Reset();
// bad version
recvOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
sendOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
SendRecvData("Wrong Version", true);
// bad ops
sendBuf.Reset();
sendOut.WriteShort((short)DataTransferProtocol.DataTransferVersion);
sendOut.WriteByte(OP.WriteBlock.code - 1);
SendRecvData("Wrong Op Code", true);
/* Test OP_WRITE_BLOCK */
sendBuf.Reset();
DataChecksum badChecksum = Org.Mockito.Mockito.Spy(DefaultChecksum);
Org.Mockito.Mockito.DoReturn(-1).When(badChecksum).GetBytesPerChecksum();
WriteBlock(poolId, newBlockId, badChecksum);
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, null, recvOut);
SendRecvData("wrong bytesPerChecksum while writing", true);
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
PacketHeader hdr = new PacketHeader(4, 0, 100, false, -1 - random.Next(oneMil), false
);
// size of packet
// offset in block,
// seqno
// last packet
// bad datalen
hdr.Write(sendOut);
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Error) }).Write(recvOut);
SendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true);
// test for writing a valid zero size block
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
hdr = new PacketHeader(8, 0, 100, true, 0, false);
// size of packet
// OffsetInBlock
// sequencenumber
// lastPacketInBlock
// chunk length
hdr.Write(sendOut);
sendOut.WriteInt(0);
// zero checksum
sendOut.Flush();
//ok finally write a block with 0 len
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Success) }).Write(recvOut);
SendRecvData("Writing a zero len block blockid " + newBlockId, false);
/* Test OP_READ_BLOCK */
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.GetLocalBlock());
long blkid = blk.GetBlockId();
// bad block id
sendBuf.Reset();
recvBuf.Reset();
blk.SetBlockId(blkid - 1);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong block ID " + newBlockId + " for read", false);
// negative block start offset -1L
sendBuf.Reset();
blk.SetBlockId(blkid);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", -1L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative start-offset for read for block " + firstBlock.GetBlockId(
), false);
// bad block start offset
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", fileLen, fileLen,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong start-offset for reading block " + firstBlock.GetBlockId(), false
);
// negative length is ok. Datanode assumes we want to read the whole block.
recvBuf.Reset();
((DataTransferProtos.BlockOpResponseProto)DataTransferProtos.BlockOpResponseProto
.NewBuilder().SetStatus(DataTransferProtos.Status.Success).SetReadOpChecksumInfo
(DataTransferProtos.ReadOpChecksumInfoProto.NewBuilder().SetChecksum(DataTransferProtoUtil
.ToProto(DefaultChecksum)).SetChunkOffset(0L)).Build()).WriteDelimitedTo(recvOut
);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, -1L - random.
Next(oneMil), true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative length for reading block " + firstBlock.GetBlockId(), false
);
// length is more than size of block.
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, "opReadBlock " + firstBlock +
" received exception java.io.IOException: " + "Offset 0 and length 4097 don't match block "
+ firstBlock + " ( blockLen 4096 )", recvOut);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen + 1,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong length for reading block " + firstBlock.GetBlockId(), false);
//At the end of all this, read the file to make sure that succeeds finally.
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
ReadFile(fileSys, file, fileLen);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestOpWrite()
{
int numDataNodes = 1;
long BlockIdFudge = 128;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId
);
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test PIPELINE_SETUP_CREATE on a finalized block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block"
, true);
// test PIPELINE_DATA_STREAMING on a finalized block
TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage"
, true);
// test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block
long newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Cannot recover data streaming to a finalized replica", true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_APPEND_RECOVERY on an existing block
file = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover appending to a finalized replica", false);
// test PIPELINE_CLOSE_RECOVERY on an existing block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// Test writing to a new block. Don't choose the next sequential
// block ID to avoid conflicting with IDs chosen by the NN.
long newBlockId = firstBlock.GetBlockId() + BlockIdFudge;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
// test PIPELINE_SETUP_CREATE on a new block
TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block"
, false);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a new block
newGS = newBlock.GetGenerationStamp() + 1;
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS,
"Recover a new block", true);
// test PIPELINE_SETUP_APPEND on a new block
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block"
, true);
// test PIPELINE_SETUP_APPEND_RECOVERY on a new block
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block"
, true);
/* Test writing to RBW replicas */
Path file1 = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream()
);
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = fileSys.Open(file1);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
try
{
// test PIPELINE_SETUP_CREATE on a RBW block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover append to a RBW replica", false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Recover a RBW replica", false);
}
finally
{
IOUtils.CloseStream(@in);
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
