public virtual void TestConvertExtendedBlock() { ExtendedBlock b = GetExtendedBlock(); HdfsProtos.ExtendedBlockProto bProto = PBHelper.Convert(b); ExtendedBlock b1 = PBHelper.Convert(bProto); NUnit.Framework.Assert.AreEqual(b, b1); b.SetBlockId(-1); bProto = PBHelper.Convert(b); b1 = PBHelper.Convert(bProto); NUnit.Framework.Assert.AreEqual(b, b1); }
public virtual void TestDataTransferProtocol() { Random random = new Random(); int oneMil = 1024 * 1024; Path file = new Path("dataprotocol.dat"); int numDataNodes = 1; Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes ).Build(); try { cluster.WaitActive(); datanode = cluster.GetFileSystem().GetDataNodeStats(HdfsConstants.DatanodeReportType .Live)[0]; dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr()); FileSystem fileSys = cluster.GetFileSystem(); int fileLen = Math.Min(conf.GetInt(DFSConfigKeys.DfsBlockSizeKey, 4096), 4096); CreateFile(fileSys, file, fileLen); // get the first blockid for the file ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); string poolId = firstBlock.GetBlockPoolId(); long newBlockId = firstBlock.GetBlockId() + 1; recvBuf.Reset(); sendBuf.Reset(); // bad version recvOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1)); sendOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1)); SendRecvData("Wrong Version", true); // bad ops sendBuf.Reset(); sendOut.WriteShort((short)DataTransferProtocol.DataTransferVersion); sendOut.WriteByte(OP.WriteBlock.code - 1); SendRecvData("Wrong Op Code", true); /* Test OP_WRITE_BLOCK */ sendBuf.Reset(); DataChecksum badChecksum = Org.Mockito.Mockito.Spy(DefaultChecksum); Org.Mockito.Mockito.DoReturn(-1).When(badChecksum).GetBytesPerChecksum(); WriteBlock(poolId, newBlockId, badChecksum); recvBuf.Reset(); SendResponse(DataTransferProtos.Status.Error, null, null, recvOut); SendRecvData("wrong bytesPerChecksum while writing", true); sendBuf.Reset(); recvBuf.Reset(); WriteBlock(poolId, ++newBlockId, DefaultChecksum); PacketHeader hdr = new PacketHeader(4, 0, 100, false, -1 - random.Next(oneMil), false ); // size of packet // offset in block, // seqno // last packet // bad datalen hdr.Write(sendOut); SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut); new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled , DataTransferProtos.Status.Error) }).Write(recvOut); SendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true); // test for writing a valid zero size block sendBuf.Reset(); recvBuf.Reset(); WriteBlock(poolId, ++newBlockId, DefaultChecksum); hdr = new PacketHeader(8, 0, 100, true, 0, false); // size of packet // OffsetInBlock // sequencenumber // lastPacketInBlock // chunk length hdr.Write(sendOut); sendOut.WriteInt(0); // zero checksum sendOut.Flush(); //ok finally write a block with 0 len SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut); new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled , DataTransferProtos.Status.Success) }).Write(recvOut); SendRecvData("Writing a zero len block blockid " + newBlockId, false); /* Test OP_READ_BLOCK */ string bpid = cluster.GetNamesystem().GetBlockPoolId(); ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.GetLocalBlock()); long blkid = blk.GetBlockId(); // bad block id sendBuf.Reset(); recvBuf.Reset(); blk.SetBlockId(blkid - 1); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true , CachingStrategy.NewDefaultStrategy()); SendRecvData("Wrong block ID " + newBlockId + " for read", false); // negative block start offset -1L sendBuf.Reset(); blk.SetBlockId(blkid); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", -1L, fileLen, true , CachingStrategy.NewDefaultStrategy()); SendRecvData("Negative start-offset for read for block " + firstBlock.GetBlockId( ), false); // bad block start offset sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", fileLen, fileLen, true, CachingStrategy.NewDefaultStrategy()); SendRecvData("Wrong start-offset for reading block " + firstBlock.GetBlockId(), false ); // negative length is ok. Datanode assumes we want to read the whole block. recvBuf.Reset(); ((DataTransferProtos.BlockOpResponseProto)DataTransferProtos.BlockOpResponseProto .NewBuilder().SetStatus(DataTransferProtos.Status.Success).SetReadOpChecksumInfo (DataTransferProtos.ReadOpChecksumInfoProto.NewBuilder().SetChecksum(DataTransferProtoUtil .ToProto(DefaultChecksum)).SetChunkOffset(0L)).Build()).WriteDelimitedTo(recvOut ); sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, -1L - random. Next(oneMil), true, CachingStrategy.NewDefaultStrategy()); SendRecvData("Negative length for reading block " + firstBlock.GetBlockId(), false ); // length is more than size of block. recvBuf.Reset(); SendResponse(DataTransferProtos.Status.Error, null, "opReadBlock " + firstBlock + " received exception java.io.IOException: " + "Offset 0 and length 4097 don't match block " + firstBlock + " ( blockLen 4096 )", recvOut); sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen + 1, true, CachingStrategy.NewDefaultStrategy()); SendRecvData("Wrong length for reading block " + firstBlock.GetBlockId(), false); //At the end of all this, read the file to make sure that succeeds finally. sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true , CachingStrategy.NewDefaultStrategy()); ReadFile(fileSys, file, fileLen); } finally { cluster.Shutdown(); } }
public virtual void TestOpWrite() { int numDataNodes = 1; long BlockIdFudge = 128; Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes ).Build(); try { cluster.WaitActive(); string poolId = cluster.GetNamesystem().GetBlockPoolId(); datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId ); dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr()); FileSystem fileSys = cluster.GetFileSystem(); /* Test writing to finalized replicas */ Path file = new Path("dataprotocol.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); // get the first blockid for the file ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); // test PIPELINE_SETUP_CREATE on a finalized block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block" , true); // test PIPELINE_DATA_STREAMING on a finalized block TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage" , true); // test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block long newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS , "Cannot recover data streaming to a finalized replica", true); // test PIPELINE_SETUP_APPEND on an existing block newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica" , false); firstBlock.SetGenerationStamp(newGS); // test PIPELINE_SETUP_APPEND_RECOVERY on an existing block file = new Path("dataprotocol1.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Recover appending to a finalized replica", false); // test PIPELINE_CLOSE_RECOVERY on an existing block file = new Path("dataprotocol2.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica" , false); firstBlock.SetGenerationStamp(newGS); // Test writing to a new block. Don't choose the next sequential // block ID to avoid conflicting with IDs chosen by the NN. long newBlockId = firstBlock.GetBlockId() + BlockIdFudge; ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId , 0, firstBlock.GetGenerationStamp()); // test PIPELINE_SETUP_CREATE on a new block TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block" , false); // test PIPELINE_SETUP_STREAMING_RECOVERY on a new block newGS = newBlock.GetGenerationStamp() + 1; newBlock.SetBlockId(newBlock.GetBlockId() + 1); TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS, "Recover a new block", true); // test PIPELINE_SETUP_APPEND on a new block newGS = newBlock.GetGenerationStamp() + 1; TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block" , true); // test PIPELINE_SETUP_APPEND_RECOVERY on a new block newBlock.SetBlockId(newBlock.GetBlockId() + 1); newGS = newBlock.GetGenerationStamp() + 1; TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block" , true); /* Test writing to RBW replicas */ Path file1 = new Path("dataprotocol1.dat"); DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L); DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream() ); @out.Write(1); @out.Hflush(); FSDataInputStream @in = fileSys.Open(file1); firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); firstBlock.SetNumBytes(2L); try { // test PIPELINE_SETUP_CREATE on a RBW block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block" , true); // test PIPELINE_SETUP_APPEND on an existing block newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica" , true); // test PIPELINE_SETUP_APPEND on an existing block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Recover append to a RBW replica", false); firstBlock.SetGenerationStamp(newGS); // test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block file = new Path("dataprotocol2.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream()); @out.Write(1); @out.Hflush(); @in = fileSys.Open(file); firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); firstBlock.SetNumBytes(2L); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS , "Recover a RBW replica", false); } finally { IOUtils.CloseStream(@in); IOUtils.CloseStream(@out); } } finally { cluster.Shutdown(); } }