/// <exception cref="System.IO.IOException"/> internal virtual int AddSomeBlocks(SimulatedFSDataset fsdataset, int startingBlockId ) { int bytesAdded = 0; for (int i = startingBlockId; i < startingBlockId + Numblocks; ++i) { ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual // data written ReplicaInPipelineInterface bInfo = fsdataset.CreateRbw(StorageType.Default, b, false ).GetReplica(); ReplicaOutputStreams @out = bInfo.CreateStreams(true, DataChecksum.NewDataChecksum (DataChecksum.Type.Crc32, 512)); try { OutputStream dataOut = @out.GetDataOut(); NUnit.Framework.Assert.AreEqual(0, fsdataset.GetLength(b)); for (int j = 1; j <= BlockIdToLen(i); ++j) { dataOut.Write(j); NUnit.Framework.Assert.AreEqual(j, bInfo.GetBytesOnDisk()); // correct length even as we write bytesAdded++; } } finally { @out.Close(); } b.SetNumBytes(BlockIdToLen(i)); fsdataset.FinalizeBlock(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(i), fsdataset.GetLength(b)); } return(bytesAdded); }
public virtual void TestOpWrite() { int numDataNodes = 1; long BlockIdFudge = 128; Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes ).Build(); try { cluster.WaitActive(); string poolId = cluster.GetNamesystem().GetBlockPoolId(); datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId ); dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr()); FileSystem fileSys = cluster.GetFileSystem(); /* Test writing to finalized replicas */ Path file = new Path("dataprotocol.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); // get the first blockid for the file ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); // test PIPELINE_SETUP_CREATE on a finalized block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block" , true); // test PIPELINE_DATA_STREAMING on a finalized block TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage" , true); // test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block long newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS , "Cannot recover data streaming to a finalized replica", true); // test PIPELINE_SETUP_APPEND on an existing block newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica" , false); firstBlock.SetGenerationStamp(newGS); // test PIPELINE_SETUP_APPEND_RECOVERY on an existing block file = new Path("dataprotocol1.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Recover appending to a finalized replica", false); // test PIPELINE_CLOSE_RECOVERY on an existing block file = new Path("dataprotocol2.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica" , false); firstBlock.SetGenerationStamp(newGS); // Test writing to a new block. Don't choose the next sequential // block ID to avoid conflicting with IDs chosen by the NN. long newBlockId = firstBlock.GetBlockId() + BlockIdFudge; ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId , 0, firstBlock.GetGenerationStamp()); // test PIPELINE_SETUP_CREATE on a new block TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block" , false); // test PIPELINE_SETUP_STREAMING_RECOVERY on a new block newGS = newBlock.GetGenerationStamp() + 1; newBlock.SetBlockId(newBlock.GetBlockId() + 1); TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS, "Recover a new block", true); // test PIPELINE_SETUP_APPEND on a new block newGS = newBlock.GetGenerationStamp() + 1; TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block" , true); // test PIPELINE_SETUP_APPEND_RECOVERY on a new block newBlock.SetBlockId(newBlock.GetBlockId() + 1); newGS = newBlock.GetGenerationStamp() + 1; TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block" , true); /* Test writing to RBW replicas */ Path file1 = new Path("dataprotocol1.dat"); DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L); DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream() ); @out.Write(1); @out.Hflush(); FSDataInputStream @in = fileSys.Open(file1); firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); firstBlock.SetNumBytes(2L); try { // test PIPELINE_SETUP_CREATE on a RBW block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block" , true); // test PIPELINE_SETUP_APPEND on an existing block newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica" , true); // test PIPELINE_SETUP_APPEND on an existing block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Recover append to a RBW replica", false); firstBlock.SetGenerationStamp(newGS); // test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block file = new Path("dataprotocol2.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream()); @out.Write(1); @out.Hflush(); @in = fileSys.Open(file); firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); firstBlock.SetNumBytes(2L); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS , "Recover a RBW replica", false); } finally { IOUtils.CloseStream(@in); IOUtils.CloseStream(@out); } } finally { cluster.Shutdown(); } }