public virtual void TestInjectionEmpty() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); int bytesAdded = AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } // Inject blocks into an empty fsdataset // - injecting the blocks we got above. SimulatedFSDataset sfsdataset = GetSimulatedFSDataset(); sfsdataset.InjectBlocks(bpid, blockReport); blockReport = sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b_1 in blockReport) { NUnit.Framework.Assert.IsNotNull(b_1); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes() ); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength (new ExtendedBlock(bpid, b_1))); } NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset .GetRemaining()); }
public virtual void TestInjectionNonEmpty() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); int bytesAdded = AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } fsdataset = null; // Inject blocks into an non-empty fsdataset // - injecting the blocks we got above. SimulatedFSDataset sfsdataset = GetSimulatedFSDataset(); // Add come blocks whose block ids do not conflict with // the ones we are going to inject. bytesAdded += AddSomeBlocks(sfsdataset, Numblocks + 1); sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); sfsdataset.InjectBlocks(bpid, blockReport); blockReport = sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks * 2, blockReport.GetNumberOfBlocks()); foreach (Block b_1 in blockReport) { NUnit.Framework.Assert.IsNotNull(b_1); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes() ); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength (new ExtendedBlock(bpid, b_1))); } NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset .GetRemaining()); // Now test that the dataset cannot be created if it does not have sufficient cap conf.SetLong(SimulatedFSDataset.ConfigPropertyCapacity, 10); try { sfsdataset = GetSimulatedFSDataset(); sfsdataset.AddBlockPool(bpid, conf); sfsdataset.InjectBlocks(bpid, blockReport); NUnit.Framework.Assert.IsTrue("Expected an IO exception", false); } catch (IOException) { } }
// Generate a block report, optionally corrupting the generation // stamp and/or length of one block. private static StorageBlockReport[] GetBlockReports(DataNode dn, string bpid, bool corruptOneBlockGs, bool corruptOneBlockLen) { IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset ().GetBlockReports(bpid); // Send block report StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count]; bool corruptedGs = false; bool corruptedLen = false; int reportIndex = 0; foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists) { DatanodeStorage dnStorage = kvPair.Key; BlockListAsLongs blockList = kvPair.Value; // Walk the list of blocks until we find one each to corrupt the // generation stamp and length, if so requested. BlockListAsLongs.Builder builder = BlockListAsLongs.Builder(); foreach (BlockListAsLongs.BlockReportReplica block in blockList) { if (corruptOneBlockGs && !corruptedGs) { long gsOld = block.GetGenerationStamp(); long gsNew; do { gsNew = rand.Next(); }while (gsNew == gsOld); block.SetGenerationStamp(gsNew); Log.Info("Corrupted the GS for block ID " + block); corruptedGs = true; } else { if (corruptOneBlockLen && !corruptedLen) { long lenOld = block.GetNumBytes(); long lenNew; do { lenNew = rand.Next((int)lenOld - 1); }while (lenNew == lenOld); block.SetNumBytes(lenNew); Log.Info("Corrupted the length for block ID " + block); corruptedLen = true; } } builder.Add(new BlockListAsLongs.BlockReportReplica(block)); } reports[reportIndex++] = new StorageBlockReport(dnStorage, builder.Build()); } return(reports); }
public virtual void TestGetBlockReport() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } }
public virtual void TestBlockHasMultipleReplicasOnSameDN() { string filename = MakeFileName(GenericTestUtils.GetMethodName()); Path filePath = new Path(filename); // Write out a file with a few blocks. DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize, NumDatanodes, seed); // Get the block list for the file with the block locations. LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize * NumBlocks); // Generate a fake block report from one of the DataNodes, such // that it reports one copy of each block on either storage. DataNode dn = cluster.GetDataNodes()[0]; DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid); StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode ()]; AList <Replica> blocks = new AList <Replica>(); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { Block localBlock = locatedBlock.GetBlock().GetLocalBlock(); blocks.AddItem(new FinalizedReplica(localBlock, null, null)); } BlockListAsLongs bll = BlockListAsLongs.Encode(blocks); for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i) { FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i]; DatanodeStorage dns = new DatanodeStorage(v.GetStorageID()); reports[i] = new StorageBlockReport(dns, bll); } // Should not assert! cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext (1, 0, Runtime.NanoTime())); // Get the block locations once again. locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks); // Make sure that each block has two replicas, one on each DataNode. foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks()) { DatanodeInfo[] locations = locatedBlock_1.GetLocations(); Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes)); Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid ())); } }
private void VerifyCapturedArguments(ArgumentCaptor <StorageBlockReport[]> captor, int expectedReportsPerCall, int expectedTotalBlockCount) { IList <StorageBlockReport[]> listOfReports = captor.GetAllValues(); int numBlocksReported = 0; foreach (StorageBlockReport[] reports in listOfReports) { Assert.AssertThat(reports.Length, IS.Is(expectedReportsPerCall)); foreach (StorageBlockReport report in reports) { BlockListAsLongs blockList = report.GetBlocks(); numBlocksReported += blockList.GetNumberOfBlocks(); } } System.Diagnostics.Debug.Assert((numBlocksReported >= expectedTotalBlockCount)); }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual DatanodeProtocolProtos.BlockReportResponseProto BlockReport(RpcController controller, DatanodeProtocolProtos.BlockReportRequestProto request) { DatanodeCommand cmd = null; StorageBlockReport[] report = new StorageBlockReport[request.GetReportsCount()]; int index = 0; foreach (DatanodeProtocolProtos.StorageBlockReportProto s in request.GetReportsList ()) { BlockListAsLongs blocks; if (s.HasNumberOfBlocks()) { // new style buffer based reports int num = (int)s.GetNumberOfBlocks(); Preconditions.CheckState(s.GetBlocksCount() == 0, "cannot send both blocks list and buffers" ); blocks = BlockListAsLongs.DecodeBuffers(num, s.GetBlocksBuffersList()); } else { blocks = BlockListAsLongs.DecodeLongs(s.GetBlocksList()); } report[index++] = new StorageBlockReport(PBHelper.Convert(s.GetStorage()), blocks ); } try { cmd = impl.BlockReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId (), report, request.HasContext() ? PBHelper.Convert(request.GetContext()) : null ); } catch (IOException e) { throw new ServiceException(e); } DatanodeProtocolProtos.BlockReportResponseProto.Builder builder = DatanodeProtocolProtos.BlockReportResponseProto .NewBuilder(); if (cmd != null) { builder.SetCmd(PBHelper.Convert(cmd)); } return((DatanodeProtocolProtos.BlockReportResponseProto)builder.Build()); }
/// <exception cref="System.IO.IOException"/> public virtual DatanodeCommand BlockReport(DatanodeRegistration registration, string poolId, StorageBlockReport[] reports, BlockReportContext context) { DatanodeProtocolProtos.BlockReportRequestProto.Builder builder = DatanodeProtocolProtos.BlockReportRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetBlockPoolId(poolId ); bool useBlocksBuffer = registration.GetNamespaceInfo().IsCapabilitySupported(NamespaceInfo.Capability .StorageBlockReportBuffers); foreach (StorageBlockReport r in reports) { DatanodeProtocolProtos.StorageBlockReportProto.Builder reportBuilder = DatanodeProtocolProtos.StorageBlockReportProto .NewBuilder().SetStorage(PBHelper.Convert(r.GetStorage())); BlockListAsLongs blocks = r.GetBlocks(); if (useBlocksBuffer) { reportBuilder.SetNumberOfBlocks(blocks.GetNumberOfBlocks()); reportBuilder.AddAllBlocksBuffers(blocks.GetBlocksBuffers()); } else { foreach (long value in blocks.GetBlockListAsLongs()) { reportBuilder.AddBlocks(value); } } builder.AddReports(((DatanodeProtocolProtos.StorageBlockReportProto)reportBuilder .Build())); } builder.SetContext(PBHelper.Convert(context)); DatanodeProtocolProtos.BlockReportResponseProto resp; try { resp = rpcProxy.BlockReport(NullController, ((DatanodeProtocolProtos.BlockReportRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } return(resp.HasCmd() ? PBHelper.Convert(resp.GetCmd()) : null); }
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.TimeoutException"/> /// <exception cref="System.IO.IOException"/> public virtual void TestRemoveOneVolume() { StartDFSCluster(1, 1); short replFactor = 1; Path testFile = new Path("/test"); CreateFile(testFile, 10, replFactor); DataNode dn = cluster.GetDataNodes()[0]; ICollection <string> oldDirs = GetDataDirs(dn); string newDirs = oldDirs.GetEnumerator().Next(); // Keep the first volume. dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newDirs); AssertFileLocksReleased(new AList <string>(oldDirs).SubList(1, oldDirs.Count)); dn.ScheduleAllBlockReport(0); try { DFSTestUtil.ReadFile(cluster.GetFileSystem(), testFile); NUnit.Framework.Assert.Fail("Expect to throw BlockMissingException."); } catch (BlockMissingException e) { GenericTestUtils.AssertExceptionContains("Could not obtain block", e); } Path newFile = new Path("/newFile"); CreateFile(newFile, 6); string bpid = cluster.GetNamesystem().GetBlockPoolId(); IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blockReports = cluster.GetAllBlockReports (bpid); NUnit.Framework.Assert.AreEqual((int)replFactor, blockReports.Count); BlockListAsLongs blocksForVolume1 = blockReports[0].Values.GetEnumerator().Next(); // The first volume has half of the testFile and full of newFile. NUnit.Framework.Assert.AreEqual(10 / 2 + 6, blocksForVolume1.GetNumberOfBlocks()); }
public virtual void TestSafeModeIBRBeforeFirstFullBR() { // pretend to be in safemode Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode(); DatanodeDescriptor node = nodes[0]; DatanodeStorageInfo ds = node.GetStorageInfos()[0]; node.isAlive = true; DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty ); // register new node bm.GetDatanodeManager().RegisterDatanode(nodeReg); bm.GetDatanodeManager().AddDatanode(node); NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node)); NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); // Build a incremental report IList <ReceivedDeletedBlockInfo> rdbiList = new AList <ReceivedDeletedBlockInfo>(); // Build a full report BlockListAsLongs.Builder builder = BlockListAsLongs.Builder(); // blk_42 is finalized. long receivedBlockId = 42; // arbitrary BlockInfoContiguous receivedBlock = AddBlockToBM(receivedBlockId); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivedBlock), ReceivedDeletedBlockInfo.BlockStatus .ReceivedBlock, null)); builder.Add(new FinalizedReplica(receivedBlock, null, null)); // blk_43 is under construction. long receivingBlockId = 43; BlockInfoContiguous receivingBlock = AddUcBlockToBM(receivingBlockId); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingBlock), ReceivedDeletedBlockInfo.BlockStatus .ReceivingBlock, null)); builder.Add(new ReplicaBeingWritten(receivingBlock, null, null, null)); // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record. long receivingReceivedBlockId = 44; BlockInfoContiguous receivingReceivedBlock = AddBlockToBM(receivingReceivedBlockId ); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), ReceivedDeletedBlockInfo.BlockStatus.ReceivingBlock, null)); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null)); builder.Add(new FinalizedReplica(receivingReceivedBlock, null, null)); // blk_45 is not in full BR, because it's deleted. long ReceivedDeletedBlockId = 45; rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null)); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId), ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null)); // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR. long existedBlockId = 46; BlockInfoContiguous existedBlock = AddBlockToBM(existedBlockId); builder.Add(new FinalizedReplica(existedBlock, null, null)); // process IBR and full BR StorageReceivedDeletedBlocks srdb = new StorageReceivedDeletedBlocks(new DatanodeStorage (ds.GetStorageID()), Sharpen.Collections.ToArray(rdbiList, new ReceivedDeletedBlockInfo [rdbiList.Count])); bm.ProcessIncrementalBlockReport(node, srdb); // Make sure it's the first full report NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), builder.Build(), null , false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); // verify the storage info is correct NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivedBlockId)).FindStorageInfo (ds) >= 0); NUnit.Framework.Assert.IsTrue(((BlockInfoContiguousUnderConstruction)bm.GetStoredBlock (new Block(receivingBlockId))).GetNumExpectedLocations() > 0); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivingReceivedBlockId )).FindStorageInfo(ds) >= 0); NUnit.Framework.Assert.IsNull(bm.GetStoredBlock(new Block(ReceivedDeletedBlockId) )); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(existedBlock)).FindStorageInfo (ds) >= 0); }
public StorageBlockReport(DatanodeStorage storage, BlockListAsLongs blocks) { this.storage = storage; this.blocks = blocks; }
public virtual void TestVolumeFailure() { System.Console.Out.WriteLine("Data dir: is " + dataDir.GetPath()); // Data dir structure is dataDir/data[1-4]/[current,tmp...] // data1,2 is for datanode 1, data2,3 - datanode2 string filename = "/test.txt"; Path filePath = new Path(filename); // we use only small number of blocks to avoid creating subdirs in the data dir.. int filesize = block_size * blocks_num; DFSTestUtil.CreateFile(fs, filePath, filesize, repl, 1L); DFSTestUtil.WaitReplication(fs, filePath, repl); System.Console.Out.WriteLine("file " + filename + "(size " + filesize + ") is created and replicated" ); // fail the volume // delete/make non-writable one of the directories (failed volume) data_fail = new FilePath(dataDir, "data3"); failedDir = MiniDFSCluster.GetFinalizedDir(dataDir, cluster.GetNamesystem().GetBlockPoolId ()); if (failedDir.Exists() && !DeteteBlocks(failedDir)) { //!FileUtil.fullyDelete(failedDir) throw new IOException("Could not delete hdfs directory '" + failedDir + "'"); } data_fail.SetReadOnly(); failedDir.SetReadOnly(); System.Console.Out.WriteLine("Deleteing " + failedDir.GetPath() + "; exist=" + failedDir .Exists()); // access all the blocks on the "failed" DataNode, // we need to make sure that the "failed" volume is being accessed - // and that will cause failure, blocks removal, "emergency" block report TriggerFailure(filename, filesize); // make sure a block report is sent DataNode dn = cluster.GetDataNodes()[1]; //corresponds to dir data3 string bpid = cluster.GetNamesystem().GetBlockPoolId(); DatanodeRegistration dnR = dn.GetDNRegistrationForBP(bpid); IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset ().GetBlockReports(bpid); // Send block report StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count]; int reportIndex = 0; foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists) { DatanodeStorage dnStorage = kvPair.Key; BlockListAsLongs blockList = kvPair.Value; reports[reportIndex++] = new StorageBlockReport(dnStorage, blockList); } cluster.GetNameNodeRpc().BlockReport(dnR, bpid, reports, null); // verify number of blocks and files... Verify(filename, filesize); // create another file (with one volume failed). System.Console.Out.WriteLine("creating file test1.txt"); Path fileName1 = new Path("/test1.txt"); DFSTestUtil.CreateFile(fs, fileName1, filesize, repl, 1L); // should be able to replicate to both nodes (2 DN, repl=2) DFSTestUtil.WaitReplication(fs, fileName1, repl); System.Console.Out.WriteLine("file " + fileName1.GetName() + " is created and replicated" ); }