public virtual void TestInjectionEmpty() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); int bytesAdded = AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } // Inject blocks into an empty fsdataset // - injecting the blocks we got above. SimulatedFSDataset sfsdataset = GetSimulatedFSDataset(); sfsdataset.InjectBlocks(bpid, blockReport); blockReport = sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b_1 in blockReport) { NUnit.Framework.Assert.IsNotNull(b_1); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes() ); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength (new ExtendedBlock(bpid, b_1))); } NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset .GetRemaining()); }
public virtual void TestInjectionNonEmpty() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); int bytesAdded = AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } fsdataset = null; // Inject blocks into an non-empty fsdataset // - injecting the blocks we got above. SimulatedFSDataset sfsdataset = GetSimulatedFSDataset(); // Add come blocks whose block ids do not conflict with // the ones we are going to inject. bytesAdded += AddSomeBlocks(sfsdataset, Numblocks + 1); sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); sfsdataset.InjectBlocks(bpid, blockReport); blockReport = sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks * 2, blockReport.GetNumberOfBlocks()); foreach (Block b_1 in blockReport) { NUnit.Framework.Assert.IsNotNull(b_1); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes() ); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength (new ExtendedBlock(bpid, b_1))); } NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset .GetRemaining()); // Now test that the dataset cannot be created if it does not have sufficient cap conf.SetLong(SimulatedFSDataset.ConfigPropertyCapacity, 10); try { sfsdataset = GetSimulatedFSDataset(); sfsdataset.AddBlockPool(bpid, conf); sfsdataset.InjectBlocks(bpid, blockReport); NUnit.Framework.Assert.IsTrue("Expected an IO exception", false); } catch (IOException) { } }
public virtual void TestGetBlockReport() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } }
private void VerifyCapturedArguments(ArgumentCaptor <StorageBlockReport[]> captor, int expectedReportsPerCall, int expectedTotalBlockCount) { IList <StorageBlockReport[]> listOfReports = captor.GetAllValues(); int numBlocksReported = 0; foreach (StorageBlockReport[] reports in listOfReports) { Assert.AssertThat(reports.Length, IS.Is(expectedReportsPerCall)); foreach (StorageBlockReport report in reports) { BlockListAsLongs blockList = report.GetBlocks(); numBlocksReported += blockList.GetNumberOfBlocks(); } } System.Diagnostics.Debug.Assert((numBlocksReported >= expectedTotalBlockCount)); }
/// <exception cref="System.IO.IOException"/> public virtual DatanodeCommand BlockReport(DatanodeRegistration registration, string poolId, StorageBlockReport[] reports, BlockReportContext context) { DatanodeProtocolProtos.BlockReportRequestProto.Builder builder = DatanodeProtocolProtos.BlockReportRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetBlockPoolId(poolId ); bool useBlocksBuffer = registration.GetNamespaceInfo().IsCapabilitySupported(NamespaceInfo.Capability .StorageBlockReportBuffers); foreach (StorageBlockReport r in reports) { DatanodeProtocolProtos.StorageBlockReportProto.Builder reportBuilder = DatanodeProtocolProtos.StorageBlockReportProto .NewBuilder().SetStorage(PBHelper.Convert(r.GetStorage())); BlockListAsLongs blocks = r.GetBlocks(); if (useBlocksBuffer) { reportBuilder.SetNumberOfBlocks(blocks.GetNumberOfBlocks()); reportBuilder.AddAllBlocksBuffers(blocks.GetBlocksBuffers()); } else { foreach (long value in blocks.GetBlockListAsLongs()) { reportBuilder.AddBlocks(value); } } builder.AddReports(((DatanodeProtocolProtos.StorageBlockReportProto)reportBuilder .Build())); } builder.SetContext(PBHelper.Convert(context)); DatanodeProtocolProtos.BlockReportResponseProto resp; try { resp = rpcProxy.BlockReport(NullController, ((DatanodeProtocolProtos.BlockReportRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } return(resp.HasCmd() ? PBHelper.Convert(resp.GetCmd()) : null); }
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.TimeoutException"/> /// <exception cref="System.IO.IOException"/> public virtual void TestRemoveOneVolume() { StartDFSCluster(1, 1); short replFactor = 1; Path testFile = new Path("/test"); CreateFile(testFile, 10, replFactor); DataNode dn = cluster.GetDataNodes()[0]; ICollection <string> oldDirs = GetDataDirs(dn); string newDirs = oldDirs.GetEnumerator().Next(); // Keep the first volume. dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newDirs); AssertFileLocksReleased(new AList <string>(oldDirs).SubList(1, oldDirs.Count)); dn.ScheduleAllBlockReport(0); try { DFSTestUtil.ReadFile(cluster.GetFileSystem(), testFile); NUnit.Framework.Assert.Fail("Expect to throw BlockMissingException."); } catch (BlockMissingException e) { GenericTestUtils.AssertExceptionContains("Could not obtain block", e); } Path newFile = new Path("/newFile"); CreateFile(newFile, 6); string bpid = cluster.GetNamesystem().GetBlockPoolId(); IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blockReports = cluster.GetAllBlockReports (bpid); NUnit.Framework.Assert.AreEqual((int)replFactor, blockReports.Count); BlockListAsLongs blocksForVolume1 = blockReports[0].Values.GetEnumerator().Next(); // The first volume has half of the testFile and full of newFile. NUnit.Framework.Assert.AreEqual(10 / 2 + 6, blocksForVolume1.GetNumberOfBlocks()); }