public virtual void TestInjectionEmpty()
{
SimulatedFSDataset fsdataset = GetSimulatedFSDataset();
BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks());
int bytesAdded = AddSomeBlocks(fsdataset);
blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes());
}
// Inject blocks into an empty fsdataset
// - injecting the blocks we got above.
SimulatedFSDataset sfsdataset = GetSimulatedFSDataset();
sfsdataset.InjectBlocks(bpid, blockReport);
blockReport = sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b_1 in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b_1);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes()
);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength
(new ExtendedBlock(bpid, b_1)));
}
NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed());
NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset
.GetRemaining());
}
public virtual void TestInjectionNonEmpty()
{
SimulatedFSDataset fsdataset = GetSimulatedFSDataset();
BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks());
int bytesAdded = AddSomeBlocks(fsdataset);
blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes());
}
fsdataset = null;
// Inject blocks into an non-empty fsdataset
// - injecting the blocks we got above.
SimulatedFSDataset sfsdataset = GetSimulatedFSDataset();
// Add come blocks whose block ids do not conflict with
// the ones we are going to inject.
bytesAdded += AddSomeBlocks(sfsdataset, Numblocks + 1);
sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
sfsdataset.InjectBlocks(bpid, blockReport);
blockReport = sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks * 2, blockReport.GetNumberOfBlocks());
foreach (Block b_1 in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b_1);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes()
);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength
(new ExtendedBlock(bpid, b_1)));
}
NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed());
NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset
.GetRemaining());
// Now test that the dataset cannot be created if it does not have sufficient cap
conf.SetLong(SimulatedFSDataset.ConfigPropertyCapacity, 10);
try
{
sfsdataset = GetSimulatedFSDataset();
sfsdataset.AddBlockPool(bpid, conf);
sfsdataset.InjectBlocks(bpid, blockReport);
NUnit.Framework.Assert.IsTrue("Expected an IO exception", false);
}
catch (IOException)
{
}
}
public virtual void TestGetBlockReport()
{
SimulatedFSDataset fsdataset = GetSimulatedFSDataset();
BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks());
AddSomeBlocks(fsdataset);
blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes());
}
}
private void VerifyCapturedArguments(ArgumentCaptor <StorageBlockReport[]> captor,
int expectedReportsPerCall, int expectedTotalBlockCount)
{
IList <StorageBlockReport[]> listOfReports = captor.GetAllValues();
int numBlocksReported = 0;
foreach (StorageBlockReport[] reports in listOfReports)
{
Assert.AssertThat(reports.Length, IS.Is(expectedReportsPerCall));
foreach (StorageBlockReport report in reports)
{
BlockListAsLongs blockList = report.GetBlocks();
numBlocksReported += blockList.GetNumberOfBlocks();
}
}
System.Diagnostics.Debug.Assert((numBlocksReported >= expectedTotalBlockCount));
}
/// <exception cref="System.IO.IOException"/>
public virtual DatanodeCommand BlockReport(DatanodeRegistration registration, string
poolId, StorageBlockReport[] reports, BlockReportContext context)
{
DatanodeProtocolProtos.BlockReportRequestProto.Builder builder = DatanodeProtocolProtos.BlockReportRequestProto
.NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetBlockPoolId(poolId
);
bool useBlocksBuffer = registration.GetNamespaceInfo().IsCapabilitySupported(NamespaceInfo.Capability
.StorageBlockReportBuffers);
foreach (StorageBlockReport r in reports)
{
DatanodeProtocolProtos.StorageBlockReportProto.Builder reportBuilder = DatanodeProtocolProtos.StorageBlockReportProto
.NewBuilder().SetStorage(PBHelper.Convert(r.GetStorage()));
BlockListAsLongs blocks = r.GetBlocks();
if (useBlocksBuffer)
{
reportBuilder.SetNumberOfBlocks(blocks.GetNumberOfBlocks());
reportBuilder.AddAllBlocksBuffers(blocks.GetBlocksBuffers());
}
else
{
foreach (long value in blocks.GetBlockListAsLongs())
{
reportBuilder.AddBlocks(value);
}
}
builder.AddReports(((DatanodeProtocolProtos.StorageBlockReportProto)reportBuilder
.Build()));
}
builder.SetContext(PBHelper.Convert(context));
DatanodeProtocolProtos.BlockReportResponseProto resp;
try
{
resp = rpcProxy.BlockReport(NullController, ((DatanodeProtocolProtos.BlockReportRequestProto
)builder.Build()));
}
catch (ServiceException se)
{
throw ProtobufHelper.GetRemoteException(se);
}
return(resp.HasCmd() ? PBHelper.Convert(resp.GetCmd()) : null);
}
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestRemoveOneVolume()
{
StartDFSCluster(1, 1);
short replFactor = 1;
Path testFile = new Path("/test");
CreateFile(testFile, 10, replFactor);
DataNode dn = cluster.GetDataNodes()[0];
ICollection <string> oldDirs = GetDataDirs(dn);
string newDirs = oldDirs.GetEnumerator().Next();
// Keep the first volume.
dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newDirs);
AssertFileLocksReleased(new AList <string>(oldDirs).SubList(1, oldDirs.Count));
dn.ScheduleAllBlockReport(0);
try
{
DFSTestUtil.ReadFile(cluster.GetFileSystem(), testFile);
NUnit.Framework.Assert.Fail("Expect to throw BlockMissingException.");
}
catch (BlockMissingException e)
{
GenericTestUtils.AssertExceptionContains("Could not obtain block", e);
}
Path newFile = new Path("/newFile");
CreateFile(newFile, 6);
string bpid = cluster.GetNamesystem().GetBlockPoolId();
IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blockReports = cluster.GetAllBlockReports
(bpid);
NUnit.Framework.Assert.AreEqual((int)replFactor, blockReports.Count);
BlockListAsLongs blocksForVolume1 = blockReports[0].Values.GetEnumerator().Next();
// The first volume has half of the testFile and full of newFile.
NUnit.Framework.Assert.AreEqual(10 / 2 + 6, blocksForVolume1.GetNumberOfBlocks());
}
