public virtual void TestInjectionEmpty()
{
SimulatedFSDataset fsdataset = GetSimulatedFSDataset();
BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks());
int bytesAdded = AddSomeBlocks(fsdataset);
blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes());
}
// Inject blocks into an empty fsdataset
// - injecting the blocks we got above.
SimulatedFSDataset sfsdataset = GetSimulatedFSDataset();
sfsdataset.InjectBlocks(bpid, blockReport);
blockReport = sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b_1 in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b_1);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes()
);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength
(new ExtendedBlock(bpid, b_1)));
}
NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed());
NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset
.GetRemaining());
}
public virtual void TestInjectionNonEmpty()
{
SimulatedFSDataset fsdataset = GetSimulatedFSDataset();
BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks());
int bytesAdded = AddSomeBlocks(fsdataset);
blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes());
}
fsdataset = null;
// Inject blocks into an non-empty fsdataset
// - injecting the blocks we got above.
SimulatedFSDataset sfsdataset = GetSimulatedFSDataset();
// Add come blocks whose block ids do not conflict with
// the ones we are going to inject.
bytesAdded += AddSomeBlocks(sfsdataset, Numblocks + 1);
sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
sfsdataset.InjectBlocks(bpid, blockReport);
blockReport = sfsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks * 2, blockReport.GetNumberOfBlocks());
foreach (Block b_1 in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b_1);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes()
);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength
(new ExtendedBlock(bpid, b_1)));
}
NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed());
NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset
.GetRemaining());
// Now test that the dataset cannot be created if it does not have sufficient cap
conf.SetLong(SimulatedFSDataset.ConfigPropertyCapacity, 10);
try
{
sfsdataset = GetSimulatedFSDataset();
sfsdataset.AddBlockPool(bpid, conf);
sfsdataset.InjectBlocks(bpid, blockReport);
NUnit.Framework.Assert.IsTrue("Expected an IO exception", false);
}
catch (IOException)
{
}
}
// Generate a block report, optionally corrupting the generation
// stamp and/or length of one block.
private static StorageBlockReport[] GetBlockReports(DataNode dn, string bpid, bool
corruptOneBlockGs, bool corruptOneBlockLen)
{
IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset
().GetBlockReports(bpid);
// Send block report
StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count];
bool corruptedGs = false;
bool corruptedLen = false;
int reportIndex = 0;
foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists)
{
DatanodeStorage dnStorage = kvPair.Key;
BlockListAsLongs blockList = kvPair.Value;
// Walk the list of blocks until we find one each to corrupt the
// generation stamp and length, if so requested.
BlockListAsLongs.Builder builder = BlockListAsLongs.Builder();
foreach (BlockListAsLongs.BlockReportReplica block in blockList)
{
if (corruptOneBlockGs && !corruptedGs)
{
long gsOld = block.GetGenerationStamp();
long gsNew;
do
{
gsNew = rand.Next();
}while (gsNew == gsOld);
block.SetGenerationStamp(gsNew);
Log.Info("Corrupted the GS for block ID " + block);
corruptedGs = true;
}
else
{
if (corruptOneBlockLen && !corruptedLen)
{
long lenOld = block.GetNumBytes();
long lenNew;
do
{
lenNew = rand.Next((int)lenOld - 1);
}while (lenNew == lenOld);
block.SetNumBytes(lenNew);
Log.Info("Corrupted the length for block ID " + block);
corruptedLen = true;
}
}
builder.Add(new BlockListAsLongs.BlockReportReplica(block));
}
reports[reportIndex++] = new StorageBlockReport(dnStorage, builder.Build());
}
return(reports);
}
public virtual void TestGetBlockReport()
{
SimulatedFSDataset fsdataset = GetSimulatedFSDataset();
BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks());
AddSomeBlocks(fsdataset);
blockReport = fsdataset.GetBlockReport(bpid);
NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks());
foreach (Block b in blockReport)
{
NUnit.Framework.Assert.IsNotNull(b);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes());
}
}
public virtual void TestBlockHasMultipleReplicasOnSameDN()
{
string filename = MakeFileName(GenericTestUtils.GetMethodName());
Path filePath = new Path(filename);
// Write out a file with a few blocks.
DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize,
NumDatanodes, seed);
// Get the block list for the file with the block locations.
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize
* NumBlocks);
// Generate a fake block report from one of the DataNodes, such
// that it reports one copy of each block on either storage.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid);
StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode
()];
AList <Replica> blocks = new AList <Replica>();
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
Block localBlock = locatedBlock.GetBlock().GetLocalBlock();
blocks.AddItem(new FinalizedReplica(localBlock, null, null));
}
BlockListAsLongs bll = BlockListAsLongs.Encode(blocks);
for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i)
{
FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i];
DatanodeStorage dns = new DatanodeStorage(v.GetStorageID());
reports[i] = new StorageBlockReport(dns, bll);
}
// Should not assert!
cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext
(1, 0, Runtime.NanoTime()));
// Get the block locations once again.
locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks);
// Make sure that each block has two replicas, one on each DataNode.
foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks())
{
DatanodeInfo[] locations = locatedBlock_1.GetLocations();
Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes));
Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid
()));
}
}
private void VerifyCapturedArguments(ArgumentCaptor <StorageBlockReport[]> captor,
int expectedReportsPerCall, int expectedTotalBlockCount)
{
IList <StorageBlockReport[]> listOfReports = captor.GetAllValues();
int numBlocksReported = 0;
foreach (StorageBlockReport[] reports in listOfReports)
{
Assert.AssertThat(reports.Length, IS.Is(expectedReportsPerCall));
foreach (StorageBlockReport report in reports)
{
BlockListAsLongs blockList = report.GetBlocks();
numBlocksReported += blockList.GetNumberOfBlocks();
}
}
System.Diagnostics.Debug.Assert((numBlocksReported >= expectedTotalBlockCount));
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual DatanodeProtocolProtos.BlockReportResponseProto BlockReport(RpcController
controller, DatanodeProtocolProtos.BlockReportRequestProto request)
{
DatanodeCommand cmd = null;
StorageBlockReport[] report = new StorageBlockReport[request.GetReportsCount()];
int index = 0;
foreach (DatanodeProtocolProtos.StorageBlockReportProto s in request.GetReportsList
())
{
BlockListAsLongs blocks;
if (s.HasNumberOfBlocks())
{
// new style buffer based reports
int num = (int)s.GetNumberOfBlocks();
Preconditions.CheckState(s.GetBlocksCount() == 0, "cannot send both blocks list and buffers"
);
blocks = BlockListAsLongs.DecodeBuffers(num, s.GetBlocksBuffersList());
}
else
{
blocks = BlockListAsLongs.DecodeLongs(s.GetBlocksList());
}
report[index++] = new StorageBlockReport(PBHelper.Convert(s.GetStorage()), blocks
);
}
try
{
cmd = impl.BlockReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId
(), report, request.HasContext() ? PBHelper.Convert(request.GetContext()) : null
);
}
catch (IOException e)
{
throw new ServiceException(e);
}
DatanodeProtocolProtos.BlockReportResponseProto.Builder builder = DatanodeProtocolProtos.BlockReportResponseProto
.NewBuilder();
if (cmd != null)
{
builder.SetCmd(PBHelper.Convert(cmd));
}
return((DatanodeProtocolProtos.BlockReportResponseProto)builder.Build());
}
/// <exception cref="System.IO.IOException"/>
public virtual DatanodeCommand BlockReport(DatanodeRegistration registration, string
poolId, StorageBlockReport[] reports, BlockReportContext context)
{
DatanodeProtocolProtos.BlockReportRequestProto.Builder builder = DatanodeProtocolProtos.BlockReportRequestProto
.NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetBlockPoolId(poolId
);
bool useBlocksBuffer = registration.GetNamespaceInfo().IsCapabilitySupported(NamespaceInfo.Capability
.StorageBlockReportBuffers);
foreach (StorageBlockReport r in reports)
{
DatanodeProtocolProtos.StorageBlockReportProto.Builder reportBuilder = DatanodeProtocolProtos.StorageBlockReportProto
.NewBuilder().SetStorage(PBHelper.Convert(r.GetStorage()));
BlockListAsLongs blocks = r.GetBlocks();
if (useBlocksBuffer)
{
reportBuilder.SetNumberOfBlocks(blocks.GetNumberOfBlocks());
reportBuilder.AddAllBlocksBuffers(blocks.GetBlocksBuffers());
}
else
{
foreach (long value in blocks.GetBlockListAsLongs())
{
reportBuilder.AddBlocks(value);
}
}
builder.AddReports(((DatanodeProtocolProtos.StorageBlockReportProto)reportBuilder
.Build()));
}
builder.SetContext(PBHelper.Convert(context));
DatanodeProtocolProtos.BlockReportResponseProto resp;
try
{
resp = rpcProxy.BlockReport(NullController, ((DatanodeProtocolProtos.BlockReportRequestProto
)builder.Build()));
}
catch (ServiceException se)
{
throw ProtobufHelper.GetRemoteException(se);
}
return(resp.HasCmd() ? PBHelper.Convert(resp.GetCmd()) : null);
}
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestRemoveOneVolume()
{
StartDFSCluster(1, 1);
short replFactor = 1;
Path testFile = new Path("/test");
CreateFile(testFile, 10, replFactor);
DataNode dn = cluster.GetDataNodes()[0];
ICollection <string> oldDirs = GetDataDirs(dn);
string newDirs = oldDirs.GetEnumerator().Next();
// Keep the first volume.
dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newDirs);
AssertFileLocksReleased(new AList <string>(oldDirs).SubList(1, oldDirs.Count));
dn.ScheduleAllBlockReport(0);
try
{
DFSTestUtil.ReadFile(cluster.GetFileSystem(), testFile);
NUnit.Framework.Assert.Fail("Expect to throw BlockMissingException.");
}
catch (BlockMissingException e)
{
GenericTestUtils.AssertExceptionContains("Could not obtain block", e);
}
Path newFile = new Path("/newFile");
CreateFile(newFile, 6);
string bpid = cluster.GetNamesystem().GetBlockPoolId();
IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blockReports = cluster.GetAllBlockReports
(bpid);
NUnit.Framework.Assert.AreEqual((int)replFactor, blockReports.Count);
BlockListAsLongs blocksForVolume1 = blockReports[0].Values.GetEnumerator().Next();
// The first volume has half of the testFile and full of newFile.
NUnit.Framework.Assert.AreEqual(10 / 2 + 6, blocksForVolume1.GetNumberOfBlocks());
}
public virtual void TestSafeModeIBRBeforeFirstFullBR()
{
// pretend to be in safemode
Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode();
DatanodeDescriptor node = nodes[0];
DatanodeStorageInfo ds = node.GetStorageInfos()[0];
node.isAlive = true;
DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty
);
// register new node
bm.GetDatanodeManager().RegisterDatanode(nodeReg);
bm.GetDatanodeManager().AddDatanode(node);
NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node));
NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount());
// Build a incremental report
IList <ReceivedDeletedBlockInfo> rdbiList = new AList <ReceivedDeletedBlockInfo>();
// Build a full report
BlockListAsLongs.Builder builder = BlockListAsLongs.Builder();
// blk_42 is finalized.
long receivedBlockId = 42;
// arbitrary
BlockInfoContiguous receivedBlock = AddBlockToBM(receivedBlockId);
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivedBlock), ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, null));
builder.Add(new FinalizedReplica(receivedBlock, null, null));
// blk_43 is under construction.
long receivingBlockId = 43;
BlockInfoContiguous receivingBlock = AddUcBlockToBM(receivingBlockId);
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingBlock), ReceivedDeletedBlockInfo.BlockStatus
.ReceivingBlock, null));
builder.Add(new ReplicaBeingWritten(receivingBlock, null, null, null));
// blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record.
long receivingReceivedBlockId = 44;
BlockInfoContiguous receivingReceivedBlock = AddBlockToBM(receivingReceivedBlockId
);
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
ReceivedDeletedBlockInfo.BlockStatus.ReceivingBlock, null));
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null));
builder.Add(new FinalizedReplica(receivingReceivedBlock, null, null));
// blk_45 is not in full BR, because it's deleted.
long ReceivedDeletedBlockId = 45;
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId),
ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null));
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId),
ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null));
// blk_46 exists in DN for a long time, so it's in full BR, but not in IBR.
long existedBlockId = 46;
BlockInfoContiguous existedBlock = AddBlockToBM(existedBlockId);
builder.Add(new FinalizedReplica(existedBlock, null, null));
// process IBR and full BR
StorageReceivedDeletedBlocks srdb = new StorageReceivedDeletedBlocks(new DatanodeStorage
(ds.GetStorageID()), Sharpen.Collections.ToArray(rdbiList, new ReceivedDeletedBlockInfo
[rdbiList.Count]));
bm.ProcessIncrementalBlockReport(node, srdb);
// Make sure it's the first full report
NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount());
bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), builder.Build(), null
, false);
NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount());
// verify the storage info is correct
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivedBlockId)).FindStorageInfo
(ds) >= 0);
NUnit.Framework.Assert.IsTrue(((BlockInfoContiguousUnderConstruction)bm.GetStoredBlock
(new Block(receivingBlockId))).GetNumExpectedLocations() > 0);
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivingReceivedBlockId
)).FindStorageInfo(ds) >= 0);
NUnit.Framework.Assert.IsNull(bm.GetStoredBlock(new Block(ReceivedDeletedBlockId)
));
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(existedBlock)).FindStorageInfo
(ds) >= 0);
}
public StorageBlockReport(DatanodeStorage storage, BlockListAsLongs blocks)
{
this.storage = storage;
this.blocks = blocks;
}
public virtual void TestVolumeFailure()
{
System.Console.Out.WriteLine("Data dir: is " + dataDir.GetPath());
// Data dir structure is dataDir/data[1-4]/[current,tmp...]
// data1,2 is for datanode 1, data2,3 - datanode2
string filename = "/test.txt";
Path filePath = new Path(filename);
// we use only small number of blocks to avoid creating subdirs in the data dir..
int filesize = block_size * blocks_num;
DFSTestUtil.CreateFile(fs, filePath, filesize, repl, 1L);
DFSTestUtil.WaitReplication(fs, filePath, repl);
System.Console.Out.WriteLine("file " + filename + "(size " + filesize + ") is created and replicated"
);
// fail the volume
// delete/make non-writable one of the directories (failed volume)
data_fail = new FilePath(dataDir, "data3");
failedDir = MiniDFSCluster.GetFinalizedDir(dataDir, cluster.GetNamesystem().GetBlockPoolId
());
if (failedDir.Exists() && !DeteteBlocks(failedDir))
{
//!FileUtil.fullyDelete(failedDir)
throw new IOException("Could not delete hdfs directory '" + failedDir + "'");
}
data_fail.SetReadOnly();
failedDir.SetReadOnly();
System.Console.Out.WriteLine("Deleteing " + failedDir.GetPath() + "; exist=" + failedDir
.Exists());
// access all the blocks on the "failed" DataNode,
// we need to make sure that the "failed" volume is being accessed -
// and that will cause failure, blocks removal, "emergency" block report
TriggerFailure(filename, filesize);
// make sure a block report is sent
DataNode dn = cluster.GetDataNodes()[1];
//corresponds to dir data3
string bpid = cluster.GetNamesystem().GetBlockPoolId();
DatanodeRegistration dnR = dn.GetDNRegistrationForBP(bpid);
IDictionary <DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.GetFSDataset
().GetBlockReports(bpid);
// Send block report
StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.Count];
int reportIndex = 0;
foreach (KeyValuePair <DatanodeStorage, BlockListAsLongs> kvPair in perVolumeBlockLists)
{
DatanodeStorage dnStorage = kvPair.Key;
BlockListAsLongs blockList = kvPair.Value;
reports[reportIndex++] = new StorageBlockReport(dnStorage, blockList);
}
cluster.GetNameNodeRpc().BlockReport(dnR, bpid, reports, null);
// verify number of blocks and files...
Verify(filename, filesize);
// create another file (with one volume failed).
System.Console.Out.WriteLine("creating file test1.txt");
Path fileName1 = new Path("/test1.txt");
DFSTestUtil.CreateFile(fs, fileName1, filesize, repl, 1L);
// should be able to replicate to both nodes (2 DN, repl=2)
DFSTestUtil.WaitReplication(fs, fileName1, repl);
System.Console.Out.WriteLine("file " + fileName1.GetName() + " is created and replicated"
);
}