internal virtual void UpdateState(StorageReport r)
{
capacity = r.GetCapacity();
dfsUsed = r.GetDfsUsed();
remaining = r.GetRemaining();
blockPoolUsed = r.GetBlockPoolUsed();
}
public virtual void TestDeadDatanode()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
dn.Shutdown();
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000);
DatanodeProtocol dnp = cluster.GetNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo
(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) };
StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[]
{ new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try
{
dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks);
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure blockReport from dead datanode is rejected with IOException
StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new
DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) };
try
{
dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime
()));
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage
(reg.GetDatanodeUuid()), false, 0, 0, 0, 0) };
DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmd.Length);
NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction
());
}
public ReportingController()
{
_db = new DistilDBContext();
_dl = new DataLayer(_db);
_productionR = new ProductionReport(_db, _dl);
_processingR = new ProcessingReport(_db, _dl);
_storageR = new StorageReport(_db, _dl);
_reportRepository = new ReportRepository(_db, _dl);
}
public virtual StorageReport[] GetStorageReports()
{
DatanodeStorageInfo[] infos = GetStorageInfos();
StorageReport[] reports = new StorageReport[infos.Length];
for (int i = 0; i < infos.Length; i++)
{
reports[i] = infos[i].ToStorageReport();
}
return(reports);
}
public static StorageReport[] GetStorageReportsForDatanode(DatanodeDescriptor dnd
)
{
AList <StorageReport> reports = new AList <StorageReport>();
foreach (DatanodeStorageInfo storage in dnd.GetStorageInfos())
{
DatanodeStorage dns = new DatanodeStorage(storage.GetStorageID(), storage.GetState
(), storage.GetStorageType());
StorageReport report = new StorageReport(dns, false, storage.GetCapacity(), storage
.GetDfsUsed(), storage.GetRemaining(), storage.GetBlockPoolUsed());
reports.AddItem(report);
}
return(Sharpen.Collections.ToArray(reports, StorageReport.EmptyArray));
}
/// <exception cref="System.IO.IOException"/>
private static void RunTest(string testCaseName, bool createFiles, int numInitialStorages
, int expectedStoragesAfterTest)
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).StoragesPerDatanode(numInitialStorages
).Build();
cluster.WaitActive();
DataNode dn0 = cluster.GetDataNodes()[0];
// Ensure NN knows about the storage.
DatanodeID dnId = dn0.GetDatanodeId();
DatanodeDescriptor dnDescriptor = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
().GetDatanode(dnId);
Assert.AssertThat(dnDescriptor.GetStorageInfos().Length, IS.Is(numInitialStorages
));
string bpid = cluster.GetNamesystem().GetBlockPoolId();
DatanodeRegistration dnReg = dn0.GetDNRegistrationForBP(bpid);
DataNodeTestUtils.TriggerBlockReport(dn0);
if (createFiles)
{
Path path = new Path("/", testCaseName);
DFSTestUtil.CreateFile(cluster.GetFileSystem(), path, 1024, (short)1, unchecked ((
int)(0x1BAD5EED)));
DataNodeTestUtils.TriggerBlockReport(dn0);
}
// Generate a fake StorageReport that is missing one storage.
StorageReport[] reports = dn0.GetFSDataset().GetStorageReports(bpid);
StorageReport[] prunedReports = new StorageReport[numInitialStorages - 1];
System.Array.Copy(reports, 0, prunedReports, 0, prunedReports.Length);
// Stop the DataNode and send fake heartbeat with missing storage.
cluster.StopDataNode(0);
cluster.GetNameNodeRpc().SendHeartbeat(dnReg, prunedReports, 0L, 0L, 0, 0, 0, null
);
// Check that the missing storage was pruned.
Assert.AssertThat(dnDescriptor.GetStorageInfos().Length, IS.Is(expectedStoragesAfterTest
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
internal virtual void ReceivedHeartbeat(StorageReport report)
{
UpdateState(report);
heartbeatedSinceFailover = true;
}
/// <exception cref="System.IO.IOException"/>
public override StorageReport[] GetStorageReports(string bpid)
{
StorageReport[] result = new StorageReport[1];
result[0] = new StorageReport(storage, false, 0, 0, 0, 0);
return(result);
}
