public virtual void Test()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNameNode().namesystem;
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"
);
string snapshotStats = (string)(mbs.GetAttribute(mxbeanName, "SnapshotStats"));
IDictionary <string, object> stat = (IDictionary <string, object>)JSON.Parse(snapshotStats
);
NUnit.Framework.Assert.IsTrue(stat.Contains("SnapshottableDirectories") && (long)
stat["SnapshottableDirectories"] == fsn.GetNumSnapshottableDirs());
NUnit.Framework.Assert.IsTrue(stat.Contains("Snapshots") && (long)stat["Snapshots"
] == fsn.GetNumSnapshots());
object pendingDeletionBlocks = mbs.GetAttribute(mxbeanName, "PendingDeletionBlocks"
);
NUnit.Framework.Assert.IsNotNull(pendingDeletionBlocks);
NUnit.Framework.Assert.IsTrue(pendingDeletionBlocks is long);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestSecondaryWebUi()
{
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=SecondaryNameNode,name=SecondaryNameNodeInfo"
);
string[] checkpointDir = (string[])mbs.GetAttribute(mxbeanName, "CheckpointDirectories"
);
Assert.AssertArrayEquals(checkpointDir, snn.GetCheckpointDirectories());
string[] checkpointEditlogDir = (string[])mbs.GetAttribute(mxbeanName, "CheckpointEditlogDirectories"
);
Assert.AssertArrayEquals(checkpointEditlogDir, snn.GetCheckpointEditlogDirectories
());
}
/// <summary>Verify the following scenario.</summary>
/// <remarks>
/// Verify the following scenario.
/// 1. NN restarts.
/// 2. Heartbeat RPC will retry and succeed. NN asks DN to reregister.
/// 3. After reregistration completes, DN will send Heartbeat, followed by
/// Blockreport.
/// 4. NN will mark DatanodeStorageInfo#blockContentsStale to false.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestStorageBlockContentsStaleAfterNNRestart()
{
MiniDFSCluster dfsCluster = null;
try
{
Configuration config = new Configuration();
dfsCluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
dfsCluster.WaitActive();
dfsCluster.RestartNameNode(true);
BlockManagerTestUtil.CheckHeartbeat(dfsCluster.GetNamesystem().GetBlockManager());
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanNameFsns = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"
);
int numStaleStorages = (int)(mbs.GetAttribute(mxbeanNameFsns, "NumStaleStorages")
);
NUnit.Framework.Assert.AreEqual(0, numStaleStorages);
}
finally
{
if (dfsCluster != null)
{
dfsCluster.Shutdown();
}
}
return;
}
/// <exception cref="System.Exception"/>
public virtual void TestTopUsersNoPeriods()
{
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.NntopEnabledKey, true);
conf.Set(DFSConfigKeys.NntopWindowsMinutesKey, string.Empty);
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanNameFsns = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"
);
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("/");
int NumOps = 10;
for (int i = 0; i < NumOps; i++)
{
fs.ListStatus(path);
fs.SetTimes(path, 0, 1);
}
string topUsers = (string)(mbs.GetAttribute(mxbeanNameFsns, "TopUserOpCounts"));
NUnit.Framework.Assert.IsNotNull("Expected TopUserOpCounts bean!", topUsers);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public override void Run()
{
try
{
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
// Metrics that belong to "FSNamesystem", these are metrics that
// come from hadoop metrics framework for the class FSNamesystem.
ObjectName mxbeanNamefsn = new ObjectName("Hadoop:service=NameNode,name=FSNamesystem"
);
// Metrics that belong to "FSNamesystemState".
// These are metrics that FSNamesystem registers directly with MBeanServer.
ObjectName mxbeanNameFsns = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"
);
// Metrics that belong to "NameNodeInfo".
// These are metrics that FSNamesystem registers directly with MBeanServer.
ObjectName mxbeanNameNni = new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"
);
ICollection <ObjectName> mbeans = new HashSet <ObjectName>();
mbeans.AddItem(mxbeanNamefsn);
mbeans.AddItem(mxbeanNameFsns);
mbeans.AddItem(mxbeanNameNni);
foreach (ObjectName mbean in mbeans)
{
MBeanInfo attributes = mbs.GetMBeanInfo(mbean);
foreach (MBeanAttributeInfo attributeInfo in attributes.GetAttributes())
{
mbs.GetAttribute(mbean, attributeInfo.GetName());
}
}
succeeded = true;
}
catch (Exception)
{
}
}
/// <exception cref="System.Exception"/>
public virtual void TestTopUsers()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanNameFsns = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"
);
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("/");
int NumOps = 10;
for (int i = 0; i < NumOps; i++)
{
fs.ListStatus(path);
fs.SetTimes(path, 0, 1);
}
string topUsers = (string)(mbs.GetAttribute(mxbeanNameFsns, "TopUserOpCounts"));
ObjectMapper mapper = new ObjectMapper();
IDictionary <string, object> map = mapper.ReadValue <IDictionary>(topUsers);
NUnit.Framework.Assert.IsTrue("Could not find map key timestamp", map.Contains("timestamp"
));
NUnit.Framework.Assert.IsTrue("Could not find map key windows", map.Contains("windows"
));
IList <IDictionary <string, IList <IDictionary <string, object> > > > windows = (IList <IDictionary
<string, IList <IDictionary <string, object> > > >)map["windows"];
NUnit.Framework.Assert.AreEqual("Unexpected num windows", 3, windows.Count);
foreach (IDictionary <string, IList <IDictionary <string, object> > > window in windows)
{
IList <IDictionary <string, object> > ops = window["ops"];
NUnit.Framework.Assert.AreEqual("Unexpected num ops", 3, ops.Count);
foreach (IDictionary <string, object> op in ops)
{
long count = long.Parse(op["totalCount"].ToString());
string opType = op["opType"].ToString();
int expected;
if (opType.Equals(TopConf.AllCmds))
{
expected = 2 * NumOps;
}
else
{
expected = NumOps;
}
NUnit.Framework.Assert.AreEqual("Unexpected total count", expected, count);
}
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="Javax.Management.MalformedObjectNameException"/>
/// <exception cref="Javax.Management.MBeanException"/>
/// <exception cref="Javax.Management.AttributeNotFoundException"/>
/// <exception cref="Javax.Management.InstanceNotFoundException"/>
/// <exception cref="Javax.Management.ReflectionException"/>
private static CompositeDataSupport GetBean()
{
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"
);
return((CompositeDataSupport)mbs.GetAttribute(mxbeanName, "RollingUpgradeStatus"));
}
public virtual void TestHostsIncludeForDeadCount()
{
Configuration conf = GetConf();
// Configure an excludes file
FileSystem localFileSys = FileSystem.GetLocal(conf);
Path workingDir = localFileSys.GetWorkingDirectory();
Path dir = new Path(workingDir, "build/test/data/temp/decommission");
Path excludeFile = new Path(dir, "exclude");
Path includeFile = new Path(dir, "include");
NUnit.Framework.Assert.IsTrue(localFileSys.Mkdirs(dir));
StringBuilder includeHosts = new StringBuilder();
includeHosts.Append("localhost:52").Append("\n").Append("127.0.0.1:7777").Append(
"\n");
DFSTestUtil.WriteFile(localFileSys, excludeFile, string.Empty);
DFSTestUtil.WriteFile(localFileSys, includeFile, includeHosts.ToString());
conf.Set(DFSConfigKeys.DfsHostsExclude, excludeFile.ToUri().GetPath());
conf.Set(DFSConfigKeys.DfsHosts, includeFile.ToUri().GetPath());
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
FSNamesystem ns = cluster.GetNameNode().GetNamesystem();
NUnit.Framework.Assert.IsTrue(ns.GetNumDeadDataNodes() == 2);
NUnit.Framework.Assert.IsTrue(ns.GetNumLiveDataNodes() == 0);
// Testing using MBeans
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"
);
string nodes = mbs.GetAttribute(mxbeanName, "NumDeadDataNodes") + string.Empty;
NUnit.Framework.Assert.IsTrue((int)mbs.GetAttribute(mxbeanName, "NumDeadDataNodes"
) == 2);
NUnit.Framework.Assert.IsTrue((int)mbs.GetAttribute(mxbeanName, "NumLiveDataNodes"
) == 0);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestLastContactTime()
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1);
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNameNode().namesystem;
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"
);
// Define include file to generate deadNodes metrics
FileSystem localFileSys = FileSystem.GetLocal(conf);
Path workingDir = localFileSys.GetWorkingDirectory();
Path dir = new Path(workingDir, "build/test/data/temp/TestNameNodeMXBean");
Path includeFile = new Path(dir, "include");
NUnit.Framework.Assert.IsTrue(localFileSys.Mkdirs(dir));
StringBuilder includeHosts = new StringBuilder();
foreach (DataNode dn in cluster.GetDataNodes())
{
includeHosts.Append(dn.GetDisplayName()).Append("\n");
}
DFSTestUtil.WriteFile(localFileSys, includeFile, includeHosts.ToString());
conf.Set(DFSConfigKeys.DfsHosts, includeFile.ToUri().GetPath());
fsn.GetBlockManager().GetDatanodeManager().RefreshNodes(conf);
cluster.StopDataNode(0);
while (fsn.GetBlockManager().GetDatanodeManager().GetNumLiveDataNodes() != 2)
{
Uninterruptibles.SleepUninterruptibly(1, TimeUnit.Seconds);
}
// get attribute deadnodeinfo
string deadnodeinfo = (string)(mbs.GetAttribute(mxbeanName, "DeadNodes"));
NUnit.Framework.Assert.AreEqual(fsn.GetDeadNodes(), deadnodeinfo);
IDictionary <string, IDictionary <string, object> > deadNodes = (IDictionary <string,
IDictionary <string, object> >)JSON.Parse(deadnodeinfo);
NUnit.Framework.Assert.IsTrue(deadNodes.Count > 0);
foreach (IDictionary <string, object> deadNode in deadNodes.Values)
{
NUnit.Framework.Assert.IsTrue(deadNode.Contains("lastContact"));
NUnit.Framework.Assert.IsTrue(deadNode.Contains("decommissioned"));
NUnit.Framework.Assert.IsTrue(deadNode.Contains("xferaddr"));
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestHostsExcludeInUI()
{
Configuration conf = GetConf();
short ReplicationFactor = 2;
Path filePath = new Path("/testFile");
// Configure an excludes file
FileSystem localFileSys = FileSystem.GetLocal(conf);
Path workingDir = localFileSys.GetWorkingDirectory();
Path dir = new Path(workingDir, "build/test/data/temp/decommission");
Path excludeFile = new Path(dir, "exclude");
Path includeFile = new Path(dir, "include");
NUnit.Framework.Assert.IsTrue(localFileSys.Mkdirs(dir));
DFSTestUtil.WriteFile(localFileSys, excludeFile, string.Empty);
DFSTestUtil.WriteFile(localFileSys, includeFile, string.Empty);
conf.Set(DFSConfigKeys.DfsHostsExclude, excludeFile.ToUri().GetPath());
conf.Set(DFSConfigKeys.DfsHosts, includeFile.ToUri().GetPath());
// Two blocks and four racks
string[] racks = new string[] { "/rack1", "/rack1", "/rack2", "/rack2" };
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length
).Racks(racks).Build();
FSNamesystem ns = cluster.GetNameNode().GetNamesystem();
try
{
// Create a file with one block
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, filePath, 1L, ReplicationFactor, 1L);
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath);
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
// Decommission one of the hosts with the block, this should cause
// the block to get replicated to another host on the same rack,
// otherwise the rack policy is violated.
BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(filePath), 0, long.MaxValue
);
string name = locs[0].GetNames()[0];
string names = name + "\n" + "localhost:42\n";
Log.Info("adding '" + names + "' to exclude file " + excludeFile.ToUri().GetPath(
));
DFSTestUtil.WriteFile(localFileSys, excludeFile, name);
ns.GetBlockManager().GetDatanodeManager().RefreshNodes(conf);
DFSTestUtil.WaitForDecommission(fs, name);
// Check the block still has sufficient # replicas across racks
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"
);
string nodes = (string)mbs.GetAttribute(mxbeanName, "LiveNodes");
NUnit.Framework.Assert.IsTrue("Live nodes should contain the decommissioned node"
, nodes.Contains("Decommissioned"));
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestSnapshotStatsMXBeanInfo()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
string pathName = "/snapshot";
Path path = new Path(pathName);
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
SnapshotManager sm = cluster.GetNamesystem().GetSnapshotManager();
DistributedFileSystem dfs = (DistributedFileSystem)cluster.GetFileSystem();
dfs.Mkdirs(path);
dfs.AllowSnapshot(path);
dfs.CreateSnapshot(path);
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=SnapshotInfo"
);
CompositeData[] directories = (CompositeData[])mbs.GetAttribute(mxbeanName, "SnapshottableDirectories"
);
int numDirectories = Sharpen.Runtime.GetArrayLength(directories);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshottableDirs(), numDirectories);
CompositeData[] snapshots = (CompositeData[])mbs.GetAttribute(mxbeanName, "Snapshots"
);
int numSnapshots = Sharpen.Runtime.GetArrayLength(snapshots);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshots(), numSnapshots);
CompositeData d = (CompositeData)Sharpen.Runtime.GetArrayValue(directories, 0);
CompositeData s = (CompositeData)Sharpen.Runtime.GetArrayValue(snapshots, 0);
NUnit.Framework.Assert.IsTrue(((string)d.Get("path")).Contains(pathName));
NUnit.Framework.Assert.IsTrue(((string)s.Get("snapshotDirectory")).Contains(pathName
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestJournalNodeMXBean()
{
// we have not formatted the journals yet, and the journal status in jmx
// should be empty since journal objects are created lazily
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=JournalNode,name=JournalNodeInfo"
);
// getJournalsStatus
string journalStatus = (string)mbs.GetAttribute(mxbeanName, "JournalsStatus");
NUnit.Framework.Assert.AreEqual(jn.GetJournalsStatus(), journalStatus);
NUnit.Framework.Assert.IsFalse(journalStatus.Contains(Nameservice));
// format the journal ns1
NamespaceInfo FakeNsinfo = new NamespaceInfo(12345, "mycluster", "my-bp", 0L);
jn.GetOrCreateJournal(Nameservice).Format(FakeNsinfo);
// check again after format
// getJournalsStatus
journalStatus = (string)mbs.GetAttribute(mxbeanName, "JournalsStatus");
NUnit.Framework.Assert.AreEqual(jn.GetJournalsStatus(), journalStatus);
IDictionary <string, IDictionary <string, string> > jMap = new Dictionary <string, IDictionary
<string, string> >();
IDictionary <string, string> infoMap = new Dictionary <string, string>();
infoMap["Formatted"] = "true";
jMap[Nameservice] = infoMap;
NUnit.Framework.Assert.AreEqual(JSON.ToString(jMap), journalStatus);
// restart journal node without formatting
jCluster = new MiniJournalCluster.Builder(new Configuration()).Format(false).NumJournalNodes
(NumJn).Build();
jn = jCluster.GetJournalNode(0);
// re-check
journalStatus = (string)mbs.GetAttribute(mxbeanName, "JournalsStatus");
NUnit.Framework.Assert.AreEqual(jn.GetJournalsStatus(), journalStatus);
jMap = new Dictionary <string, IDictionary <string, string> >();
infoMap = new Dictionary <string, string>();
infoMap["Formatted"] = "true";
jMap[Nameservice] = infoMap;
NUnit.Framework.Assert.AreEqual(JSON.ToString(jMap), journalStatus);
}
public void GetAttributePermissionTest()
{
IMBeanServer server = null;
DoPermissionTest(
delegate()
{
server = new MBeanServer();
server.RegisterMBean(new Dummy(), new ObjectName("dummy:"));
},
delegate()
{
object o = server.GetAttribute(new ObjectName("dummy:"), "IntValue");
}, "IntValue", "DoSomething", MBeanPermissionAction.GetAttribute, MBeanPermissionAction.SetAttribute);
}
public virtual void TestDataNodeMXBean()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
IList <DataNode> datanodes = cluster.GetDataNodes();
NUnit.Framework.Assert.AreEqual(datanodes.Count, 1);
DataNode datanode = datanodes[0];
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=DataNode,name=DataNodeInfo"
);
// get attribute "ClusterId"
string clusterId = (string)mbs.GetAttribute(mxbeanName, "ClusterId");
NUnit.Framework.Assert.AreEqual(datanode.GetClusterId(), clusterId);
// get attribute "Version"
string version = (string)mbs.GetAttribute(mxbeanName, "Version");
NUnit.Framework.Assert.AreEqual(datanode.GetVersion(), version);
// get attribute "RpcPort"
string rpcPort = (string)mbs.GetAttribute(mxbeanName, "RpcPort");
NUnit.Framework.Assert.AreEqual(datanode.GetRpcPort(), rpcPort);
// get attribute "HttpPort"
string httpPort = (string)mbs.GetAttribute(mxbeanName, "HttpPort");
NUnit.Framework.Assert.AreEqual(datanode.GetHttpPort(), httpPort);
// get attribute "NamenodeAddresses"
string namenodeAddresses = (string)mbs.GetAttribute(mxbeanName, "NamenodeAddresses"
);
NUnit.Framework.Assert.AreEqual(datanode.GetNamenodeAddresses(), namenodeAddresses
);
// get attribute "getVolumeInfo"
string volumeInfo = (string)mbs.GetAttribute(mxbeanName, "VolumeInfo");
NUnit.Framework.Assert.AreEqual(ReplaceDigits(datanode.GetVolumeInfo()), ReplaceDigits
(volumeInfo));
// Ensure mxbean's XceiverCount is same as the DataNode's
// live value.
int xceiverCount = (int)mbs.GetAttribute(mxbeanName, "XceiverCount");
NUnit.Framework.Assert.AreEqual(datanode.GetXceiverCount(), xceiverCount);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestTimeoutMetric()
{
Configuration conf = new HdfsConfiguration();
Path path = new Path("/test");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
IList <FSDataOutputStream> streams = Lists.NewArrayList();
try
{
FSDataOutputStream @out = cluster.GetFileSystem().Create(path, (short)2);
DataNodeFaultInjector injector = Org.Mockito.Mockito.Mock <DataNodeFaultInjector>(
);
Org.Mockito.Mockito.DoThrow(new IOException("mock IOException")).When(injector).WriteBlockAfterFlush
();
DataNodeFaultInjector.instance = injector;
streams.AddItem(@out);
@out.WriteBytes("old gs data\n");
@out.Hflush();
/* Test the metric. */
MetricsRecordBuilder dnMetrics = MetricsAsserts.GetMetrics(cluster.GetDataNodes()
[0].GetMetrics().Name());
MetricsAsserts.AssertCounter("DatanodeNetworkErrors", 1L, dnMetrics);
/* Test JMX datanode network counts. */
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=DataNode,name=DataNodeInfo"
);
object dnc = mbs.GetAttribute(mxbeanName, "DatanodeNetworkCounts");
string allDnc = dnc.ToString();
NUnit.Framework.Assert.IsTrue("expected to see loopback address", allDnc.IndexOf(
"127.0.0.1") >= 0);
NUnit.Framework.Assert.IsTrue("expected to see networkErrors", allDnc.IndexOf("networkErrors"
) >= 0);
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
if (cluster != null)
{
cluster.Shutdown();
}
DataNodeFaultInjector.instance = new DataNodeFaultInjector();
}
}
public virtual void TestMissingBlocksAlert()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new HdfsConfiguration();
//minimize test delay
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 0);
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
int fileLen = 10 * 1024;
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, fileLen / 2);
//start a cluster with single datanode
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
BlockManager bm = cluster.GetNamesystem().GetBlockManager();
DistributedFileSystem dfs = cluster.GetFileSystem();
// create a normal file
DFSTestUtil.CreateFile(dfs, new Path("/testMissingBlocksAlert/file1"), fileLen, (
short)3, 0);
Path corruptFile = new Path("/testMissingBlocks/corruptFile");
DFSTestUtil.CreateFile(dfs, corruptFile, fileLen, (short)3, 0);
// Corrupt the block
ExtendedBlock block = DFSTestUtil.GetFirstBlock(dfs, corruptFile);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block));
// read the file so that the corrupt block is reported to NN
FSDataInputStream @in = dfs.Open(corruptFile);
try
{
@in.ReadFully(new byte[fileLen]);
}
catch (ChecksumException)
{
}
// checksum error is expected.
@in.Close();
Log.Info("Waiting for missing blocks count to increase...");
while (dfs.GetMissingBlocksCount() <= 0)
{
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.IsTrue(dfs.GetMissingBlocksCount() == 1);
NUnit.Framework.Assert.AreEqual(4, dfs.GetUnderReplicatedBlocksCount());
NUnit.Framework.Assert.AreEqual(3, bm.GetUnderReplicatedNotMissingBlocks());
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"
);
NUnit.Framework.Assert.AreEqual(1, (long)(long)mbs.GetAttribute(mxbeanName, "NumberOfMissingBlocks"
));
// now do the reverse : remove the file expect the number of missing
// blocks to go to zero
dfs.Delete(corruptFile, true);
Log.Info("Waiting for missing blocks count to be zero...");
while (dfs.GetMissingBlocksCount() > 0)
{
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.AreEqual(2, dfs.GetUnderReplicatedBlocksCount());
NUnit.Framework.Assert.AreEqual(2, bm.GetUnderReplicatedNotMissingBlocks());
NUnit.Framework.Assert.AreEqual(0, (long)(long)mbs.GetAttribute(mxbeanName, "NumberOfMissingBlocks"
));
Path replOneFile = new Path("/testMissingBlocks/replOneFile");
DFSTestUtil.CreateFile(dfs, replOneFile, fileLen, (short)1, 0);
ExtendedBlock replOneBlock = DFSTestUtil.GetFirstBlock(dfs, replOneFile);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, replOneBlock));
// read the file so that the corrupt block is reported to NN
@in = dfs.Open(replOneFile);
try
{
@in.ReadFully(new byte[fileLen]);
}
catch (ChecksumException)
{
}
// checksum error is expected.
@in.Close();
NUnit.Framework.Assert.AreEqual(1, dfs.GetMissingReplOneBlocksCount());
NUnit.Framework.Assert.AreEqual(1, (long)(long)mbs.GetAttribute(mxbeanName, "NumberOfMissingBlocksWithReplicationFactorOne"
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestHAMetrics()
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
conf.SetInt(DFSConfigKeys.DfsHaLogrollPeriodKey, int.MaxValue);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(1).Build();
FileSystem fs = null;
try
{
cluster.WaitActive();
FSNamesystem nn0 = cluster.GetNamesystem(0);
FSNamesystem nn1 = cluster.GetNamesystem(1);
NUnit.Framework.Assert.AreEqual(nn0.GetHAState(), "standby");
NUnit.Framework.Assert.IsTrue(0 < nn0.GetMillisSinceLastLoadedEdits());
NUnit.Framework.Assert.AreEqual(nn1.GetHAState(), "standby");
NUnit.Framework.Assert.IsTrue(0 < nn1.GetMillisSinceLastLoadedEdits());
cluster.TransitionToActive(0);
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=NameNodeStatus"
);
long ltt1 = (long)mbs.GetAttribute(mxbeanName, "LastHATransitionTime");
NUnit.Framework.Assert.IsTrue("lastHATransitionTime should be > 0", ltt1 > 0);
NUnit.Framework.Assert.AreEqual("active", nn0.GetHAState());
NUnit.Framework.Assert.AreEqual(0, nn0.GetMillisSinceLastLoadedEdits());
NUnit.Framework.Assert.AreEqual("standby", nn1.GetHAState());
NUnit.Framework.Assert.IsTrue(0 < nn1.GetMillisSinceLastLoadedEdits());
cluster.TransitionToStandby(0);
long ltt2 = (long)mbs.GetAttribute(mxbeanName, "LastHATransitionTime");
NUnit.Framework.Assert.IsTrue("lastHATransitionTime should be > " + ltt1, ltt2 >
ltt1);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.AreEqual("standby", nn0.GetHAState());
NUnit.Framework.Assert.IsTrue(0 < nn0.GetMillisSinceLastLoadedEdits());
NUnit.Framework.Assert.AreEqual("active", nn1.GetHAState());
NUnit.Framework.Assert.AreEqual(0, nn1.GetMillisSinceLastLoadedEdits());
Sharpen.Thread.Sleep(2000);
// make sure standby gets a little out-of-date
NUnit.Framework.Assert.IsTrue(2000 <= nn0.GetMillisSinceLastLoadedEdits());
NUnit.Framework.Assert.AreEqual(0, nn0.GetPendingDataNodeMessageCount());
NUnit.Framework.Assert.AreEqual(0, nn1.GetPendingDataNodeMessageCount());
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.CreateFile(fs, new Path("/foo"), 10, (short)1, 1L);
NUnit.Framework.Assert.IsTrue(0 < nn0.GetPendingDataNodeMessageCount());
NUnit.Framework.Assert.AreEqual(0, nn1.GetPendingDataNodeMessageCount());
long millisSinceLastLoadedEdits = nn0.GetMillisSinceLastLoadedEdits();
HATestUtil.WaitForStandbyToCatchUp(cluster.GetNameNode(1), cluster.GetNameNode(0)
);
NUnit.Framework.Assert.AreEqual(0, nn0.GetPendingDataNodeMessageCount());
NUnit.Framework.Assert.AreEqual(0, nn1.GetPendingDataNodeMessageCount());
long newMillisSinceLastLoadedEdits = nn0.GetMillisSinceLastLoadedEdits();
// Since we just waited for the standby to catch up, the time since we
// last loaded edits should be very low.
NUnit.Framework.Assert.IsTrue("expected " + millisSinceLastLoadedEdits + " > " +
newMillisSinceLastLoadedEdits, millisSinceLastLoadedEdits > newMillisSinceLastLoadedEdits
);
}
finally
{
IOUtils.Cleanup(Log, fs);
cluster.Shutdown();
}
}
// --------------------------------------------------------- Private Methods
/// <exception cref="System.IO.IOException"/>
private void ListBeans(JsonGenerator jg, ObjectName qry, string attribute, HttpServletResponse
response)
{
Log.Debug("Listing beans for " + qry);
ICollection <ObjectName> names = null;
names = mBeanServer.QueryNames(qry, null);
jg.WriteArrayFieldStart("beans");
IEnumerator <ObjectName> it = names.GetEnumerator();
while (it.HasNext())
{
ObjectName oname = it.Next();
MBeanInfo minfo;
string code = string.Empty;
object attributeinfo = null;
try
{
minfo = mBeanServer.GetMBeanInfo(oname);
code = minfo.GetClassName();
string prs = string.Empty;
try
{
if ("org.apache.commons.modeler.BaseModelMBean".Equals(code))
{
prs = "modelerType";
code = (string)mBeanServer.GetAttribute(oname, prs);
}
if (attribute != null)
{
prs = attribute;
attributeinfo = mBeanServer.GetAttribute(oname, prs);
}
}
catch (AttributeNotFoundException e)
{
// If the modelerType attribute was not found, the class name is used
// instead.
Log.Error("getting attribute " + prs + " of " + oname + " threw an exception", e);
}
catch (MBeanException e)
{
// The code inside the attribute getter threw an exception so log it,
// and fall back on the class name
Log.Error("getting attribute " + prs + " of " + oname + " threw an exception", e);
}
catch (RuntimeException e)
{
// For some reason even with an MBeanException available to them
// Runtime exceptionscan still find their way through, so treat them
// the same as MBeanException
Log.Error("getting attribute " + prs + " of " + oname + " threw an exception", e);
}
catch (ReflectionException e)
{
// This happens when the code inside the JMX bean (setter?? from the
// java docs) threw an exception, so log it and fall back on the
// class name
Log.Error("getting attribute " + prs + " of " + oname + " threw an exception", e);
}
}
catch (InstanceNotFoundException)
{
//Ignored for some reason the bean was not found so don't output it
continue;
}
catch (IntrospectionException e)
{
// This is an internal error, something odd happened with reflection so
// log it and don't output the bean.
Log.Error("Problem while trying to process JMX query: " + qry + " with MBean " +
oname, e);
continue;
}
catch (ReflectionException e)
{
// This happens when the code inside the JMX bean threw an exception, so
// log it and don't output the bean.
Log.Error("Problem while trying to process JMX query: " + qry + " with MBean " +
oname, e);
continue;
}
jg.WriteStartObject();
jg.WriteStringField("name", oname.ToString());
jg.WriteStringField("modelerType", code);
if ((attribute != null) && (attributeinfo == null))
{
jg.WriteStringField("result", "ERROR");
jg.WriteStringField("message", "No attribute with name " + attribute + " was found."
);
jg.WriteEndObject();
jg.WriteEndArray();
jg.Close();
response.SetStatus(HttpServletResponse.ScNotFound);
return;
}
if (attribute != null)
{
WriteAttribute(jg, attribute, attributeinfo);
}
else
{
MBeanAttributeInfo[] attrs = minfo.GetAttributes();
for (int i = 0; i < attrs.Length; i++)
{
WriteAttribute(jg, oname, attrs[i]);
}
}
jg.WriteEndObject();
}
jg.WriteEndArray();
}
public virtual void TestNameNodeMXBeanInfo()
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, NativeIO.POSIX.GetCacheManipulator
().GetMemlockLimit());
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNameNode().namesystem;
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"
);
// get attribute "ClusterId"
string clusterId = (string)mbs.GetAttribute(mxbeanName, "ClusterId");
NUnit.Framework.Assert.AreEqual(fsn.GetClusterId(), clusterId);
// get attribute "BlockPoolId"
string blockpoolId = (string)mbs.GetAttribute(mxbeanName, "BlockPoolId");
NUnit.Framework.Assert.AreEqual(fsn.GetBlockPoolId(), blockpoolId);
// get attribute "Version"
string version = (string)mbs.GetAttribute(mxbeanName, "Version");
NUnit.Framework.Assert.AreEqual(fsn.GetVersion(), version);
NUnit.Framework.Assert.IsTrue(version.Equals(VersionInfo.GetVersion() + ", r" + VersionInfo
.GetRevision()));
// get attribute "Used"
long used = (long)mbs.GetAttribute(mxbeanName, "Used");
NUnit.Framework.Assert.AreEqual(fsn.GetUsed(), used);
// get attribute "Total"
long total = (long)mbs.GetAttribute(mxbeanName, "Total");
NUnit.Framework.Assert.AreEqual(fsn.GetTotal(), total);
// get attribute "safemode"
string safemode = (string)mbs.GetAttribute(mxbeanName, "Safemode");
NUnit.Framework.Assert.AreEqual(fsn.GetSafemode(), safemode);
// get attribute nondfs
long nondfs = (long)(mbs.GetAttribute(mxbeanName, "NonDfsUsedSpace"));
NUnit.Framework.Assert.AreEqual(fsn.GetNonDfsUsedSpace(), nondfs);
// get attribute percentremaining
float percentremaining = (float)(mbs.GetAttribute(mxbeanName, "PercentRemaining")
);
NUnit.Framework.Assert.AreEqual(fsn.GetPercentRemaining(), percentremaining, Delta
);
// get attribute Totalblocks
long totalblocks = (long)(mbs.GetAttribute(mxbeanName, "TotalBlocks"));
NUnit.Framework.Assert.AreEqual(fsn.GetTotalBlocks(), totalblocks);
// get attribute alivenodeinfo
string alivenodeinfo = (string)(mbs.GetAttribute(mxbeanName, "LiveNodes"));
IDictionary <string, IDictionary <string, object> > liveNodes = (IDictionary <string,
IDictionary <string, object> >)JSON.Parse(alivenodeinfo);
NUnit.Framework.Assert.IsTrue(liveNodes.Count == 2);
foreach (IDictionary <string, object> liveNode in liveNodes.Values)
{
NUnit.Framework.Assert.IsTrue(liveNode.Contains("nonDfsUsedSpace"));
NUnit.Framework.Assert.IsTrue(((long)liveNode["nonDfsUsedSpace"]) > 0);
NUnit.Framework.Assert.IsTrue(liveNode.Contains("capacity"));
NUnit.Framework.Assert.IsTrue(((long)liveNode["capacity"]) > 0);
NUnit.Framework.Assert.IsTrue(liveNode.Contains("numBlocks"));
NUnit.Framework.Assert.IsTrue(((long)liveNode["numBlocks"]) == 0);
}
NUnit.Framework.Assert.AreEqual(fsn.GetLiveNodes(), alivenodeinfo);
// get attribute deadnodeinfo
string deadnodeinfo = (string)(mbs.GetAttribute(mxbeanName, "DeadNodes"));
NUnit.Framework.Assert.AreEqual(fsn.GetDeadNodes(), deadnodeinfo);
// get attribute NodeUsage
string nodeUsage = (string)(mbs.GetAttribute(mxbeanName, "NodeUsage"));
NUnit.Framework.Assert.AreEqual("Bad value for NodeUsage", fsn.GetNodeUsage(), nodeUsage
);
// get attribute NameJournalStatus
string nameJournalStatus = (string)(mbs.GetAttribute(mxbeanName, "NameJournalStatus"
));
NUnit.Framework.Assert.AreEqual("Bad value for NameJournalStatus", fsn.GetNameJournalStatus
(), nameJournalStatus);
// get attribute JournalTransactionInfo
string journalTxnInfo = (string)mbs.GetAttribute(mxbeanName, "JournalTransactionInfo"
);
NUnit.Framework.Assert.AreEqual("Bad value for NameTxnIds", fsn.GetJournalTransactionInfo
(), journalTxnInfo);
// get attribute "NNStarted"
string nnStarted = (string)mbs.GetAttribute(mxbeanName, "NNStarted");
NUnit.Framework.Assert.AreEqual("Bad value for NNStarted", fsn.GetNNStarted(), nnStarted
);
// get attribute "CompileInfo"
string compileInfo = (string)mbs.GetAttribute(mxbeanName, "CompileInfo");
NUnit.Framework.Assert.AreEqual("Bad value for CompileInfo", fsn.GetCompileInfo()
, compileInfo);
// get attribute CorruptFiles
string corruptFiles = (string)(mbs.GetAttribute(mxbeanName, "CorruptFiles"));
NUnit.Framework.Assert.AreEqual("Bad value for CorruptFiles", fsn.GetCorruptFiles
(), corruptFiles);
// get attribute NameDirStatuses
string nameDirStatuses = (string)(mbs.GetAttribute(mxbeanName, "NameDirStatuses")
);
NUnit.Framework.Assert.AreEqual(fsn.GetNameDirStatuses(), nameDirStatuses);
IDictionary <string, IDictionary <string, string> > statusMap = (IDictionary <string,
IDictionary <string, string> >)JSON.Parse(nameDirStatuses);
ICollection <URI> nameDirUris = cluster.GetNameDirs(0);
foreach (URI nameDirUri in nameDirUris)
{
FilePath nameDir = new FilePath(nameDirUri);
System.Console.Out.WriteLine("Checking for the presence of " + nameDir + " in active name dirs."
);
NUnit.Framework.Assert.IsTrue(statusMap["active"].Contains(nameDir.GetAbsolutePath
()));
}
NUnit.Framework.Assert.AreEqual(2, statusMap["active"].Count);
NUnit.Framework.Assert.AreEqual(0, statusMap["failed"].Count);
// This will cause the first dir to fail.
FilePath failedNameDir = new FilePath(nameDirUris.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(new FilePath(failedNameDir, "current"
).GetAbsolutePath(), "000"));
cluster.GetNameNodeRpc().RollEditLog();
nameDirStatuses = (string)(mbs.GetAttribute(mxbeanName, "NameDirStatuses"));
statusMap = (IDictionary <string, IDictionary <string, string> >)JSON.Parse(nameDirStatuses
);
foreach (URI nameDirUri_1 in nameDirUris)
{
FilePath nameDir = new FilePath(nameDirUri_1);
string expectedStatus = nameDir.Equals(failedNameDir) ? "failed" : "active";
System.Console.Out.WriteLine("Checking for the presence of " + nameDir + " in " +
expectedStatus + " name dirs.");
NUnit.Framework.Assert.IsTrue(statusMap[expectedStatus].Contains(nameDir.GetAbsolutePath
()));
}
NUnit.Framework.Assert.AreEqual(1, statusMap["active"].Count);
NUnit.Framework.Assert.AreEqual(1, statusMap["failed"].Count);
NUnit.Framework.Assert.AreEqual(0L, mbs.GetAttribute(mxbeanName, "CacheUsed"));
NUnit.Framework.Assert.AreEqual(NativeIO.POSIX.GetCacheManipulator().GetMemlockLimit
() * cluster.GetDataNodes().Count, mbs.GetAttribute(mxbeanName, "CacheCapacity")
);
NUnit.Framework.Assert.IsNull("RollingUpgradeInfo should be null when there is no rolling"
+ " upgrade", mbs.GetAttribute(mxbeanName, "RollingUpgradeStatus"));
}
finally
{
if (cluster != null)
{
foreach (URI dir in cluster.GetNameDirs(0))
{
FileUtil.Chmod(new FilePath(new FilePath(dir), "current").GetAbsolutePath(), "755"
);
}
cluster.Shutdown();
}
}
}