public virtual void TestPutMetrics2()
{
GraphiteSink sink = new GraphiteSink();
IList <MetricsTag> tags = new AList <MetricsTag>();
tags.AddItem(new MetricsTag(MsInfo.Context, "all"));
tags.AddItem(new MetricsTag(MsInfo.Hostname, null));
ICollection <AbstractMetric> metrics = new HashSet <AbstractMetric>();
metrics.AddItem(MakeMetric("foo1", 1));
metrics.AddItem(MakeMetric("foo2", 2));
MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long)10000, tags, metrics
);
ArgumentCaptor <string> argument = ArgumentCaptor.ForClass <string>();
GraphiteSink.Graphite mockGraphite = MakeGraphite();
Whitebox.SetInternalState(sink, "graphite", mockGraphite);
sink.PutMetrics(record);
try
{
Org.Mockito.Mockito.Verify(mockGraphite).Write(argument.Capture());
}
catch (IOException e)
{
Runtime.PrintStackTrace(e);
}
string result = argument.GetValue();
Assert.Equal(true, result.Equals("null.all.Context.Context=all.foo1 1 10\n"
+ "null.all.Context.Context=all.foo2 2 10\n") || result.Equals("null.all.Context.Context=all.foo2 2 10\n"
+ "null.all.Context.Context=all.foo1 1 10\n"));
}
/// <summary>Make sure the client retries when the active NN is in safemode</summary>
/// <exception cref="System.Exception"/>
public virtual void TestClientRetrySafeMode()
{
IDictionary <Path, bool> results = Collections.SynchronizedMap(new Dictionary <Path
, bool>());
Path test = new Path("/test");
// let nn0 enter safemode
NameNodeAdapter.EnterSafeMode(nn0, false);
FSNamesystem.SafeModeInfo safeMode = (FSNamesystem.SafeModeInfo)Whitebox.GetInternalState
(nn0.GetNamesystem(), "safeMode");
Whitebox.SetInternalState(safeMode, "extension", Sharpen.Extensions.ValueOf(30000
));
Log.Info("enter safemode");
new _Thread_133(this, test, results).Start();
// make sure the client's call has actually been handled by the active NN
NUnit.Framework.Assert.IsFalse("The directory should not be created while NN in safemode"
, fs.Exists(test));
Sharpen.Thread.Sleep(1000);
// let nn0 leave safemode
NameNodeAdapter.LeaveSafeMode(nn0);
Log.Info("leave safemode");
lock (this)
{
while (!results.Contains(test))
{
Sharpen.Runtime.Wait(this);
}
NUnit.Framework.Assert.IsTrue(results[test]);
}
}
public virtual void TestRaceWhileNNStartup()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
NameNode namenode = cluster.GetNameNode();
NamenodeProtocols rpcServer = namenode.GetRpcServer();
Whitebox.SetInternalState(namenode, "rpcServer", null);
Path foo = new Path("/foo");
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
try
{
webHdfs.Mkdirs(foo);
NUnit.Framework.Assert.Fail("Expected RetriableException");
}
catch (RetriableException e)
{
GenericTestUtils.AssertExceptionContains("Namenode is in startup mode", e);
}
Whitebox.SetInternalState(namenode, "rpcServer", rpcServer);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
//Do Nothing
/// <exception cref="System.Exception"/>
public virtual void TestWebHdfsDoAs()
{
WebHdfsTestUtil.Log.Info("START: testWebHdfsDoAs()");
WebHdfsTestUtil.Log.Info("ugi.getShortUserName()=" + ugi.GetShortUserName());
WebHdfsFileSystem webhdfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(ugi, config, WebHdfsFileSystem
.Scheme);
Path root = new Path("/");
cluster.GetFileSystem().SetPermission(root, new FsPermission((short)0x1ff));
Whitebox.SetInternalState(webhdfs, "ugi", proxyUgi);
{
Path responsePath = webhdfs.GetHomeDirectory();
WebHdfsTestUtil.Log.Info("responsePath=" + responsePath);
NUnit.Framework.Assert.AreEqual(webhdfs.GetUri() + "/user/" + ProxyUser, responsePath
.ToString());
}
Path f = new Path("/testWebHdfsDoAs/a.txt");
{
FSDataOutputStream @out = webhdfs.Create(f);
@out.Write(Sharpen.Runtime.GetBytesForString("Hello, webhdfs user!"));
@out.Close();
FileStatus status = webhdfs.GetFileStatus(f);
WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner());
NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner());
}
{
FSDataOutputStream @out = webhdfs.Append(f);
@out.Write(Sharpen.Runtime.GetBytesForString("\nHello again!"));
@out.Close();
FileStatus status = webhdfs.GetFileStatus(f);
WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner());
WebHdfsTestUtil.Log.Info("status.getLen() =" + status.GetLen());
NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner());
}
}
public virtual void TestReplQueuesActiveAfterStartupSafemode()
{
Configuration conf = new Configuration();
FSEditLog fsEditLog = Org.Mockito.Mockito.Mock <FSEditLog>();
FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>();
Org.Mockito.Mockito.When(fsImage.GetEditLog()).ThenReturn(fsEditLog);
FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage);
FSNamesystem fsn = Org.Mockito.Mockito.Spy(fsNamesystem);
//Make shouldPopulaeReplQueues return true
HAContext haContext = Org.Mockito.Mockito.Mock <HAContext>();
HAState haState = Org.Mockito.Mockito.Mock <HAState>();
Org.Mockito.Mockito.When(haContext.GetState()).ThenReturn(haState);
Org.Mockito.Mockito.When(haState.ShouldPopulateReplQueues()).ThenReturn(true);
Whitebox.SetInternalState(fsn, "haContext", haContext);
//Make NameNode.getNameNodeMetrics() not return null
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
fsn.EnterSafeMode(false);
NUnit.Framework.Assert.IsTrue("FSNamesystem didn't enter safemode", fsn.IsInSafeMode
());
NUnit.Framework.Assert.IsTrue("Replication queues were being populated during very first "
+ "safemode", !fsn.IsPopulatingReplQueues());
fsn.LeaveSafeMode();
NUnit.Framework.Assert.IsTrue("FSNamesystem didn't leave safemode", !fsn.IsInSafeMode
());
NUnit.Framework.Assert.IsTrue("Replication queues weren't being populated even after leaving "
+ "safemode", fsn.IsPopulatingReplQueues());
fsn.EnterSafeMode(false);
NUnit.Framework.Assert.IsTrue("FSNamesystem didn't enter safemode", fsn.IsInSafeMode
());
NUnit.Framework.Assert.IsTrue("Replication queues weren't being populated after entering "
+ "safemode 2nd time", fsn.IsPopulatingReplQueues());
}
public virtual void TestPendingDeleteUnknownBlocks()
{
int fileNum = 5;
// 5 files
Path[] files = new Path[fileNum];
MiniDFSCluster.DataNodeProperties[] dnprops = new MiniDFSCluster.DataNodeProperties
[Replication];
// create a group of files, each file contains 1 block
for (int i = 0; i < fileNum; i++)
{
files[i] = new Path("/file" + i);
DFSTestUtil.CreateFile(dfs, files[i], Blocksize, Replication, i);
}
// wait until all DataNodes have replicas
WaitForReplication();
for (int i_1 = Replication - 1; i_1 >= 0; i_1--)
{
dnprops[i_1] = cluster.StopDataNode(i_1);
}
Sharpen.Thread.Sleep(2000);
// delete 2 files, we still have 3 files remaining so that we can cover
// every DN storage
for (int i_2 = 0; i_2 < 2; i_2++)
{
dfs.Delete(files[i_2], true);
}
// restart NameNode
cluster.RestartNameNode(false);
InvalidateBlocks invalidateBlocks = (InvalidateBlocks)Whitebox.GetInternalState(cluster
.GetNamesystem().GetBlockManager(), "invalidateBlocks");
InvalidateBlocks mockIb = Org.Mockito.Mockito.Spy(invalidateBlocks);
Org.Mockito.Mockito.DoReturn(1L).When(mockIb).GetInvalidationDelay();
Whitebox.SetInternalState(cluster.GetNamesystem().GetBlockManager(), "invalidateBlocks"
, mockIb);
NUnit.Framework.Assert.AreEqual(0L, cluster.GetNamesystem().GetPendingDeletionBlocks
());
// restart DataNodes
for (int i_3 = 0; i_3 < Replication; i_3++)
{
cluster.RestartDataNode(dnprops[i_3], true);
}
cluster.WaitActive();
for (int i_4 = 0; i_4 < Replication; i_4++)
{
DataNodeTestUtils.TriggerBlockReport(cluster.GetDataNodes()[i_4]);
}
Sharpen.Thread.Sleep(2000);
// make sure we have received block reports by checking the total block #
NUnit.Framework.Assert.AreEqual(3, cluster.GetNamesystem().GetBlocksTotal());
NUnit.Framework.Assert.AreEqual(4, cluster.GetNamesystem().GetPendingDeletionBlocks
());
cluster.RestartNameNode(true);
Sharpen.Thread.Sleep(6000);
NUnit.Framework.Assert.AreEqual(3, cluster.GetNamesystem().GetBlocksTotal());
NUnit.Framework.Assert.AreEqual(0, cluster.GetNamesystem().GetPendingDeletionBlocks
());
}
public static FSImage SpyOnFsImage(NameNode nn1)
{
FSNamesystem fsn = nn1.GetNamesystem();
FSImage spy = Org.Mockito.Mockito.Spy(fsn.GetFSImage());
Whitebox.SetInternalState(fsn, "fsImage", spy);
return(spy);
}
public virtual void TestIncludeExcludeLists()
{
BlockManager bm = Org.Mockito.Mockito.Mock <BlockManager>();
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Configuration conf = new Configuration();
HostFileManager hm = new HostFileManager();
HostFileManager.HostSet includedNodes = new HostFileManager.HostSet();
HostFileManager.HostSet excludedNodes = new HostFileManager.HostSet();
includedNodes.Add(Entry("127.0.0.1:12345"));
includedNodes.Add(Entry("localhost:12345"));
includedNodes.Add(Entry("127.0.0.1:12345"));
includedNodes.Add(Entry("127.0.0.2"));
excludedNodes.Add(Entry("127.0.0.1:12346"));
excludedNodes.Add(Entry("127.0.30.1:12346"));
NUnit.Framework.Assert.AreEqual(2, includedNodes.Size());
NUnit.Framework.Assert.AreEqual(2, excludedNodes.Size());
hm.Refresh(includedNodes, excludedNodes);
DatanodeManager dm = new DatanodeManager(bm, fsn, conf);
Whitebox.SetInternalState(dm, "hostFileManager", hm);
IDictionary <string, DatanodeDescriptor> dnMap = (IDictionary <string, DatanodeDescriptor
>)Whitebox.GetInternalState(dm, "datanodeMap");
// After the de-duplication, there should be only one DN from the included
// nodes declared as dead.
NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All).Count);
NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
dnMap["uuid-foo"] = new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost"
, "uuid-foo", 12345, 1020, 1021, 1022));
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
dnMap["uuid-bar"] = new DatanodeDescriptor(new DatanodeID("127.0.0.2", "127.0.0.2"
, "uuid-bar", 12345, 1020, 1021, 1022));
NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
DatanodeDescriptor spam = new DatanodeDescriptor(new DatanodeID("127.0.0" + ".3",
"127.0.0.3", "uuid-spam", 12345, 1020, 1021, 1022));
DFSTestUtil.SetDatanodeDead(spam);
includedNodes.Add(Entry("127.0.0.3:12345"));
dnMap["uuid-spam"] = spam;
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
Sharpen.Collections.Remove(dnMap, "uuid-spam");
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
excludedNodes.Add(Entry("127.0.0.3"));
NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
}
/// <exception cref="System.Exception"/>
private void TestUpgrade(TestBootstrapStandbyWithQJM.UpgradeState state)
{
cluster.TransitionToActive(0);
Configuration confNN1 = cluster.GetConfiguration(1);
FilePath current = cluster.GetNameNode(1).GetFSImage().GetStorage().GetStorageDir
(0).GetCurrentDir();
FilePath tmp = cluster.GetNameNode(1).GetFSImage().GetStorage().GetStorageDir(0).
GetPreviousTmp();
// shut down nn1
cluster.ShutdownNameNode(1);
// make NN0 in upgrade state
FSImage fsImage0 = cluster.GetNameNode(0).GetNamesystem().GetFSImage();
Whitebox.SetInternalState(fsImage0, "isUpgradeFinalized", false);
switch (state)
{
case TestBootstrapStandbyWithQJM.UpgradeState.Recover:
{
// rename the current directory to previous.tmp in nn1
NNStorage.Rename(current, tmp);
break;
}
case TestBootstrapStandbyWithQJM.UpgradeState.Format:
{
// rename the current directory to a random name so it's not formatted
FilePath wrongPath = new FilePath(current.GetParentFile(), "wrong");
NNStorage.Rename(current, wrongPath);
break;
}
default:
{
break;
}
}
int rc = BootstrapStandby.Run(new string[] { "-force" }, confNN1);
NUnit.Framework.Assert.AreEqual(0, rc);
// Should have copied over the namespace from the standby
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of(0));
FSImageTestUtil.AssertNNFilesMatch(cluster);
// make sure the NN1 is in upgrade state, i.e., the previous directory has
// been successfully created
cluster.RestartNameNode(1);
NUnit.Framework.Assert.IsFalse(cluster.GetNameNode(1).GetNamesystem().IsUpgradeFinalized
());
}
public virtual void TestPutMetrics3()
{
// setup GraphiteSink
GraphiteSink sink = new GraphiteSink();
GraphiteSink.Graphite mockGraphite = MakeGraphite();
Whitebox.SetInternalState(sink, "graphite", mockGraphite);
// given two metrics records with timestamps 1000 milliseconds apart.
IList <MetricsTag> tags = Collections.EmptyList();
ICollection <AbstractMetric> metrics = new HashSet <AbstractMetric>();
metrics.AddItem(MakeMetric("foo1", 1));
MetricsRecord record1 = new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags
, metrics);
MetricsRecord record2 = new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags
, metrics);
sink.PutMetrics(record1);
sink.PutMetrics(record2);
sink.Flush();
try
{
sink.Close();
}
catch (IOException e)
{
Runtime.PrintStackTrace(e);
}
// then the timestamps in the graphite stream should differ by one second.
try
{
Org.Mockito.Mockito.Verify(mockGraphite).Write(Matchers.Eq("null.default.Context.foo1 1 1000000000\n"
));
Org.Mockito.Mockito.Verify(mockGraphite).Write(Matchers.Eq("null.default.Context.foo1 1 1000000001\n"
));
}
catch (IOException e)
{
Runtime.PrintStackTrace(e);
}
}
public virtual void TestSetTokenServiceAndKind()
{
MiniDFSCluster cluster = null;
try
{
Configuration clusterConf = new HdfsConfiguration(conf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple
, clusterConf);
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true
);
// trick the NN into thinking s[ecurity is enabled w/o it trying
// to login from a keytab
UserGroupInformation.SetConfiguration(clusterConf);
cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(0).Build();
cluster.WaitActive();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, clusterConf);
WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(clusterConf, "webhdfs"
);
Whitebox.SetInternalState(fs, "canRefreshDelegationToken", true);
URLConnectionFactory factory = new _URLConnectionFactory_268(new _ConnectionConfigurator_262
());
Whitebox.SetInternalState(fs, "connectionFactory", factory);
Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs.GetDelegationToken();
NUnit.Framework.Assert.AreEqual(new Text("bar"), token1.GetKind());
HttpOpParam.OP op = GetOpParam.OP.Getdelegationtoken;
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = new _FsPathResponseRunner_281
(op, null, new RenewerParam(null)).Run();
NUnit.Framework.Assert.AreEqual(new Text("bar"), token2.GetKind());
NUnit.Framework.Assert.AreEqual(new Text("foo"), token2.GetService());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestFileStatusPipeFile()
{
RawLocalFileSystem origFs = new RawLocalFileSystem();
RawLocalFileSystem fs = Org.Mockito.Mockito.Spy(origFs);
Configuration conf = Org.Mockito.Mockito.Mock <Configuration>();
fs.SetConf(conf);
Whitebox.SetInternalState(fs, "useDeprecatedFileStatus", false);
Path path = new Path("/foo");
FilePath pipe = Org.Mockito.Mockito.Mock <FilePath>();
Org.Mockito.Mockito.When(pipe.IsFile()).ThenReturn(false);
Org.Mockito.Mockito.When(pipe.IsDirectory()).ThenReturn(false);
Org.Mockito.Mockito.When(pipe.Exists()).ThenReturn(true);
FileStatus stat = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.DoReturn(pipe).When(fs).PathToFile(path);
Org.Mockito.Mockito.DoReturn(stat).When(fs).GetFileStatus(path);
FileStatus[] stats = fs.ListStatus(path);
Assert.True(stats != null && stats.Length == 1 && stats[0] == stat
);
}
public virtual void TestClose()
{
GraphiteSink sink = new GraphiteSink();
GraphiteSink.Graphite mockGraphite = MakeGraphite();
Whitebox.SetInternalState(sink, "graphite", mockGraphite);
try
{
sink.Close();
}
catch (IOException ioe)
{
Runtime.PrintStackTrace(ioe);
}
try
{
Org.Mockito.Mockito.Verify(mockGraphite).Close();
}
catch (IOException ioe)
{
Runtime.PrintStackTrace(ioe);
}
}
/// <summary>
/// Make sure the WebHdfsFileSystem will retry based on RetriableException when
/// rpcServer is null in NamenodeWebHdfsMethods while NameNode starts up.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestRetryWhileNNStartup()
{
Configuration conf = DFSTestUtil.NewHAConfiguration(LogicalName);
MiniDFSCluster cluster = null;
IDictionary <string, bool> resultMap = new Dictionary <string, bool>();
try
{
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topo).NumDataNodes(0).Build
();
HATestUtil.SetFailoverConfigurations(cluster, conf, LogicalName);
cluster.WaitActive();
cluster.TransitionToActive(0);
NameNode namenode = cluster.GetNameNode(0);
NamenodeProtocols rpcServer = namenode.GetRpcServer();
Whitebox.SetInternalState(namenode, "rpcServer", null);
new _Thread_212(this, conf, resultMap).Start();
Sharpen.Thread.Sleep(1000);
Whitebox.SetInternalState(namenode, "rpcServer", rpcServer);
lock (this)
{
while (!resultMap.Contains("mkdirs"))
{
Sharpen.Runtime.Wait(this);
}
NUnit.Framework.Assert.IsTrue(resultMap["mkdirs"]);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestFailureAndPutMetrics()
{
GraphiteSink sink = new GraphiteSink();
IList <MetricsTag> tags = new AList <MetricsTag>();
tags.AddItem(new MetricsTag(MsInfo.Context, "all"));
tags.AddItem(new MetricsTag(MsInfo.Hostname, "host"));
ICollection <AbstractMetric> metrics = new HashSet <AbstractMetric>();
metrics.AddItem(MakeMetric("foo1", 1.25));
metrics.AddItem(MakeMetric("foo2", 2.25));
MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long)10000, tags, metrics
);
GraphiteSink.Graphite mockGraphite = MakeGraphite();
Whitebox.SetInternalState(sink, "graphite", mockGraphite);
// throw exception when first try
Org.Mockito.Mockito.DoThrow(new IOException("IO exception")).When(mockGraphite).Write
(Matchers.AnyString());
sink.PutMetrics(record);
Org.Mockito.Mockito.Verify(mockGraphite).Write(Matchers.AnyString());
Org.Mockito.Mockito.Verify(mockGraphite).Close();
// reset mock and try again
Org.Mockito.Mockito.Reset(mockGraphite);
Org.Mockito.Mockito.When(mockGraphite.IsConnected()).ThenReturn(false);
ArgumentCaptor <string> argument = ArgumentCaptor.ForClass <string>();
sink.PutMetrics(record);
Org.Mockito.Mockito.Verify(mockGraphite).Write(argument.Capture());
string result = argument.GetValue();
Assert.Equal(true, result.Equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n"
+ "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || result.Equals
("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n"
));
}
/// <summary>
/// Verify that a saveNamespace command brings faulty directories
/// in fs.name.dir and fs.edit.dir back online.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestReinsertnamedirsInSavenamespace()
{
// create a configuration with the key to restore error
// directories in fs.name.dir
Configuration conf = GetConf();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeNameDirRestoreKey, true);
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage originalImage = fsn.GetFSImage();
NNStorage storage = originalImage.GetStorage();
FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);
Whitebox.SetInternalState(fsn, "fsImage", spyImage);
FileSystem fs = FileSystem.GetLocal(conf);
FilePath rootDir = storage.GetStorageDir(0).GetRoot();
Path rootPath = new Path(rootDir.GetPath(), "current");
FsPermission permissionNone = new FsPermission((short)0);
FsPermission permissionAll = new FsPermission(FsAction.All, FsAction.ReadExecute,
FsAction.ReadExecute);
fs.SetPermission(rootPath, permissionNone);
try
{
DoAnEdit(fsn, 1);
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
// Save namespace - should mark the first storage dir as faulty
// since it's not traversable.
Log.Info("Doing the first savenamespace.");
fsn.SaveNamespace();
Log.Info("First savenamespace sucessful.");
NUnit.Framework.Assert.IsTrue("Savenamespace should have marked one directory as bad."
+ " But found " + storage.GetRemovedStorageDirs().Count + " bad directories.",
storage.GetRemovedStorageDirs().Count == 1);
fs.SetPermission(rootPath, permissionAll);
// The next call to savenamespace should try inserting the
// erroneous directory back to fs.name.dir. This command should
// be successful.
Log.Info("Doing the second savenamespace.");
fsn.SaveNamespace();
Log.Warn("Second savenamespace sucessful.");
NUnit.Framework.Assert.IsTrue("Savenamespace should have been successful in removing "
+ " bad directories from Image." + " But found " + storage.GetRemovedStorageDirs
().Count + " bad directories.", storage.GetRemovedStorageDirs().Count == 0);
// Now shut down and restart the namesystem
Log.Info("Shutting down fsimage.");
originalImage.Close();
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
Log.Info("Loading new FSmage from disk.");
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edit.
Log.Info("Checking reloaded image.");
CheckEditExists(fsn, 1);
Log.Info("Reloaded image is good.");
}
finally
{
if (rootDir.Exists())
{
fs.SetPermission(rootPath, permissionAll);
}
if (fsn != null)
{
try
{
fsn.Close();
}
catch (Exception t)
{
Log.Fatal("Failed to shut down", t);
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCancelSaveNamespace()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage image = fsn.GetFSImage();
NNStorage storage = image.GetStorage();
storage.Close();
// unlock any directories that FSNamesystem's initialization may have locked
storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem.GetNamespaceEditsDirs
(conf));
FSNamesystem spyFsn = Org.Mockito.Mockito.Spy(fsn);
FSNamesystem finalFsn = spyFsn;
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
BlockIdManager bid = Org.Mockito.Mockito.Spy(spyFsn.GetBlockIdManager());
Whitebox.SetInternalState(finalFsn, "blockIdManager", bid);
Org.Mockito.Mockito.DoAnswer(delayer).When(bid).GetGenerationStampV2();
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
DoAnEdit(fsn, 1);
Canceler canceler = new Canceler();
// Save namespace
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
Future <Void> saverFuture = pool.Submit(new _Callable_561(image, finalFsn, canceler
));
// Wait until saveNamespace calls getGenerationStamp
delayer.WaitForCall();
// then cancel the saveNamespace
Future <Void> cancelFuture = pool.Submit(new _Callable_572(canceler));
// give the cancel call time to run
Sharpen.Thread.Sleep(500);
// allow saveNamespace to proceed - it should check the cancel flag after
// this point and throw an exception
delayer.Proceed();
cancelFuture.Get();
saverFuture.Get();
NUnit.Framework.Assert.Fail("saveNamespace did not fail even though cancelled!");
}
catch (Exception t)
{
GenericTestUtils.AssertExceptionContains("SaveNamespaceCancelledException", t);
}
Log.Info("Successfully cancelled a saveNamespace");
// Check that we have only the original image and not any
// cruft left over from half-finished images
FSImageTestUtil.LogStorageContents(Log, storage);
foreach (Storage.StorageDirectory sd in storage.DirIterable(null))
{
FilePath curDir = sd.GetCurrentDir();
GenericTestUtils.AssertGlobEquals(curDir, "fsimage_.*", NNStorage.GetImageFileName
(0), NNStorage.GetImageFileName(0) + MD5FileUtils.Md5Suffix);
}
}
finally
{
fsn.Close();
}
}
/// <exception cref="System.Exception"/>
private void SaveNamespaceWithInjectedFault(TestSaveNamespace.Fault fault)
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage originalImage = fsn.GetFSImage();
NNStorage storage = originalImage.GetStorage();
NNStorage spyStorage = Org.Mockito.Mockito.Spy(storage);
originalImage.storage = spyStorage;
FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);
Whitebox.SetInternalState(fsn, "fsImage", spyImage);
bool shouldFail = false;
switch (fault)
{
case TestSaveNamespace.Fault.SaveSecondFsimageRte:
{
// should we expect the save operation to fail
// inject fault
// The spy throws a RuntimeException when writing to the second directory
Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyImage
).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
)Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
shouldFail = false;
break;
}
case TestSaveNamespace.Fault.SaveSecondFsimageIoe:
{
// The spy throws an IOException when writing to the second directory
Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(false)).When(spyImage
).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
)Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
shouldFail = false;
break;
}
case TestSaveNamespace.Fault.SaveAllFsimages:
{
// The spy throws IOException in all directories
Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyImage).SaveFSImage
((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory)Matchers.AnyObject
(), (NNStorage.NameNodeFile)Matchers.AnyObject());
shouldFail = true;
break;
}
case TestSaveNamespace.Fault.WriteStorageAll:
{
// The spy throws an exception before writing any VERSION files
Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyStorage).WriteAll
();
shouldFail = true;
break;
}
case TestSaveNamespace.Fault.WriteStorageOne:
{
// The spy throws on exception on one particular storage directory
Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyStorage
).WriteProperties((Storage.StorageDirectory)Matchers.AnyObject());
// TODO: unfortunately this fails -- should be improved.
// See HDFS-2173.
shouldFail = true;
break;
}
}
try
{
DoAnEdit(fsn, 1);
// Save namespace - this may fail, depending on fault injected
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
fsn.SaveNamespace();
if (shouldFail)
{
NUnit.Framework.Assert.Fail("Did not fail!");
}
}
catch (Exception e)
{
if (!shouldFail)
{
throw;
}
else
{
Log.Info("Test caught expected exception", e);
}
}
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
// Should still be able to perform edits
DoAnEdit(fsn, 2);
// Now shut down and restart the namesystem
originalImage.Close();
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edits.
CheckEditExists(fsn, 1);
CheckEditExists(fsn, 2);
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}
public virtual void TestPropagatedClose()
{
ByteRangeInputStream bris = Org.Mockito.Mockito.Mock <ByteRangeInputStream>(Org.Mockito.Mockito
.CallsRealMethods);
InputStream mockStream = Org.Mockito.Mockito.Mock <InputStream>();
Org.Mockito.Mockito.DoReturn(mockStream).When(bris).OpenInputStream();
Whitebox.SetInternalState(bris, "status", ByteRangeInputStream.StreamStatus.Seek);
int brisOpens = 0;
int brisCloses = 0;
int isCloses = 0;
// first open, shouldn't close underlying stream
bris.GetInputStream();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(++brisOpens)).OpenInputStream
();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(isCloses)).Close
();
// stream is open, shouldn't close underlying stream
bris.GetInputStream();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisOpens)).OpenInputStream
();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(isCloses)).Close
();
// seek forces a reopen, should close underlying stream
bris.Seek(1);
bris.GetInputStream();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(++brisOpens)).OpenInputStream
();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(++isCloses)).Close
();
// verify that the underlying stream isn't closed after a seek
// ie. the state was correctly updated
bris.GetInputStream();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisOpens)).OpenInputStream
();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(isCloses)).Close
();
// seeking to same location should be a no-op
bris.Seek(1);
bris.GetInputStream();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisOpens)).OpenInputStream
();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(isCloses)).Close
();
// close should of course close
bris.Close();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(++brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(++isCloses)).Close
();
// it's already closed, underlying stream should not close
bris.Close();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(++brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(isCloses)).Close
();
// it's closed, don't reopen it
bool errored = false;
try
{
bris.GetInputStream();
}
catch (IOException e)
{
errored = true;
NUnit.Framework.Assert.AreEqual("Stream closed", e.Message);
}
finally
{
NUnit.Framework.Assert.IsTrue("Read a closed steam", errored);
}
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisOpens)).OpenInputStream
();
Org.Mockito.Mockito.Verify(bris, Org.Mockito.Mockito.Times(brisCloses)).Close();
Org.Mockito.Mockito.Verify(mockStream, Org.Mockito.Mockito.Times(isCloses)).Close
();
}
public virtual void TestRemoveIncludedNode()
{
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
// Set the write lock so that the DatanodeManager can start
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
DatanodeManager dm = MockDatanodeManager(fsn, new Configuration());
HostFileManager hm = new HostFileManager();
HostFileManager.HostSet noNodes = new HostFileManager.HostSet();
HostFileManager.HostSet oneNode = new HostFileManager.HostSet();
HostFileManager.HostSet twoNodes = new HostFileManager.HostSet();
DatanodeRegistration dr1 = new DatanodeRegistration(new DatanodeID("127.0.0.1", "127.0.0.1"
, "someStorageID-123", 12345, 12345, 12345, 12345), new StorageInfo(HdfsServerConstants.NodeType
.DataNode), new ExportedBlockKeys(), "test");
DatanodeRegistration dr2 = new DatanodeRegistration(new DatanodeID("127.0.0.1", "127.0.0.1"
, "someStorageID-234", 23456, 23456, 23456, 23456), new StorageInfo(HdfsServerConstants.NodeType
.DataNode), new ExportedBlockKeys(), "test");
twoNodes.Add(Entry("127.0.0.1:12345"));
twoNodes.Add(Entry("127.0.0.1:23456"));
oneNode.Add(Entry("127.0.0.1:23456"));
hm.Refresh(twoNodes, noNodes);
Whitebox.SetInternalState(dm, "hostFileManager", hm);
// Register two data nodes to simulate them coming up.
// We need to add two nodes, because if we have only one node, removing it
// will cause the includes list to be empty, which means all hosts will be
// allowed.
dm.RegisterDatanode(dr1);
dm.RegisterDatanode(dr2);
// Make sure that both nodes are reported
IList <DatanodeDescriptor> both = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
// Sort the list so that we know which one is which
both.Sort();
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 2, both.Count
);
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:12345", both[0].GetInfoAddr());
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:23456", both[1].GetInfoAddr());
// Remove one node from includes, but do not add it to excludes.
hm.Refresh(oneNode, noNodes);
// Make sure that only one node is still reported
IList <DatanodeDescriptor> onlyOne = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 1, onlyOne.
Count);
NUnit.Framework.Assert.AreEqual("Unexpected host reported", "127.0.0.1:23456", onlyOne
[0].GetInfoAddr());
// Remove all nodes from includes
hm.Refresh(noNodes, noNodes);
// Check that both nodes are reported again
IList <DatanodeDescriptor> bothAgain = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
// Sort the list so that we know which one is which
bothAgain.Sort();
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 2, bothAgain
.Count);
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:12345", bothAgain[0].GetInfoAddr());
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:23456", bothAgain[1].GetInfoAddr());
}
/// <summary>Injects a failure on all storage directories while saving namespace.</summary>
/// <param name="restoreStorageAfterFailure">
/// if true, will try to save again after
/// clearing the failure injection
/// </param>
/// <exception cref="System.Exception"/>
public virtual void DoTestFailedSaveNamespace(bool restoreStorageAfterFailure)
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage originalImage = fsn.GetFSImage();
NNStorage storage = originalImage.GetStorage();
storage.Close();
// unlock any directories that FSNamesystem's initialization may have locked
NNStorage spyStorage = Org.Mockito.Mockito.Spy(storage);
originalImage.storage = spyStorage;
FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);
Whitebox.SetInternalState(fsn, "fsImage", spyImage);
spyImage.storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem
.GetNamespaceEditsDirs(conf));
Org.Mockito.Mockito.DoThrow(new IOException("Injected fault: saveFSImage")).When(
spyImage).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
)Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
try
{
DoAnEdit(fsn, 1);
// Save namespace
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
fsn.SaveNamespace();
NUnit.Framework.Assert.Fail("saveNamespace did not fail even when all directories failed!"
);
}
catch (IOException ioe)
{
Log.Info("Got expected exception", ioe);
}
// Ensure that, if storage dirs come back online, things work again.
if (restoreStorageAfterFailure)
{
Org.Mockito.Mockito.Reset(spyImage);
spyStorage.SetRestoreFailedStorage(true);
fsn.SaveNamespace();
CheckEditExists(fsn, 1);
}
// Now shut down and restart the NN
originalImage.Close();
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edits.
CheckEditExists(fsn, 1);
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}