public virtual void SetupCluster()
{
conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
HAUtil.SetAllowStandbyReads(conf, true);
fsHelper = new FileSystemTestHelper();
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri());
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(1).Build();
cluster.WaitActive();
cluster.TransitionToActive(0);
fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 0, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 1, conf);
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf);
dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf);
KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider
();
fs.GetClient().SetKeyProvider(nn0Provider);
}
public virtual void TestFailoverWithBK()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/hotfailover"
).ToString());
BKJMUtil.AddJournalManagerDefinition(conf);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).ManageNameDfsSharedDirs(false).Build();
NameNode nn1 = cluster.GetNameNode(0);
NameNode nn2 = cluster.GetNameNode(1);
cluster.WaitActive();
cluster.TransitionToActive(0);
Path p = new Path("/testBKJMfailover");
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
fs.Mkdirs(p);
cluster.ShutdownNameNode(0);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.IsTrue(fs.Exists(p));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="Org.Apache.Hadoop.HA.ServiceFailedException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
private void AssertCanStartHANameNodes(MiniDFSCluster cluster, Configuration conf
, string path)
{
// Now should be able to start both NNs. Pass "false" here so that we don't
// try to waitActive on all NNs, since the second NN doesn't exist yet.
cluster.RestartNameNode(0, false);
cluster.RestartNameNode(1, true);
// Make sure HA is working.
cluster.GetNameNode(0).GetRpcServer().TransitionToActive(new HAServiceProtocol.StateChangeRequestInfo
(HAServiceProtocol.RequestSource.RequestByUser));
FileSystem fs = null;
try
{
Path newPath = new Path(path);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(newPath));
HATestUtil.WaitForStandbyToCatchUp(cluster.GetNameNode(0), cluster.GetNameNode(1)
);
NUnit.Framework.Assert.IsTrue(NameNodeAdapter.GetFileInfo(cluster.GetNameNode(1),
newPath.ToString(), false).IsDir());
}
finally
{
if (fs != null)
{
fs.Close();
}
}
}
public virtual void Setup()
{
conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeEnableRetryCacheKey, true);
conf.SetInt(DFSConfigKeys.DfsClientTestDropNamenodeResponseNumKey, 2);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(3).Build();
cluster.WaitActive();
cluster.TransitionToActive(namenodeId);
HATestUtil.SetFailoverConfigurations(cluster, conf);
filesystem = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf);
namesystem = cluster.GetNamesystem(namenodeId);
metrics = namesystem.GetRetryCache().GetMetricsForTests();
}
public virtual void TestDoesntDnsResolveLogicalURI()
{
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NameService spyNS = SpyOnNameService();
string logicalHost = fs.GetUri().GetHost();
Path qualifiedRoot = fs.MakeQualified(new Path("/"));
// Make a few calls against the filesystem.
fs.GetCanonicalServiceName();
fs.ListStatus(qualifiedRoot);
// Ensure that the logical hostname was never resolved.
Org.Mockito.Mockito.Verify(spyNS, Org.Mockito.Mockito.Never()).LookupAllHostAddr(
Org.Mockito.Mockito.Eq(logicalHost));
}
public virtual void TestBootstrapStandbyWithActiveNN()
{
// make nn0 active
cluster.TransitionToActive(0);
// do ops and generate in-progress edit log data
Configuration confNN1 = cluster.GetConfiguration(1);
DistributedFileSystem dfs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs
(cluster, confNN1);
for (int i = 1; i <= 10; i++)
{
dfs.Mkdirs(new Path("/test" + i));
}
dfs.Close();
// shutdown nn1 and delete its edit log files
cluster.ShutdownNameNode(1);
DeleteEditLogIfExists(confNN1);
cluster.GetNameNodeRpc(0).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter,
true);
cluster.GetNameNodeRpc(0).SaveNamespace();
cluster.GetNameNodeRpc(0).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
true);
// check without -skipSharedEditsCheck, Bootstrap should fail for BKJM
// immediately after saveNamespace
int rc = BootstrapStandby.Run(new string[] { "-force", "-nonInteractive" }, confNN1
);
NUnit.Framework.Assert.AreEqual("Mismatches return code", 6, rc);
// check with -skipSharedEditsCheck
rc = BootstrapStandby.Run(new string[] { "-force", "-nonInteractive", "-skipSharedEditsCheck" }, confNN1);
NUnit.Framework.Assert.AreEqual("Mismatches return code", 0, rc);
// Checkpoint as fast as we can, in a tight loop.
confNN1.SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey, 1);
cluster.RestartNameNode(1);
cluster.TransitionToStandby(1);
NameNode nn0 = cluster.GetNameNode(0);
HATestUtil.WaitForStandbyToCatchUp(nn0, cluster.GetNameNode(1));
long expectedCheckpointTxId = NameNodeAdapter.GetNamesystem(nn0).GetFSImage().GetMostRecentCheckpointTxId
();
HATestUtil.WaitForCheckpoint(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
// Should have copied over the namespace
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
FSImageTestUtil.AssertNNFilesMatch(cluster);
}
public virtual void TestFileContextDoesntDnsResolveLogicalURI()
{
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NameService spyNS = SpyOnNameService();
string logicalHost = fs.GetUri().GetHost();
Configuration haClientConf = fs.GetConf();
FileContext fc = FileContext.GetFileContext(haClientConf);
Path root = new Path("/");
fc.ListStatus(root);
fc.ListStatus(fc.MakeQualified(root));
fc.GetDefaultFileSystem().GetCanonicalServiceName();
// Ensure that the logical hostname was never resolved.
Org.Mockito.Mockito.Verify(spyNS, Org.Mockito.Mockito.Never()).LookupAllHostAddr(
Org.Mockito.Mockito.Eq(logicalHost));
}
public virtual void TestFailoverOnConnectTimeout()
{
conf.SetClass(CommonConfigurationKeysPublic.HadoopRpcSocketFactoryClassDefaultKey
, typeof(TestDFSClientFailover.InjectingSocketFactory), typeof(SocketFactory));
// Set up the InjectingSocketFactory to throw a ConnectTimeoutException
// when connecting to the first NN.
TestDFSClientFailover.InjectingSocketFactory.portToInjectOn = cluster.GetNameNodePort
(0);
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
// Make the second NN the active one.
cluster.ShutdownNameNode(0);
cluster.TransitionToActive(1);
// Call a non-idempotent method, and ensure the failover of the call proceeds
// successfully.
IOUtils.CloseStream(fs.Create(TestFile));
}
public override void SetupCluster()
{
Configuration conf = SetupCommonConfig();
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/checkpointing"
+ journalCount++).ToString());
BKJMUtil.AddJournalManagerDefinition(conf);
MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf
("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetHttpPort(10001)).AddNN(new
MiniDFSNNTopology.NNConf("nn2").SetHttpPort(10002)));
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(1).ManageNameDfsSharedDirs
(false).Build();
cluster.WaitActive();
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
cluster.TransitionToActive(0);
}
public virtual void TestMultiplePrimariesStarted()
{
Path p1 = new Path("/testBKJMMultiplePrimary");
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/hotfailoverMultiple"
).ToString());
BKJMUtil.AddJournalManagerDefinition(conf);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).ManageNameDfsSharedDirs(false).CheckExitOnShutdown(false).Build
();
NameNode nn1 = cluster.GetNameNode(0);
NameNode nn2 = cluster.GetNameNode(1);
cluster.WaitActive();
cluster.TransitionToActive(0);
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
fs.Mkdirs(p1);
nn1.GetRpcServer().RollEditLog();
cluster.TransitionToActive(1);
fs = cluster.GetFileSystem(0);
// get the older active server.
try
{
fs.Delete(p1, true);
NUnit.Framework.Assert.Fail("Log update on older active should cause it to exit");
}
catch (RemoteException re)
{
NUnit.Framework.Assert.IsTrue(re.GetClassName().Contains("ExitException"));
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestDfsClientFailover()
{
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.CreateFile(fs, TestFile, FileLengthToVerify, (short)1, 1L);
NUnit.Framework.Assert.AreEqual(fs.GetFileStatus(TestFile).GetLen(), FileLengthToVerify
);
cluster.ShutdownNameNode(0);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.AreEqual(fs.GetFileStatus(TestFile).GetLen(), FileLengthToVerify
);
// Check that it functions even if the URL becomes canonicalized
// to include a port number.
Path withPort = new Path("hdfs://" + HATestUtil.GetLogicalHostname(cluster) + ":"
+ NameNode.DefaultPort + "/" + TestFile.ToUri().GetPath());
FileSystem fs2 = withPort.GetFileSystem(fs.GetConf());
NUnit.Framework.Assert.IsTrue(fs2.Exists(withPort));
fs.Close();
}
public virtual void SetUp()
{
conf = new Configuration();
// Stall the standby checkpointer in two ways
conf.SetLong(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey, long.MaxValue);
conf.SetLong(DFSConfigKeys.DfsNamenodeCheckpointTxnsKey, 20);
// Make it autoroll after 10 edits
conf.SetFloat(DFSConfigKeys.DfsNamenodeEditLogAutorollMultiplierThreshold, 0.5f);
conf.SetInt(DFSConfigKeys.DfsNamenodeEditLogAutorollCheckIntervalMs, 100);
int retryCount = 0;
while (true)
{
try
{
int basePort = 10060 + random.Next(100) * 2;
MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf
("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetHttpPort(basePort)).AddNN(new
MiniDFSNNTopology.NNConf("nn2").SetHttpPort(basePort + 1)));
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).Build
();
cluster.WaitActive();
nn0 = cluster.GetNameNode(0);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
cluster.TransitionToActive(0);
fs = cluster.GetFileSystem(0);
editLog = nn0.GetNamesystem().GetEditLog();
++retryCount;
break;
}
catch (BindException)
{
Log.Info("Set up MiniDFSCluster failed due to port conflicts, retry " + retryCount
+ " times");
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/>
public virtual void TestNNFailover()
{
Configuration conf = new HdfsConfiguration();
MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).Build();
try
{
cluster.GetDfsCluster().WaitActive();
cluster.GetDfsCluster().TransitionToActive(0);
DFSClient client = ((DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster
.GetDfsCluster(), conf)).dfs;
DFSInotifyEventInputStream eis = client.GetInotifyEventStream();
for (int i = 0; i < 10; i++)
{
client.Mkdirs("/dir" + i, null, false);
}
cluster.GetDfsCluster().ShutdownNameNode(0);
cluster.GetDfsCluster().TransitionToActive(1);
EventBatch batch = null;
// we can read all of the edits logged by the old active from the new
// active
for (int i_1 = 0; i_1 < 10; i_1++)
{
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Create);
NUnit.Framework.Assert.IsTrue(((Event.CreateEvent)batch.GetEvents()[0]).GetPath()
.Equals("/dir" + i_1));
}
NUnit.Framework.Assert.IsTrue(eis.Poll() == null);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestFailoverWithFailingBKCluster()
{
int ensembleSize = numBookies + 1;
BookieServer newBookie = bkutil.NewBookie();
NUnit.Framework.Assert.AreEqual("New bookie didn't start", ensembleSize, bkutil.CheckBookiesUp
(ensembleSize, 10));
BookieServer replacementBookie = null;
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/hotfailoverWithFail"
).ToString());
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize);
BKJMUtil.AddJournalManagerDefinition(conf);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).ManageNameDfsSharedDirs(false).CheckExitOnShutdown(false).Build
();
NameNode nn1 = cluster.GetNameNode(0);
NameNode nn2 = cluster.GetNameNode(1);
cluster.WaitActive();
cluster.TransitionToActive(0);
Path p1 = new Path("/testBKJMFailingBKCluster1");
Path p2 = new Path("/testBKJMFailingBKCluster2");
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
fs.Mkdirs(p1);
newBookie.Shutdown();
// will take down shared storage
NUnit.Framework.Assert.AreEqual("New bookie didn't stop", numBookies, bkutil.CheckBookiesUp
(numBookies, 10));
try
{
fs.Mkdirs(p2);
NUnit.Framework.Assert.Fail("mkdirs should result in the NN exiting");
}
catch (RemoteException re)
{
NUnit.Framework.Assert.IsTrue(re.GetClassName().Contains("ExitException"));
}
cluster.ShutdownNameNode(0);
try
{
cluster.TransitionToActive(1);
NUnit.Framework.Assert.Fail("Shouldn't have been able to transition with bookies down"
);
}
catch (ExitUtil.ExitException ee)
{
NUnit.Framework.Assert.IsTrue("Should shutdown due to required journal failure",
ee.Message.Contains("starting log segment 3 failed for required journal"));
}
replacementBookie = bkutil.NewBookie();
NUnit.Framework.Assert.AreEqual("Replacement bookie didn't start", ensembleSize,
bkutil.CheckBookiesUp(ensembleSize, 10));
cluster.TransitionToActive(1);
// should work fine now
NUnit.Framework.Assert.IsTrue(fs.Exists(p1));
NUnit.Framework.Assert.IsFalse(fs.Exists(p2));
}
finally
{
newBookie.Shutdown();
if (replacementBookie != null)
{
replacementBookie.Shutdown();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}