public virtual void TestUpgradeFromCorruptRel22Image()
{
UnpackStorage(Hadoop22Image, HadoopDfsDirTxt);
// Overwrite the md5 stored in the VERSION files
FilePath baseDir = new FilePath(MiniDFSCluster.GetBaseDirectory());
FSImageTestUtil.CorruptVersionFile(new FilePath(baseDir, "name1/current/VERSION")
, "imageMD5Digest", "22222222222222222222222222222222");
FSImageTestUtil.CorruptVersionFile(new FilePath(baseDir, "name2/current/VERSION")
, "imageMD5Digest", "22222222222222222222222222222222");
// Attach our own log appender so we can verify output
LogVerificationAppender appender = new LogVerificationAppender();
Logger logger = Logger.GetRootLogger();
logger.AddAppender(appender);
// Upgrade should now fail
try
{
UpgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).NumDataNodes(4), null);
NUnit.Framework.Assert.Fail("Upgrade did not fail with bad MD5");
}
catch (IOException ioe)
{
string msg = StringUtils.StringifyException(ioe);
if (!msg.Contains("Failed to load an FSImage file"))
{
throw;
}
int md5failures = appender.CountExceptionsWithMessage(" is corrupt with MD5 checksum of "
);
NUnit.Framework.Assert.AreEqual("Upgrade did not fail with bad MD5", 1, md5failures
);
}
}
/// <exception cref="System.IO.IOException"/>
private MiniJournalCluster(MiniJournalCluster.Builder b)
{
Log.Info("Starting MiniJournalCluster with " + b.numJournalNodes + " journal nodes"
);
if (b.baseDir != null)
{
this.baseDir = new FilePath(b.baseDir);
}
else
{
this.baseDir = new FilePath(MiniDFSCluster.GetBaseDirectory());
}
nodes = new MiniJournalCluster.JNInfo[b.numJournalNodes];
for (int i = 0; i < b.numJournalNodes; i++)
{
if (b.format)
{
FilePath dir = GetStorageDir(i);
Log.Debug("Fully deleting JN directory " + dir);
FileUtil.FullyDelete(dir);
}
JournalNode jn = new JournalNode();
jn.SetConf(CreateConfForNode(b, i));
jn.Start();
nodes[i] = new MiniJournalCluster.JNInfo(jn);
}
}
//cluster.shutdown();
/// <exception cref="System.Exception"/>
public virtual void TestMismatchedNNIsRejected()
{
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, MiniDFSCluster.GetBaseDirectory() +
"/TestNNWithQJM/image");
string defaultEditsDir = conf.Get(DFSConfigKeys.DfsNamenodeEditsDirKey);
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal"
).ToString());
// Start a NN, so the storage is formatted -- both on-disk
// and QJM.
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs
(false).Build();
cluster.Shutdown();
// Reformat just the on-disk portion
Configuration onDiskOnly = new Configuration(conf);
onDiskOnly.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, defaultEditsDir);
NameNode.Format(onDiskOnly);
// Start the NN - should fail because the JNs are still formatted
// with the old namespace ID.
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false
).Format(false).Build();
NUnit.Framework.Assert.Fail("New NN with different namespace should have been rejected"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Unable to start log segment 1: too few journals"
, ioe);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestLogAndRestart()
{
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, MiniDFSCluster.GetBaseDirectory() +
"/TestNNWithQJM/image");
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal"
).ToString());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs
(false).Build();
try
{
cluster.GetFileSystem().Mkdirs(TestPath);
// Restart the NN and make sure the edit was persisted
// and loaded again
cluster.RestartNameNode();
NUnit.Framework.Assert.IsTrue(cluster.GetFileSystem().Exists(TestPath));
cluster.GetFileSystem().Mkdirs(TestPath2);
// Restart the NN again and make sure both edits are persisted.
cluster.RestartNameNode();
NUnit.Framework.Assert.IsTrue(cluster.GetFileSystem().Exists(TestPath));
NUnit.Framework.Assert.IsTrue(cluster.GetFileSystem().Exists(TestPath2));
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Tests setting the rpc port to the same as the web port to test that
/// an exception
/// is thrown when trying to re-use the same port
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestThatMatchingRPCandHttpPortsThrowException()
{
NameNode nameNode = null;
try
{
Configuration conf = new HdfsConfiguration();
FilePath nameDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "name");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
Random rand = new Random();
int port = 30000 + rand.Next(30000);
// set both of these to the same port. It should fail.
FileSystem.SetDefaultUri(conf, "hdfs://localhost:" + port);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "127.0.0.1:" + port);
DFSTestUtil.FormatNameNode(conf);
nameNode = new NameNode(conf);
}
finally
{
if (nameNode != null)
{
nameNode.Stop();
}
}
}
public virtual void TestInvalidateOverReplicatedBlock()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
FileSystem fs = cluster.GetFileSystem();
Path p = new Path(MiniDFSCluster.GetBaseDirectory(), "/foo1");
FSDataOutputStream @out = fs.Create(p, (short)2);
@out.WriteBytes("HDFS-3119: " + p);
@out.Hsync();
fs.SetReplication(p, (short)1);
@out.Close();
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, p);
NUnit.Framework.Assert.AreEqual("Expected only one live replica for the block", 1
, bm.CountNodes(block.GetLocalBlock()).LiveReplicas());
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// HDFS-3013: NameNode format command doesn't pick up
/// dfs.namenode.name.dir.NameServiceId configuration.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestGenericKeysForNameNodeFormat()
{
Configuration conf = new HdfsConfiguration();
// Set ephemeral ports
conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, "127.0.0.1:0");
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "127.0.0.1:0");
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
// Set a nameservice-specific configuration for name dir
FilePath dir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "testGenericKeysForNameNodeFormat"
);
if (dir.Exists())
{
FileUtil.FullyDelete(dir);
}
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey + ".ns1", dir.GetAbsolutePath());
// Format and verify the right dir is formatted.
DFSTestUtil.FormatNameNode(conf);
GenericTestUtils.AssertExists(dir);
// Ensure that the same dir is picked up by the running NN
NameNode nameNode = new NameNode(conf);
nameNode.Stop();
}
public virtual void SetUpNameDirs()
{
config = new HdfsConfiguration();
hdfsDir = new FilePath(MiniDFSCluster.GetBaseDirectory()).GetCanonicalFile();
if (hdfsDir.Exists() && !FileUtil.FullyDelete(hdfsDir))
{
throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
}
hdfsDir.Mkdirs();
path1 = new FilePath(hdfsDir, "name1");
path2 = new FilePath(hdfsDir, "name2");
path3 = new FilePath(hdfsDir, "name3");
path1.Mkdir();
path2.Mkdir();
path3.Mkdir();
if (!path2.Exists() || !path3.Exists() || !path1.Exists())
{
throw new IOException("Couldn't create dfs.name dirs in " + hdfsDir.GetAbsolutePath
());
}
string dfs_name_dir = new string(path1.GetPath() + "," + path2.GetPath());
System.Console.Out.WriteLine("configuring hdfsdir is " + hdfsDir.GetAbsolutePath(
) + "; dfs_name_dir = " + dfs_name_dir + ";dfs_name_edits_dir(only)=" + path3.GetPath
());
config.Set(DFSConfigKeys.DfsNamenodeNameDirKey, dfs_name_dir);
config.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, dfs_name_dir + "," + path3.GetPath
());
config.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, new FilePath(hdfsDir, "secondary"
).GetPath());
FileSystem.SetDefaultUri(config, "hdfs://" + NameNodeHost + "0");
config.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, "0.0.0.0:0");
// set the restore feature on
config.SetBoolean(DFSConfigKeys.DfsNamenodeNameDirRestoreKey, true);
}
private void InitConfig()
{
conf.Set(DFSConfigKeys.DfsNameserviceId, "ns1");
conf.Set(DFSConfigKeys.DfsHaNamenodesKeyPrefix + ".ns1", "nn1");
conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey + ".ns1.nn1", MiniDFSCluster.GetBaseDirectory
() + "1");
conf.Unset(DFSConfigKeys.DfsNamenodeNameDirKey);
}
public virtual void CleanupCluster()
{
FilePath hdfsDir = new FilePath(MiniDFSCluster.GetBaseDirectory()).GetCanonicalFile
();
System.Console.Out.WriteLine("cleanupCluster deleting " + hdfsDir);
if (hdfsDir.Exists() && !FileUtil.FullyDelete(hdfsDir))
{
throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
}
}
public virtual void TestNNThroughput()
{
Configuration conf = new HdfsConfiguration();
FilePath nameDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "name");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
FileSystem.SetDefaultUri(conf, "hdfs://localhost:" + 0);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "0.0.0.0:0");
DFSTestUtil.FormatNameNode(conf);
string[] args = new string[] { "-op", "all" };
NNThroughputBenchmark.RunBenchmark(conf, Arrays.AsList(args));
}
/// <exception cref="System.IO.IOException"/>
private Configuration GetConf()
{
string baseDir = MiniDFSCluster.GetBaseDirectory();
string nameDirs = Org.Apache.Hadoop.Hdfs.Server.Common.Util.FileAsURI(new FilePath
(baseDir, "name1")) + "," + Org.Apache.Hadoop.Hdfs.Server.Common.Util.FileAsURI(
new FilePath(baseDir, "name2"));
Configuration conf = new HdfsConfiguration();
FileSystem.SetDefaultUri(conf, "hdfs://localhost:0");
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "0.0.0.0:0");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDirs);
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameDirs);
conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, "0.0.0.0:0");
conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
return(conf);
}
public virtual void Setup()
{
FilePath editsDir = new FilePath(MiniDFSCluster.GetBaseDirectory() + FilePath.separator
+ "TestJournalNode");
FileUtil.FullyDelete(editsDir);
conf.Set(DFSConfigKeys.DfsJournalnodeEditsDirKey, editsDir.GetAbsolutePath());
conf.Set(DFSConfigKeys.DfsJournalnodeRpcAddressKey, "0.0.0.0:0");
jn = new JournalNode();
jn.SetConf(conf);
jn.Start();
journalId = "test-journalid-" + GenericTestUtils.UniqueSequenceId();
journal = jn.GetOrCreateJournal(journalId);
journal.Format(FakeNsinfo);
ch = new IPCLoggerChannel(conf, FakeNsinfo, journalId, jn.GetBoundIpcAddress());
}
public virtual void TestFSNamespaceClearLeases()
{
Configuration conf = new HdfsConfiguration();
FilePath nameDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "name");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
LeaseManager leaseMan = fsn.GetLeaseManager();
leaseMan.AddLease("client1", "importantFile");
NUnit.Framework.Assert.AreEqual(1, leaseMan.CountLease());
fsn.Clear();
leaseMan = fsn.GetLeaseManager();
NUnit.Framework.Assert.AreEqual(0, leaseMan.CountLease());
}
public virtual void TestValidVolumesAtStartup()
{
Assume.AssumeTrue(!Runtime.GetProperty("os.name").StartsWith("Windows"));
// Make sure no DNs are running.
cluster.ShutdownDataNodes();
// Bring up a datanode with two default data dirs, but with one bad one.
conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, 1);
// We use subdirectories 0 and 1 in order to have only a single
// data dir's parent inject a failure.
FilePath tld = new FilePath(MiniDFSCluster.GetBaseDirectory(), "badData");
FilePath dataDir1 = new FilePath(tld, "data1");
FilePath dataDir1Actual = new FilePath(dataDir1, "1");
dataDir1Actual.Mkdirs();
// Force an IOE to occur on one of the dfs.data.dir.
FilePath dataDir2 = new FilePath(tld, "data2");
PrepareDirToFail(dataDir2);
FilePath dataDir2Actual = new FilePath(dataDir2, "2");
// Start one DN, with manually managed DN dir
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dataDir1Actual.GetPath() + "," + dataDir2Actual
.GetPath());
cluster.StartDataNodes(conf, 1, false, null, null);
cluster.WaitActive();
try
{
NUnit.Framework.Assert.IsTrue("The DN should have started up fine.", cluster.IsDataNodeUp
());
DataNode dn = cluster.GetDataNodes()[0];
string si = DataNodeTestUtils.GetFSDataset(dn).GetStorageInfo();
NUnit.Framework.Assert.IsTrue("The DN should have started with this directory", si
.Contains(dataDir1Actual.GetPath()));
NUnit.Framework.Assert.IsFalse("The DN shouldn't have a bad directory.", si.Contains
(dataDir2Actual.GetPath()));
}
finally
{
cluster.ShutdownDataNodes();
FileUtil.Chmod(dataDir2.ToString(), "755");
}
}
public virtual void TestStartStop()
{
Configuration conf = new Configuration();
MiniJournalCluster c = new MiniJournalCluster.Builder(conf).Build();
try
{
URI uri = c.GetQuorumJournalURI("myjournal");
string[] addrs = uri.GetAuthority().Split(";");
NUnit.Framework.Assert.AreEqual(3, addrs.Length);
JournalNode node = c.GetJournalNode(0);
string dir = node.GetConf().Get(DFSConfigKeys.DfsJournalnodeEditsDirKey);
NUnit.Framework.Assert.AreEqual(new FilePath(MiniDFSCluster.GetBaseDirectory() +
"journalnode-0").GetAbsolutePath(), dir);
}
finally
{
c.Shutdown();
}
}
public virtual void SetUp()
{
config = new HdfsConfiguration();
hdfsDir = new FilePath(MiniDFSCluster.GetBaseDirectory());
if (hdfsDir.Exists() && !FileUtil.FullyDelete(hdfsDir))
{
throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
}
Log.Info("--hdfsdir is " + hdfsDir.GetAbsolutePath());
config.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Util.FileAsURI(new FilePath(hdfsDir
, "name")).ToString());
config.Set(DFSConfigKeys.DfsDatanodeDataDirKey, new FilePath(hdfsDir, "data").GetPath
());
config.Set(DFSConfigKeys.DfsDatanodeAddressKey, "0.0.0.0:0");
config.Set(DFSConfigKeys.DfsDatanodeHttpAddressKey, "0.0.0.0:0");
config.Set(DFSConfigKeys.DfsDatanodeIpcAddressKey, "0.0.0.0:0");
config.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, Util.FileAsURI(new FilePath
(hdfsDir, "secondary")).ToString());
config.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, WildcardHttpHost + "0"
);
FileSystem.SetDefaultUri(config, "hdfs://" + NameNodeHost + "0");
}
/// <summary>
/// Tests setting the rpc port to a different as the web port that an
/// exception is NOT thrown
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestThatDifferentRPCandHttpPortsAreOK()
{
Configuration conf = new HdfsConfiguration();
FilePath nameDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "name");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
Random rand = new Random();
// A few retries in case the ports we choose are in use.
for (int i = 0; i < 5; ++i)
{
int port1 = 30000 + rand.Next(10000);
int port2 = port1 + 1 + rand.Next(10000);
FileSystem.SetDefaultUri(conf, "hdfs://localhost:" + port1);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "127.0.0.1:" + port2);
DFSTestUtil.FormatNameNode(conf);
NameNode nameNode = null;
try
{
nameNode = new NameNode(conf);
// should be OK!
break;
}
catch (BindException)
{
continue;
}
finally
{
// Port in use? Try another.
if (nameNode != null)
{
nameNode.Stop();
}
}
}
}
/// <summary>
/// Test cancellation of ongoing checkpoints when failover happens
/// mid-checkpoint.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCheckpointCancellation()
{
cluster.TransitionToStandby(0);
// Create an edit log in the shared edits dir with a lot
// of mkdirs operations. This is solely so that the image is
// large enough to take a non-trivial amount of time to load.
// (only ~15MB)
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FilePath tmpDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "testCheckpointCancellation-tmp"
);
FSNamesystem fsn = cluster.GetNamesystem(0);
FSImageTestUtil.CreateAbortedLogWithMkdirs(tmpDir, NumDirsInLog, 3, fsn.GetFSDirectory
().GetLastInodeId() + 1);
string fname = NNStorage.GetInProgressEditsFileName(3);
new FilePath(tmpDir, fname).RenameTo(new FilePath(sharedDir, fname));
// Checkpoint as fast as we can, in a tight loop.
cluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey,
0);
cluster.RestartNameNode(1);
nn1 = cluster.GetNameNode(1);
cluster.TransitionToActive(0);
bool canceledOne = false;
for (int i = 0; i < 10 && !canceledOne; i++)
{
DoEdits(i * 10, i * 10 + 10);
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
cluster.TransitionToStandby(1);
cluster.TransitionToActive(0);
canceledOne = StandbyCheckpointer.GetCanceledCount() > 0;
}
NUnit.Framework.Assert.IsTrue(canceledOne);
}
public virtual void CleanUp()
{
FileUtil.FullyDeleteContents(new FilePath(MiniDFSCluster.GetBaseDirectory()));
}
public virtual void SetUp()
{
conf = new Configuration();
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Util.FileAsURI(new FilePath(MiniDFSCluster
.GetBaseDirectory(), "namenode")).ToString());
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
fs = null;
fsIsReady = true;
}
/// <exception cref="System.Exception"/>
public virtual void TestRollingUpgradeWithQJM()
{
string nnDirPrefix = MiniDFSCluster.GetBaseDirectory() + "/nn/";
FilePath nn1Dir = new FilePath(nnDirPrefix + "image1");
FilePath nn2Dir = new FilePath(nnDirPrefix + "image2");
Log.Info("nn1Dir=" + nn1Dir);
Log.Info("nn2Dir=" + nn2Dir);
Configuration conf = new HdfsConfiguration();
MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).Build();
SetConf(conf, nn1Dir, mjc);
{
// Start the cluster once to generate the dfs dirs
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs
(false).CheckExitOnShutdown(false).Build();
// Shutdown the cluster before making a copy of the namenode dir to release
// all file locks, otherwise, the copy will fail on some platforms.
cluster.Shutdown();
}
MiniDFSCluster cluster2 = null;
try
{
// Start a second NN pointed to the same quorum.
// We need to copy the image dir from the first NN -- or else
// the new NN will just be rejected because of Namespace mismatch.
FileUtil.FullyDelete(nn2Dir);
FileUtil.Copy(nn1Dir, FileSystem.GetLocal(conf).GetRaw(), new Path(nn2Dir.GetAbsolutePath
()), false, conf);
// Start the cluster again
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(
false).ManageNameDfsDirs(false).CheckExitOnShutdown(false).Build();
Path foo = new Path("/foo");
Path bar = new Path("/bar");
Path baz = new Path("/baz");
RollingUpgradeInfo info1;
{
DistributedFileSystem dfs = cluster.GetFileSystem();
dfs.Mkdirs(foo);
//start rolling upgrade
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
info1 = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare);
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
Log.Info("START\n" + info1);
//query rolling upgrade
NUnit.Framework.Assert.AreEqual(info1, dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction
.Query));
dfs.Mkdirs(bar);
cluster.Shutdown();
}
// cluster2 takes over QJM
Configuration conf2 = SetConf(new Configuration(), nn2Dir, mjc);
cluster2 = new MiniDFSCluster.Builder(conf2).NumDataNodes(0).Format(false).ManageNameDfsDirs
(false).Build();
DistributedFileSystem dfs2 = cluster2.GetFileSystem();
// Check that cluster2 sees the edits made on cluster1
NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar));
NUnit.Framework.Assert.IsFalse(dfs2.Exists(baz));
//query rolling upgrade in cluster2
NUnit.Framework.Assert.AreEqual(info1, dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction
.Query));
dfs2.Mkdirs(baz);
Log.Info("RESTART cluster 2");
cluster2.RestartNameNode();
NUnit.Framework.Assert.AreEqual(info1, dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction
.Query));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(baz));
//restart cluster with -upgrade should fail.
try
{
cluster2.RestartNameNode("-upgrade");
}
catch (IOException e)
{
Log.Info("The exception is expected.", e);
}
Log.Info("RESTART cluster 2 again");
cluster2.RestartNameNode();
NUnit.Framework.Assert.AreEqual(info1, dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction
.Query));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(baz));
//finalize rolling upgrade
RollingUpgradeInfo finalize = dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction
.Finalize);
NUnit.Framework.Assert.IsTrue(finalize.IsFinalized());
Log.Info("RESTART cluster 2 with regular startup option");
cluster2.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Regular
);
cluster2.RestartNameNode();
NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar));
NUnit.Framework.Assert.IsTrue(dfs2.Exists(baz));
}
finally
{
if (cluster2 != null)
{
cluster2.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNewNamenodeTakesOverWriter()
{
FilePath nn1Dir = new FilePath(MiniDFSCluster.GetBaseDirectory() + "/TestNNWithQJM/image-nn1"
);
FilePath nn2Dir = new FilePath(MiniDFSCluster.GetBaseDirectory() + "/TestNNWithQJM/image-nn2"
);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nn1Dir.GetAbsolutePath());
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal"
).ToString());
// Start the cluster once to generate the dfs dirs
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs
(false).CheckExitOnShutdown(false).Build();
// Shutdown the cluster before making a copy of the namenode dir
// to release all file locks, otherwise, the copy will fail on
// some platforms.
cluster.Shutdown();
try
{
// Start a second NN pointed to the same quorum.
// We need to copy the image dir from the first NN -- or else
// the new NN will just be rejected because of Namespace mismatch.
FileUtil.FullyDelete(nn2Dir);
FileUtil.Copy(nn1Dir, FileSystem.GetLocal(conf).GetRaw(), new Path(nn2Dir.GetAbsolutePath
()), false, conf);
// Start the cluster again
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(false).ManageNameDfsDirs
(false).CheckExitOnShutdown(false).Build();
cluster.GetFileSystem().Mkdirs(TestPath);
Configuration conf2 = new Configuration();
conf2.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nn2Dir.GetAbsolutePath());
conf2.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal"
).ToString());
MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf2).NumDataNodes(0).Format
(false).ManageNameDfsDirs(false).Build();
// Check that the new cluster sees the edits made on the old cluster
try
{
NUnit.Framework.Assert.IsTrue(cluster2.GetFileSystem().Exists(TestPath));
}
finally
{
cluster2.Shutdown();
}
// Check that, if we try to write to the old NN
// that it aborts.
try
{
cluster.GetFileSystem().Mkdirs(new Path("/x"));
NUnit.Framework.Assert.Fail("Did not abort trying to write to a fenced NN");
}
catch (RemoteException re)
{
GenericTestUtils.AssertExceptionContains("Could not sync enough journals to persistent storage"
, re);
}
}
finally
{
}
}