public virtual void TestPurgeLogs()
{
for (int txid = 1; txid <= 5; txid++)
{
QJMTestUtil.WriteSegment(cluster, qjm, txid, 1, true);
}
FilePath curDir = cluster.GetCurrentDir(0, QJMTestUtil.Jid);
GenericTestUtils.AssertGlobEquals(curDir, "edits_.*", NNStorage.GetFinalizedEditsFileName
(1, 1), NNStorage.GetFinalizedEditsFileName(2, 2), NNStorage.GetFinalizedEditsFileName
(3, 3), NNStorage.GetFinalizedEditsFileName(4, 4), NNStorage.GetFinalizedEditsFileName
(5, 5));
FilePath paxosDir = new FilePath(curDir, "paxos");
GenericTestUtils.AssertExists(paxosDir);
// Create new files in the paxos directory, which should get purged too.
NUnit.Framework.Assert.IsTrue(new FilePath(paxosDir, "1").CreateNewFile());
NUnit.Framework.Assert.IsTrue(new FilePath(paxosDir, "3").CreateNewFile());
GenericTestUtils.AssertGlobEquals(paxosDir, "\\d+", "1", "3");
// Create some temporary files of the sort that are used during recovery.
NUnit.Framework.Assert.IsTrue(new FilePath(curDir, "edits_inprogress_0000000000000000001.epoch=140"
).CreateNewFile());
NUnit.Framework.Assert.IsTrue(new FilePath(curDir, "edits_inprogress_0000000000000000002.empty"
).CreateNewFile());
qjm.PurgeLogsOlderThan(3);
// Log purging is asynchronous, so we have to wait for the calls
// to be sent and respond before verifying.
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
// Older edits should be purged
GenericTestUtils.AssertGlobEquals(curDir, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 3), NNStorage.GetFinalizedEditsFileName(4, 4), NNStorage.GetFinalizedEditsFileName
(5, 5));
// Older paxos files should be purged
GenericTestUtils.AssertGlobEquals(paxosDir, "\\d+", "3");
}
public virtual void TestLoadLogsFromBuggyEarlierVersions()
{
Configuration conf = new HdfsConfiguration();
string tarFile = Runtime.GetProperty("test.cache.data", "build/test/cache") + "/"
+ Hadoop23BrokenAppendTgz;
string testDir = PathUtils.GetTestDirName(GetType());
FilePath dfsDir = new FilePath(testDir, "image-with-buggy-append");
if (dfsDir.Exists() && !FileUtil.FullyDelete(dfsDir))
{
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
}
FileUtil.UnTar(new FilePath(tarFile), new FilePath(testDir));
FilePath nameDir = new FilePath(dfsDir, "name");
GenericTestUtils.AssertExists(nameDir);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(
false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).NumDataNodes(0).WaitSafeMode
(false).StartupOption(HdfsServerConstants.StartupOption.Upgrade).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/tmp/io_data/test_io_0");
NUnit.Framework.Assert.AreEqual(2 * 1024 * 1024, fs.GetFileStatus(testPath).GetLen
());
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestJournalLocking()
{
Assume.AssumeTrue(journal.GetStorage().GetStorageDir(0).IsLockSupported());
Storage.StorageDirectory sd = journal.GetStorage().GetStorageDir(0);
FilePath lockFile = new FilePath(sd.GetRoot(), Storage.StorageFileLock);
// Journal should be locked, since the format() call locks it.
GenericTestUtils.AssertExists(lockFile);
journal.NewEpoch(FakeNsinfo, 1);
try
{
new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular, mockErrorReporter
);
NUnit.Framework.Assert.Fail("Did not fail to create another journal in same dir");
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Cannot lock storage", ioe);
}
journal.Close();
// Journal should no longer be locked after the close() call.
// Hence, should be able to create a new Journal in the same dir.
Journal journal2 = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption
.Regular, mockErrorReporter);
journal2.NewEpoch(FakeNsinfo, 2);
journal2.Close();
}
public virtual void TestSharedEditsMissingLogs()
{
RemoveStandbyNameDirs();
CheckpointSignature sig = nn0.GetRpcServer().RollEditLog();
NUnit.Framework.Assert.AreEqual(3, sig.GetCurSegmentTxId());
// Should have created edits_1-2 in shared edits dir
URI editsUri = cluster.GetSharedEditsDir(0, 1);
FilePath editsDir = new FilePath(editsUri);
FilePath editsSegment = new FilePath(new FilePath(editsDir, "current"), NNStorage
.GetFinalizedEditsFileName(1, 2));
GenericTestUtils.AssertExists(editsSegment);
// Delete the segment.
NUnit.Framework.Assert.IsTrue(editsSegment.Delete());
// Trying to bootstrap standby should now fail since the edit
// logs aren't available in the shared dir.
GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(BootstrapStandby)));
try
{
int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(BootstrapStandby.ErrCodeLogsUnavailable, rc);
}
finally
{
logs.StopCapturing();
}
GenericTestUtils.AssertMatches(logs.GetOutput(), "FATAL.*Unable to read transaction ids 1-3 from the configured shared"
);
}
/// <summary>
/// HDFS-3013: NameNode format command doesn't pick up
/// dfs.namenode.name.dir.NameServiceId configuration.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestGenericKeysForNameNodeFormat()
{
Configuration conf = new HdfsConfiguration();
// Set ephemeral ports
conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, "127.0.0.1:0");
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "127.0.0.1:0");
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
// Set a nameservice-specific configuration for name dir
FilePath dir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "testGenericKeysForNameNodeFormat"
);
if (dir.Exists())
{
FileUtil.FullyDelete(dir);
}
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey + ".ns1", dir.GetAbsolutePath());
// Format and verify the right dir is formatted.
DFSTestUtil.FormatNameNode(conf);
GenericTestUtils.AssertExists(dir);
// Ensure that the same dir is picked up by the running NN
NameNode nameNode = new NameNode(conf);
nameNode.Stop();
}
public virtual void TestEarlierVersionEditLog()
{
Configuration conf = new HdfsConfiguration();
string tarFile = Runtime.GetProperty("test.cache.data", "build/test/cache") + "/"
+ Hadoop10MultiblockTgz;
string testDir = PathUtils.GetTestDirName(GetType());
FilePath dfsDir = new FilePath(testDir, "image-1.0");
if (dfsDir.Exists() && !FileUtil.FullyDelete(dfsDir))
{
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
}
FileUtil.UnTar(new FilePath(tarFile), new FilePath(testDir));
FilePath nameDir = new FilePath(dfsDir, "name");
GenericTestUtils.AssertExists(nameDir);
FilePath dataDir = new FilePath(dfsDir, "data");
GenericTestUtils.AssertExists(dataDir);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dataDir.GetAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(
false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).NumDataNodes(1).StartupOption
(HdfsServerConstants.StartupOption.Upgrade).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/user/todd/4blocks");
// Read it without caring about the actual data within - we just need
// to make sure that the block states and locations are OK.
DFSTestUtil.ReadFile(fs, testPath);
// Ensure that we can append to it - if the blocks were in some funny
// state we'd get some kind of issue here.
FSDataOutputStream stm = fs.Append(testPath);
try
{
stm.Write(1);
}
finally
{
IOUtils.CloseStream(stm);
}
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Assume that a client is writing to a journal, but loses its connection
/// in the middle of a segment.
/// </summary>
/// <remarks>
/// Assume that a client is writing to a journal, but loses its connection
/// in the middle of a segment. Thus, any future journal() calls in that
/// segment may fail, because some txns were missed while the connection was
/// down.
/// Eventually, the connection comes back, and the NN tries to start a new
/// segment at a higher txid. This should abort the old one and succeed.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestAbortOldSegmentIfFinalizeIsMissed()
{
journal.NewEpoch(FakeNsinfo, 1);
// Start a segment at txid 1, and write a batch of 3 txns.
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 3, QJMTestUtil.CreateTxnData(1, 3));
GenericTestUtils.AssertExists(journal.GetStorage().GetInProgressEditLog(1));
// Try to start new segment at txid 6, this should abort old segment and
// then succeed, allowing us to write txid 6-9.
journal.StartLogSegment(MakeRI(3), 6, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(4), 6, 6, 3, QJMTestUtil.CreateTxnData(6, 3));
// The old segment should *not* be finalized.
GenericTestUtils.AssertExists(journal.GetStorage().GetInProgressEditLog(1));
GenericTestUtils.AssertExists(journal.GetStorage().GetInProgressEditLog(6));
}
/// <summary>
/// Check that the given list of edits files are present in the given storage
/// dirs.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void AssertEditFiles(IEnumerable <URI> dirs, params string[] files)
{
foreach (URI u in dirs)
{
FilePath editDirRoot = new FilePath(u.GetPath());
FilePath editDir = new FilePath(editDirRoot, "current");
GenericTestUtils.AssertExists(editDir);
if (files.Length == 0)
{
Log.Info("Checking no edit files exist in " + editDir);
}
else
{
Log.Info("Checking for following edit files in " + editDir + ": " + Joiner.On(","
).Join(files));
}
GenericTestUtils.AssertGlobEquals(editDir, "edits_.*", files);
}
}
/// <summary>For namenode, Verify that the current and previous directories exist.</summary>
/// <remarks>
/// For namenode, Verify that the current and previous directories exist.
/// Verify that previous hasn't been modified by comparing the checksum of all
/// its files with their original checksum. It is assumed that the
/// server has recovered and upgraded.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal virtual void CheckNameNode(string[] baseDirs, long imageTxId)
{
foreach (string baseDir in baseDirs)
{
Log.Info("Checking namenode directory " + baseDir);
Log.Info("==== Contents ====:\n " + Joiner.On(" \n").Join(new FilePath(baseDir,
"current").List()));
Log.Info("==================");
GenericTestUtils.AssertExists(new FilePath(baseDir, "current"));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/VERSION"));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetInProgressEditsFileName
(imageTxId + 1)));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetImageFileName
(imageTxId)));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/seen_txid"));
FilePath previous = new FilePath(baseDir, "previous");
GenericTestUtils.AssertExists(previous);
NUnit.Framework.Assert.AreEqual(UpgradeUtilities.ChecksumContents(HdfsServerConstants.NodeType
.NameNode, previous, false), UpgradeUtilities.ChecksumMasterNameNodeContents());
}
}
public virtual void TestZeroBlockSize()
{
Configuration conf = new HdfsConfiguration();
string tarFile = Runtime.GetProperty("test.cache.data", "build/test/cache") + "/"
+ Hadoop27Zer0BlockSizeTgz;
string testDir = PathUtils.GetTestDirName(GetType());
FilePath dfsDir = new FilePath(testDir, "image-with-zero-block-size");
if (dfsDir.Exists() && !FileUtil.FullyDelete(dfsDir))
{
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
}
FileUtil.UnTar(new FilePath(tarFile), new FilePath(testDir));
FilePath nameDir = new FilePath(dfsDir, "name");
GenericTestUtils.AssertExists(nameDir);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).WaitSafeMode(false).StartupOption
(HdfsServerConstants.StartupOption.Upgrade).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/tmp/zeroBlockFile");
NUnit.Framework.Assert.IsTrue("File /tmp/zeroBlockFile doesn't exist ", fs.Exists
(testPath));
NUnit.Framework.Assert.IsTrue("Name node didn't come up", cluster.IsNameNodeUp(0)
);
}
finally
{
cluster.Shutdown();
//Clean up
FileUtil.FullyDelete(dfsDir);
}
}
