public virtual void TestStartup()
{
Configuration conf = new Configuration();
HAUtil.SetAllowStandbyReads(conf, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(0).Build();
try
{
// During HA startup, both nodes should be in
// standby and we shouldn't have any edits files
// in any edits directory!
IList <URI> allDirs = Lists.NewArrayList();
Sharpen.Collections.AddAll(allDirs, cluster.GetNameDirs(0));
Sharpen.Collections.AddAll(allDirs, cluster.GetNameDirs(1));
allDirs.AddItem(cluster.GetSharedEditsDir(0, 1));
AssertNoEditFiles(allDirs);
// Set the first NN to active, make sure it creates edits
// in its own dirs and the shared dir. The standby
// should still have no edits!
cluster.TransitionToActive(0);
AssertEditFiles(cluster.GetNameDirs(0), NNStorage.GetInProgressEditsFileName(1));
AssertEditFiles(Sharpen.Collections.SingletonList(cluster.GetSharedEditsDir(0, 1)
), NNStorage.GetInProgressEditsFileName(1));
AssertNoEditFiles(cluster.GetNameDirs(1));
cluster.GetNameNode(0).GetRpcServer().Mkdirs("/test", FsPermission.CreateImmutable
((short)0x1ed), true);
// Restarting the standby should not finalize any edits files
// in the shared directory when it starts up!
cluster.RestartNameNode(1);
AssertEditFiles(cluster.GetNameDirs(0), NNStorage.GetInProgressEditsFileName(1));
AssertEditFiles(Sharpen.Collections.SingletonList(cluster.GetSharedEditsDir(0, 1)
), NNStorage.GetInProgressEditsFileName(1));
AssertNoEditFiles(cluster.GetNameDirs(1));
// Additionally it should not have applied any in-progress logs
// at start-up -- otherwise, it would have read half-way into
// the current log segment, and on the next roll, it would have to
// either replay starting in the middle of the segment (not allowed)
// or double-replay the edits (incorrect).
NUnit.Framework.Assert.IsNull(NameNodeAdapter.GetFileInfo(cluster.GetNameNode(1),
"/test", true));
cluster.GetNameNode(0).GetRpcServer().Mkdirs("/test2", FsPermission.CreateImmutable
((short)0x1ed), true);
// If we restart NN0, it'll come back as standby, and we can
// transition NN1 to active and make sure it reads edits correctly at this point.
cluster.RestartNameNode(0);
cluster.TransitionToActive(1);
// NN1 should have both the edits that came before its restart, and the edits that
// came after its restart.
NUnit.Framework.Assert.IsNotNull(NameNodeAdapter.GetFileInfo(cluster.GetNameNode(
1), "/test", true));
NUnit.Framework.Assert.IsNotNull(NameNodeAdapter.GetFileInfo(cluster.GetNameNode(
1), "/test2", true));
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
private static void WaitForLogRollInSharedDir(MiniDFSCluster cluster, long startTxId
)
{
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FilePath expectedLog = new FilePath(sharedDir, NNStorage.GetInProgressEditsFileName
(startTxId));
GenericTestUtils.WaitFor(new _Supplier_153(expectedLog), 100, 10000);
}
public virtual void TestChangeWritersLogsInSync()
{
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, false);
QJMTestUtil.AssertExistsInQuorum(cluster, NNStorage.GetInProgressEditsFileName(1)
);
// Make a new QJM
qjm = CloseLater(new QuorumJournalManager(conf, cluster.GetQuorumJournalURI(QJMTestUtil
.Jid), QJMTestUtil.FakeNsinfo));
qjm.RecoverUnfinalizedSegments();
CheckRecovery(cluster, 1, 3);
}
public virtual void TestOldInProgress()
{
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(101), true);
RunTest(tc);
}
/// <exception cref="System.Exception"/>
private void DoOutOfSyncTest(int missingOnRecoveryIdx, long expectedRecoveryTxnId
)
{
SetupLoggers345();
QJMTestUtil.AssertExistsInQuorum(cluster, NNStorage.GetInProgressEditsFileName(1)
);
// Shut down the specified JN, so it's not present during recovery.
cluster.GetJournalNode(missingOnRecoveryIdx).StopAndJoin(0);
// Make a new QJM
qjm = CreateSpyingQJM();
qjm.RecoverUnfinalizedSegments();
CheckRecovery(cluster, 1, expectedRecoveryTxnId);
}
/// <exception cref="System.Exception"/>
private void TestFailoverFinalizesAndReadsInProgress(bool partialTxAtEnd)
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(0).Build();
try
{
// Create a fake in-progress edit-log in the shared directory
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FSNamesystem fsn = cluster.GetNamesystem(0);
FSImageTestUtil.CreateAbortedLogWithMkdirs(sharedDir, NumDirsInLog, 1, fsn.GetFSDirectory
().GetLastInodeId() + 1);
AssertEditFiles(Sharpen.Collections.SingletonList(sharedUri), NNStorage.GetInProgressEditsFileName
(1));
if (partialTxAtEnd)
{
FileOutputStream outs = null;
try
{
FilePath editLogFile = new FilePath(sharedDir, NNStorage.GetInProgressEditsFileName
(1));
outs = new FileOutputStream(editLogFile, true);
outs.Write(new byte[] { unchecked ((int)(0x18)), unchecked ((int)(0x00)), unchecked (
(int)(0x00)), unchecked ((int)(0x00)) });
Log.Error("editLogFile = " + editLogFile);
}
finally
{
IOUtils.Cleanup(Log, outs);
}
}
// Transition one of the NNs to active
cluster.TransitionToActive(0);
// In the transition to active, it should have read the log -- and
// hence see one of the dirs we made in the fake log.
string testPath = "/dir" + NumDirsInLog;
NUnit.Framework.Assert.IsNotNull(cluster.GetNameNode(0).GetRpcServer().GetFileInfo
(testPath));
// It also should have finalized that log in the shared directory and started
// writing to a new one at the next txid.
AssertEditFiles(Sharpen.Collections.SingletonList(sharedUri), NNStorage.GetFinalizedEditsFileName
(1, NumDirsInLog + 1), NNStorage.GetInProgressEditsFileName(NumDirsInLog + 2));
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Create an unfinalized edit log for testing purposes</summary>
/// <param name="testDir">Directory to create the edit log in</param>
/// <param name="numTx">Number of transactions to add to the new edit log</param>
/// <param name="offsetToTxId">
/// A map from transaction IDs to offsets in the
/// edit log file.
/// </param>
/// <returns>The new edit log file name.</returns>
/// <exception cref="System.IO.IOException"/>
private static FilePath PrepareUnfinalizedTestEditLog(FilePath testDir, int numTx
, SortedDictionary <long, long> offsetToTxId)
{
FilePath inProgressFile = new FilePath(testDir, NNStorage.GetInProgressEditsFileName
(1));
FSEditLog fsel = null;
FSEditLog spyLog = null;
try
{
fsel = FSImageTestUtil.CreateStandaloneEditLog(testDir);
spyLog = Org.Mockito.Mockito.Spy(fsel);
// Normally, the in-progress edit log would be finalized by
// FSEditLog#endCurrentLogSegment. For testing purposes, we
// disable that here.
Org.Mockito.Mockito.DoNothing().When(spyLog).EndCurrentLogSegment(true);
spyLog.OpenForWrite();
NUnit.Framework.Assert.IsTrue("should exist: " + inProgressFile, inProgressFile.Exists
());
for (int i = 0; i < numTx; i++)
{
long trueOffset = GetNonTrailerLength(inProgressFile);
long thisTxId = spyLog.GetLastWrittenTxId() + 1;
offsetToTxId[trueOffset] = thisTxId;
System.Console.Error.WriteLine("txid " + thisTxId + " at offset " + trueOffset);
spyLog.LogDelete("path" + i, i, false);
spyLog.LogSync();
}
}
finally
{
if (spyLog != null)
{
spyLog.Close();
}
else
{
if (fsel != null)
{
fsel.Close();
}
}
}
return(inProgressFile);
}
public virtual void TestPurgeEasyCase()
{
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), true);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
);
tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false);
// Test that other files don't get purged
tc.AddLog("/foo1/current/VERSION", false);
RunTest(tc);
}
public virtual void TestRetainExtraLogs()
{
conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 50);
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
// Since we need 50 extra edits, *do* retain the 201-300 segment
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
RunTest(tc);
}
/// <exception cref="System.IO.IOException"/>
public static EditLogOutputStream WriteSegment(MiniJournalCluster cluster, QuorumJournalManager
qjm, long startTxId, int numTxns, bool finalize)
{
EditLogOutputStream stm = qjm.StartLogSegment(startTxId, NameNodeLayoutVersion.CurrentLayoutVersion
);
// Should create in-progress
AssertExistsInQuorum(cluster, NNStorage.GetInProgressEditsFileName(startTxId));
WriteTxns(stm, startTxId, numTxns);
if (finalize)
{
stm.Close();
qjm.FinalizeLogSegment(startTxId, startTxId + numTxns - 1);
return(null);
}
else
{
return(stm);
}
}
/// <summary>For namenode, Verify that the current and previous directories exist.</summary>
/// <remarks>
/// For namenode, Verify that the current and previous directories exist.
/// Verify that previous hasn't been modified by comparing the checksum of all
/// its files with their original checksum. It is assumed that the
/// server has recovered and upgraded.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal virtual void CheckNameNode(string[] baseDirs, long imageTxId)
{
foreach (string baseDir in baseDirs)
{
Log.Info("Checking namenode directory " + baseDir);
Log.Info("==== Contents ====:\n " + Joiner.On(" \n").Join(new FilePath(baseDir,
"current").List()));
Log.Info("==================");
GenericTestUtils.AssertExists(new FilePath(baseDir, "current"));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/VERSION"));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetInProgressEditsFileName
(imageTxId + 1)));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetImageFileName
(imageTxId)));
GenericTestUtils.AssertExists(new FilePath(baseDir, "current/seen_txid"));
FilePath previous = new FilePath(baseDir, "previous");
GenericTestUtils.AssertExists(previous);
NUnit.Framework.Assert.AreEqual(UpgradeUtilities.ChecksumContents(HdfsServerConstants.NodeType
.NameNode, previous, false), UpgradeUtilities.ChecksumMasterNameNodeContents());
}
}
public virtual void TestRetainExtraLogsLimitedSegments()
{
conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 150);
conf.SetLong(DFSConfigKeys.DfsNamenodeMaxExtraEditsSegmentsRetainedKey, 2);
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
// Segments containing txns upto txId 250 are extra and should be purged.
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(1, 100), true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 175), true);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(176) + ".empty"
, true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(176, 200), true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 225), true);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(226) + ".corrupt"
, true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(226, 240), true);
// Only retain 2 extra segments. The 301-350 and 351-400 segments are
// considered required, not extra.
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(241, 275), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(276, 300), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(301) + ".empty"
, false);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 350), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(351) + ".corrupt"
, false);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(351, 400), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
RunTest(tc);
}
/// <summary>
/// Test cancellation of ongoing checkpoints when failover happens
/// mid-checkpoint.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCheckpointCancellation()
{
cluster.TransitionToStandby(0);
// Create an edit log in the shared edits dir with a lot
// of mkdirs operations. This is solely so that the image is
// large enough to take a non-trivial amount of time to load.
// (only ~15MB)
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FilePath tmpDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "testCheckpointCancellation-tmp"
);
FSNamesystem fsn = cluster.GetNamesystem(0);
FSImageTestUtil.CreateAbortedLogWithMkdirs(tmpDir, NumDirsInLog, 3, fsn.GetFSDirectory
().GetLastInodeId() + 1);
string fname = NNStorage.GetInProgressEditsFileName(3);
new FilePath(tmpDir, fname).RenameTo(new FilePath(sharedDir, fname));
// Checkpoint as fast as we can, in a tight loop.
cluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey,
0);
cluster.RestartNameNode(1);
nn1 = cluster.GetNameNode(1);
cluster.TransitionToActive(0);
bool canceledOne = false;
for (int i = 0; i < 10 && !canceledOne; i++)
{
DoEdits(i * 10, i * 10 + 10);
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
cluster.TransitionToStandby(1);
cluster.TransitionToActive(0);
canceledOne = StandbyCheckpointer.GetCanceledCount() > 0;
}
NUnit.Framework.Assert.IsTrue(canceledOne);
}
private void CheckNNStorage(NNStorage storage, long imageTxId, long trashEndTxId)
{
IList <FilePath> finalizedEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits
, NNStorage.GetFinalizedEditsFileName(1, imageTxId));
NUnit.Framework.Assert.IsTrue(FileExists(finalizedEdits));
IList <FilePath> inprogressEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits
, NNStorage.GetInProgressEditsFileName(imageTxId + 1));
// For rollback case we will have an inprogress file for future transactions
NUnit.Framework.Assert.IsTrue(FileExists(inprogressEdits));
if (trashEndTxId > 0)
{
IList <FilePath> trashedEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits,
NNStorage.GetFinalizedEditsFileName(imageTxId + 1, trashEndTxId) + ".trash");
NUnit.Framework.Assert.IsTrue(FileExists(trashedEdits));
}
string imageFileName = trashEndTxId > 0 ? NNStorage.GetImageFileName(imageTxId) :
NNStorage.GetRollbackImageFileName(imageTxId);
IList <FilePath> imageFiles = storage.GetFiles(NNStorage.NameNodeDirType.Image, imageFileName
);
NUnit.Framework.Assert.IsTrue(FileExists(imageFiles));
}
public virtual void TestFailureOfSharedDir()
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsNamenodeResourceCheckIntervalKey, 2000);
// The shared edits dir will automatically be marked required.
MiniDFSCluster cluster = null;
FilePath sharedEditsDir = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).CheckExitOnShutdown(false).Build();
cluster.WaitActive();
cluster.TransitionToActive(0);
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/test1")));
// Blow away the shared edits dir.
URI sharedEditsUri = cluster.GetSharedEditsDir(0, 1);
sharedEditsDir = new FilePath(sharedEditsUri);
NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(sharedEditsDir.GetAbsolutePath(
), "-w", true));
Sharpen.Thread.Sleep(conf.GetLong(DFSConfigKeys.DfsNamenodeResourceCheckIntervalKey
, DFSConfigKeys.DfsNamenodeResourceCheckIntervalDefault) * 2);
NameNode nn1 = cluster.GetNameNode(1);
NUnit.Framework.Assert.IsTrue(nn1.IsStandbyState());
NUnit.Framework.Assert.IsFalse("StandBy NameNode should not go to SafeMode on resource unavailability"
, nn1.IsInSafeMode());
NameNode nn0 = cluster.GetNameNode(0);
try
{
// Make sure that subsequent operations on the NN fail.
nn0.GetRpcServer().RollEditLog();
NUnit.Framework.Assert.Fail("Succeeded in rolling edit log despite shared dir being deleted"
);
}
catch (ExitUtil.ExitException ee)
{
GenericTestUtils.AssertExceptionContains("finalize log segment 1, 3 failed for required journal"
, ee);
}
// Check that none of the edits dirs rolled, since the shared edits
// dir didn't roll. Regression test for HDFS-2874.
foreach (URI editsUri in cluster.GetNameEditsDirs(0))
{
if (editsUri.Equals(sharedEditsUri))
{
continue;
}
FilePath editsDir = new FilePath(editsUri.GetPath());
FilePath curDir = new FilePath(editsDir, "current");
GenericTestUtils.AssertGlobEquals(curDir, "edits_.*", NNStorage.GetInProgressEditsFileName
(1));
}
}
finally
{
if (sharedEditsDir != null)
{
// without this test cleanup will fail
FileUtil.Chmod(sharedEditsDir.GetAbsolutePath(), "+w", true);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Set up the following tricky edge case state which is used by
/// multiple tests:
/// Initial writer:
/// - Writing to 3 JNs: JN0, JN1, JN2:
/// - A log segment with txnid 1 through 100 succeeds.
/// </summary>
/// <remarks>
/// Set up the following tricky edge case state which is used by
/// multiple tests:
/// Initial writer:
/// - Writing to 3 JNs: JN0, JN1, JN2:
/// - A log segment with txnid 1 through 100 succeeds.
/// - The first transaction in the next segment only goes to JN0
/// before the writer crashes (eg it is partitioned)
/// Recovery by another writer:
/// - The new NN starts recovery and talks to all three. Thus, it sees
/// that the newest log segment which needs recovery is 101.
/// - It sends the prepareRecovery(101) call, and decides that the
/// recovery length for 101 is only the 1 transaction.
/// - It sends acceptRecovery(101-101) to only JN0, before crashing
/// This yields the following state:
/// - JN0: 1-100 finalized, 101_inprogress, accepted recovery: 101-101
/// - JN1: 1-100 finalized, 101_inprogress.empty
/// - JN2: 1-100 finalized, 101_inprogress.empty
/// (the .empty files got moved aside during recovery)
/// </remarks>
/// <exception cref="System.Exception"></exception>
private void SetupEdgeCaseOneJnHasSegmentWithAcceptedRecovery()
{
// Log segment with txns 1-100 succeeds
QJMTestUtil.WriteSegment(cluster, qjm, 1, 100, true);
// startLogSegment only makes it to one of the three nodes
FailLoggerAtTxn(spies[1], 101);
FailLoggerAtTxn(spies[2], 101);
try
{
QJMTestUtil.WriteSegment(cluster, qjm, 101, 1, true);
NUnit.Framework.Assert.Fail("Should have failed");
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("mock failure", qe);
}
finally
{
qjm.Close();
}
// Recovery 1:
// make acceptRecovery() only make it to the node which has txid 101
// this should fail because only 1/3 accepted the recovery
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spies
[1]).AcceptRecovery(Org.Mockito.Mockito.Any <QJournalProtocolProtos.SegmentStateProto
>(), Org.Mockito.Mockito.Any <Uri>());
TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spies
[2]).AcceptRecovery(Org.Mockito.Mockito.Any <QJournalProtocolProtos.SegmentStateProto
>(), Org.Mockito.Mockito.Any <Uri>());
try
{
qjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed to recover");
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("mock failure", qe);
}
finally
{
qjm.Close();
}
// Check that we have entered the expected state as described in the
// method javadoc.
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(0, QJMTestUtil.Jid), "edits_.*"
, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName
(101));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(1, QJMTestUtil.Jid), "edits_.*"
, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName
(101) + ".empty");
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(2, QJMTestUtil.Jid), "edits_.*"
, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName
(101) + ".empty");
FilePath paxos0 = new FilePath(cluster.GetCurrentDir(0, QJMTestUtil.Jid), "paxos"
);
FilePath paxos1 = new FilePath(cluster.GetCurrentDir(1, QJMTestUtil.Jid), "paxos"
);
FilePath paxos2 = new FilePath(cluster.GetCurrentDir(2, QJMTestUtil.Jid), "paxos"
);
GenericTestUtils.AssertGlobEquals(paxos0, ".*", "101");
GenericTestUtils.AssertGlobEquals(paxos1, ".*");
GenericTestUtils.AssertGlobEquals(paxos2, ".*");
}
/// <returns>
/// the path for an in-progress edits file starting at the given
/// transaction ID. This does not verify existence of the file.
/// </returns>
internal virtual FilePath GetInProgressEditLog(long startTxId)
{
return(new FilePath(sd.GetCurrentDir(), NNStorage.GetInProgressEditsFileName(startTxId
)));
}
/// <param name="segmentTxId">the first txid of the segment</param>
/// <param name="epoch">
/// the epoch number of the writer which is coordinating
/// recovery
/// </param>
/// <returns>
/// the temporary path in which an edits log should be stored
/// while it is being downloaded from a remote JournalNode
/// </returns>
internal virtual FilePath GetSyncLogTemporaryFile(long segmentTxId, long epoch)
{
string name = NNStorage.GetInProgressEditsFileName(segmentTxId) + ".epoch=" + epoch;
return(new FilePath(sd.GetCurrentDir(), name));
}
/// <summary>
/// Test the case where, at the beginning of a segment, transactions
/// have been written to one JN but not others.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void DoTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn)
{
int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3;
int nodeMissingSegment = (nodeWithOneTxn + 2) % 3;
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
cluster.GetJournalNode(nodeMissingSegment).StopAndJoin(0);
// Open segment on 2/3 nodes
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
// Write transactions to only 1/3 nodes
FailLoggerAtTxn(spies[nodeWithEmptySegment], 4);
try
{
QJMTestUtil.WriteTxns(stm, 4, 1);
NUnit.Framework.Assert.Fail("Did not fail even though 2/3 failed");
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("mock failure", qe);
}
}
finally
{
stm.Abort();
}
// Bring back the down JN.
cluster.RestartJournalNode(nodeMissingSegment);
// Make a new QJM. At this point, the state is as follows:
// A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty)
// B: nodeWithOneTxn: 1-3 finalized, 4_inprogress (1 txn)
// C: nodeMissingSegment: 1-3 finalized
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithEmptySegment, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName
(4));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithOneTxn, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName
(4));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeMissingSegment, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3));
// Stop one of the nodes. Since we run this test three
// times, rotating the roles of the nodes, we'll test
// all the permutations.
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
qjm.RecoverUnfinalizedSegments();
if (nodeWithOneTxn == 0 || nodeWithOneTxn == 1)
{
// If the node that had the transaction committed was one of the nodes
// that responded during recovery, then we should have recovered txid
// 4.
CheckRecovery(cluster, 4, 4);
QJMTestUtil.WriteSegment(cluster, qjm, 5, 3, true);
}
else
{
// Otherwise, we should have recovered only 1-3 and should be able to
// start a segment at 4.
CheckRecovery(cluster, 1, 3);
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true);
}
}
public virtual void TestSaveRightBeforeSync()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);
try
{
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = Org.Mockito.Mockito.Spy(fsimage.GetEditLog());
DFSTestUtil.SetEditLogForTesting(namesystem, editLog);
AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
CountDownLatch waitToEnterSync = new CountDownLatch(1);
Sharpen.Thread doAnEditThread = new _Thread_467(namesystem, deferredException, waitToEnterSync
);
Answer <Void> blockingSync = new _Answer_484(doAnEditThread, waitToEnterSync);
Org.Mockito.Mockito.DoAnswer(blockingSync).When(editLog).LogSync();
doAnEditThread.Start();
Log.Info("Main thread: waiting to just before logSync...");
waitToEnterSync.Await();
NUnit.Framework.Assert.IsNull(deferredException.Get());
Log.Info("Main thread: detected that logSync about to be called.");
Log.Info("Trying to enter safe mode.");
Log.Info("This should block for " + BlockTime + "sec, since we have pending edits"
);
long st = Time.Now();
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
long et = Time.Now();
Log.Info("Entered safe mode");
// Make sure we really waited for the flush to complete!
NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.SaveNamespace();
Log.Info("Joining on edit thread...");
doAnEditThread.Join();
NUnit.Framework.Assert.IsNull(deferredException.Get());
// We did 3 edits: begin, txn, and end
NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetFinalizedEditsFileName(1, 3), 1));
// after the save, just the one "begin"
NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetInProgressEditsFileName(4), 4));
}
finally
{
Log.Info("Closing nn");
if (namesystem != null)
{
namesystem.Close();
}
}
}
public virtual void TestSaveImageWhileSyncInProgress()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);
try
{
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = fsimage.GetEditLog();
JournalSet.JournalAndStream jas = editLog.GetJournals()[0];
EditLogFileOutputStream spyElos = Org.Mockito.Mockito.Spy((EditLogFileOutputStream
)jas.GetCurrentStream());
jas.SetCurrentStreamForTests(spyElos);
AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
CountDownLatch waitToEnterFlush = new CountDownLatch(1);
Sharpen.Thread doAnEditThread = new _Thread_371(namesystem, deferredException, waitToEnterFlush
);
Answer <Void> blockingFlush = new _Answer_388(doAnEditThread, waitToEnterFlush);
// Signal to main thread that the edit thread is in the racy section
Org.Mockito.Mockito.DoAnswer(blockingFlush).When(spyElos).Flush();
doAnEditThread.Start();
// Wait for the edit thread to get to the logsync unsynchronized section
Log.Info("Main thread: waiting to enter flush...");
waitToEnterFlush.Await();
NUnit.Framework.Assert.IsNull(deferredException.Get());
Log.Info("Main thread: detected that logSync is in unsynchronized section.");
Log.Info("Trying to enter safe mode.");
Log.Info("This should block for " + BlockTime + "sec, since flush will sleep that long"
);
long st = Time.Now();
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
long et = Time.Now();
Log.Info("Entered safe mode");
// Make sure we really waited for the flush to complete!
NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.SaveNamespace();
Log.Info("Joining on edit thread...");
doAnEditThread.Join();
NUnit.Framework.Assert.IsNull(deferredException.Get());
// We did 3 edits: begin, txn, and end
NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetFinalizedEditsFileName(1, 3), 1));
// after the save, just the one "begin"
NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetInProgressEditsFileName(4), 4));
}
finally
{
Log.Info("Closing nn");
if (namesystem != null)
{
namesystem.Close();
}
}
}
public virtual void TestSaveNamespace()
{
// start a cluster
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
AtomicReference <Exception> caughtErr = new AtomicReference <Exception>();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build();
cluster.WaitActive();
fileSys = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
NamenodeProtocols nn = cluster.GetNameNodeRpc();
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = fsimage.GetEditLog();
StartTransactionWorkers(nn, caughtErr);
for (int i = 0; i < NumSaveImage && caughtErr.Get() == null; i++)
{
try
{
Sharpen.Thread.Sleep(20);
}
catch (Exception)
{
}
Log.Info("Save " + i + ": entering safe mode");
namesystem.EnterSafeMode(false);
// Verify edit logs before the save
// They should start with the first edit after the checkpoint
long logStartTxId = fsimage.GetStorage().GetMostRecentCheckpointTxId() + 1;
VerifyEditLogs(namesystem, fsimage, NNStorage.GetInProgressEditsFileName(logStartTxId
), logStartTxId);
Log.Info("Save " + i + ": saving namespace");
namesystem.SaveNamespace();
Log.Info("Save " + i + ": leaving safemode");
long savedImageTxId = fsimage.GetStorage().GetMostRecentCheckpointTxId();
// Verify that edit logs post save got finalized and aren't corrupt
VerifyEditLogs(namesystem, fsimage, NNStorage.GetFinalizedEditsFileName(logStartTxId
, savedImageTxId), logStartTxId);
// The checkpoint id should be 1 less than the last written ID, since
// the log roll writes the "BEGIN" transaction to the new log.
NUnit.Framework.Assert.AreEqual(fsimage.GetStorage().GetMostRecentCheckpointTxId(
), editLog.GetLastWrittenTxId() - 1);
namesystem.LeaveSafeMode();
Log.Info("Save " + i + ": complete");
}
}
finally
{
StopTransactionWorkers();
if (caughtErr.Get() != null)
{
throw new RuntimeException(caughtErr.Get());
}
if (fileSys != null)
{
fileSys.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestGetRemoteEditLog()
{
Storage.StorageDirectory sd = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType
.Edits, false, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetFinalizedEditsFileName
(101, 200), NNStorage.GetInProgressEditsFileName(201), NNStorage.GetFinalizedEditsFileName
(1001, 1100));
// passing null for NNStorage because this unit test will not use it
FileJournalManager fjm = new FileJournalManager(conf, sd, null);
NUnit.Framework.Assert.AreEqual("[1,100],[101,200],[1001,1100]", GetLogsAsString(
fjm, 1));
NUnit.Framework.Assert.AreEqual("[101,200],[1001,1100]", GetLogsAsString(fjm, 101
));
NUnit.Framework.Assert.AreEqual("[101,200],[1001,1100]", GetLogsAsString(fjm, 150
));
NUnit.Framework.Assert.AreEqual("[1001,1100]", GetLogsAsString(fjm, 201));
NUnit.Framework.Assert.AreEqual("Asking for a newer log than exists should return empty list"
, string.Empty, GetLogsAsString(fjm, 9999));
}
public virtual void TestCurrentStorageInspector()
{
FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
();
Storage.StorageDirectory mockDir = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType
.ImageAndEdits, false, "/foo/current/" + NNStorage.GetImageFileName(123), "/foo/current/"
+ NNStorage.GetFinalizedEditsFileName(123, 456), "/foo/current/" + NNStorage.GetImageFileName
(456), "/foo/current/" + NNStorage.GetInProgressEditsFileName(457));
inspector.InspectDirectory(mockDir);
NUnit.Framework.Assert.AreEqual(2, inspector.foundImages.Count);
FSImageStorageInspector.FSImageFile latestImage = inspector.GetLatestImages()[0];
NUnit.Framework.Assert.AreEqual(456, latestImage.txId);
NUnit.Framework.Assert.AreSame(mockDir, latestImage.sd);
NUnit.Framework.Assert.IsTrue(inspector.IsUpgradeFinalized());
NUnit.Framework.Assert.AreEqual(new FilePath("/foo/current/" + NNStorage.GetImageFileName
(456)), latestImage.GetFile());
}
public virtual void TestPurgingWithNameEditsDirAfterFailure()
{
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 0);
FilePath sd0 = new FilePath(TestRootDir, "nn0");
FilePath sd1 = new FilePath(TestRootDir, "nn1");
FilePath cd0 = new FilePath(sd0, "current");
FilePath cd1 = new FilePath(sd1, "current");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Joiner.On(",").Join(sd0, sd1));
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false
).Format(true).Build();
NameNode nn = cluster.GetNameNode();
DoSaveNamespace(nn);
Log.Info("After first save, images 0 and 2 should exist in both dirs");
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(0), NNStorage.GetImageFileName(2));
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(0), NNStorage.GetImageFileName(2));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
(1, 2), NNStorage.GetInProgressEditsFileName(3));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(1, 2), NNStorage.GetInProgressEditsFileName(3));
DoSaveNamespace(nn);
Log.Info("After second save, image 0 should be purged, " + "and image 4 should exist in both."
);
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 4), NNStorage.GetInProgressEditsFileName(5));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 4), NNStorage.GetInProgressEditsFileName(5));
Log.Info("Failing first storage dir by chmodding it");
NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "000"));
DoSaveNamespace(nn);
Log.Info("Restoring accessibility of first storage dir");
NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "755"));
Log.Info("nothing should have been purged in first storage dir");
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 4), NNStorage.GetInProgressEditsFileName(5));
Log.Info("fsimage_2 should be purged in second storage dir");
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(4), NNStorage.GetImageFileName(6));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(5, 6), NNStorage.GetInProgressEditsFileName(7));
Log.Info("On next save, we should purge logs from the failed dir," + " but not images, since the image directory is in failed state."
);
DoSaveNamespace(nn);
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(6), NNStorage.GetImageFileName(8));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(7, 8), NNStorage.GetInProgressEditsFileName(9));
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetInProgressEditsFileName
(9));
}
finally
{
FileUtil.Chmod(cd0.GetAbsolutePath(), "755");
Log.Info("Shutting down...");
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestStorageRestore()
{
int numDatanodes = 0;
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(numDatanodes).ManageNameDfsDirs
(false).Build();
cluster.WaitActive();
SecondaryNameNode secondary = new SecondaryNameNode(config);
System.Console.Out.WriteLine("****testStorageRestore: Cluster and SNN started");
PrintStorages(cluster.GetNameNode().GetFSImage());
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("/", "test");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
System.Console.Out.WriteLine("****testStorageRestore: dir 'test' created, invalidating storage..."
);
InvalidateStorage(cluster.GetNameNode().GetFSImage(), ImmutableSet.Of(path2, path3
));
PrintStorages(cluster.GetNameNode().GetFSImage());
System.Console.Out.WriteLine("****testStorageRestore: storage invalidated");
path = new Path("/", "test1");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
System.Console.Out.WriteLine("****testStorageRestore: dir 'test1' created");
// We did another edit, so the still-active directory at 'path1'
// should now differ from the others
FSImageTestUtil.AssertFileContentsDifferent(2, new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
(1)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(1)),
new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1)));
FSImageTestUtil.AssertFileContentsSame(new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName
(1)), new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1)));
System.Console.Out.WriteLine("****testStorageRestore: checkfiles(false) run");
secondary.DoCheckpoint();
///should enable storage..
// We should have a checkpoint through txid 4 in the two image dirs
// (txid=4 for BEGIN, mkdir, mkdir, END)
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName
(4)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(4)));
NUnit.Framework.Assert.IsFalse("Should not have any image in an edits-only directory"
, new FilePath(path3, "current/" + NNStorage.GetImageFileName(4)).Exists());
// Should have finalized logs in the directory that didn't fail
NUnit.Framework.Assert.IsTrue("Should have finalized logs in the directory that didn't fail"
, new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
());
// Should not have finalized logs in the failed directories
NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories"
, new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
());
NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories"
, new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
());
// The new log segment should be in all of the directories.
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
(5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)),
new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5)));
string md5BeforeEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/"
+ NNStorage.GetInProgressEditsFileName(5)));
// The original image should still be the previously failed image
// directory after it got restored, since it's still useful for
// a recovery!
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName
(0)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(0)));
// Do another edit to verify that all the logs are active.
path = new Path("/", "test2");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
// Logs should be changed by the edit.
string md5AfterEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/" +
NNStorage.GetInProgressEditsFileName(5)));
NUnit.Framework.Assert.IsFalse(md5BeforeEdit.Equals(md5AfterEdit));
// And all logs should be changed.
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
(5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)),
new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5)));
secondary.Shutdown();
cluster.Shutdown();
// All logs should be finalized by clean shutdown
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName
(5, 7)), new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(5,
7)), new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(5, 7))
);
}
public virtual void TestAppendRestart()
{
Configuration conf = new HdfsConfiguration();
// Turn off persistent IPC, so that the DFSClient can survive NN restart
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
MiniDFSCluster cluster = null;
FSDataOutputStream stream = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
FileSystem fs = cluster.GetFileSystem();
FilePath editLog = new FilePath(FSImageTestUtil.GetNameNodeCurrentDirs(cluster, 0
)[0], NNStorage.GetInProgressEditsFileName(1));
EnumMap <FSEditLogOpCodes, Holder <int> > counts;
Path p1 = new Path("/block-boundaries");
WriteAndAppend(fs, p1, BlockSize, BlockSize);
counts = FSImageTestUtil.CountEditLogOpTypes(editLog);
// OP_ADD to create file
// OP_ADD_BLOCK for first block
// OP_CLOSE to close file
// OP_APPEND to reopen file
// OP_ADD_BLOCK for second block
// OP_CLOSE to close file
NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAdd].held);
NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAppend].held);
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAddBlock].held);
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpClose].held);
Path p2 = new Path("/not-block-boundaries");
WriteAndAppend(fs, p2, BlockSize / 2, BlockSize);
counts = FSImageTestUtil.CountEditLogOpTypes(editLog);
// OP_ADD to create file
// OP_ADD_BLOCK for first block
// OP_CLOSE to close file
// OP_APPEND to re-establish the lease
// OP_UPDATE_BLOCKS from the updatePipeline call (increments genstamp of last block)
// OP_ADD_BLOCK at the start of the second block
// OP_CLOSE to close file
// Total: 2 OP_ADDs, 1 OP_UPDATE_BLOCKS, 2 OP_ADD_BLOCKs, and 2 OP_CLOSEs
// in addition to the ones above
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAdd].held);
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAppend].held);
NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpUpdateBlocks].held
);
NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpAddBlock].held
);
NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpClose].held
);
cluster.RestartNameNode();
AppendTestUtil.Check(fs, p1, 2 * BlockSize);
AppendTestUtil.Check(fs, p2, 3 * BlockSize / 2);
}
finally
{
IOUtils.CloseStream(stream);
if (cluster != null)
{
cluster.Shutdown();
}
}
}