public virtual void TestStartup() { Configuration conf = new Configuration(); HAUtil.SetAllowStandbyReads(conf, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleHATopology()).NumDataNodes(0).Build(); try { // During HA startup, both nodes should be in // standby and we shouldn't have any edits files // in any edits directory! IList <URI> allDirs = Lists.NewArrayList(); Sharpen.Collections.AddAll(allDirs, cluster.GetNameDirs(0)); Sharpen.Collections.AddAll(allDirs, cluster.GetNameDirs(1)); allDirs.AddItem(cluster.GetSharedEditsDir(0, 1)); AssertNoEditFiles(allDirs); // Set the first NN to active, make sure it creates edits // in its own dirs and the shared dir. The standby // should still have no edits! cluster.TransitionToActive(0); AssertEditFiles(cluster.GetNameDirs(0), NNStorage.GetInProgressEditsFileName(1)); AssertEditFiles(Sharpen.Collections.SingletonList(cluster.GetSharedEditsDir(0, 1) ), NNStorage.GetInProgressEditsFileName(1)); AssertNoEditFiles(cluster.GetNameDirs(1)); cluster.GetNameNode(0).GetRpcServer().Mkdirs("/test", FsPermission.CreateImmutable ((short)0x1ed), true); // Restarting the standby should not finalize any edits files // in the shared directory when it starts up! cluster.RestartNameNode(1); AssertEditFiles(cluster.GetNameDirs(0), NNStorage.GetInProgressEditsFileName(1)); AssertEditFiles(Sharpen.Collections.SingletonList(cluster.GetSharedEditsDir(0, 1) ), NNStorage.GetInProgressEditsFileName(1)); AssertNoEditFiles(cluster.GetNameDirs(1)); // Additionally it should not have applied any in-progress logs // at start-up -- otherwise, it would have read half-way into // the current log segment, and on the next roll, it would have to // either replay starting in the middle of the segment (not allowed) // or double-replay the edits (incorrect). NUnit.Framework.Assert.IsNull(NameNodeAdapter.GetFileInfo(cluster.GetNameNode(1), "/test", true)); cluster.GetNameNode(0).GetRpcServer().Mkdirs("/test2", FsPermission.CreateImmutable ((short)0x1ed), true); // If we restart NN0, it'll come back as standby, and we can // transition NN1 to active and make sure it reads edits correctly at this point. cluster.RestartNameNode(0); cluster.TransitionToActive(1); // NN1 should have both the edits that came before its restart, and the edits that // came after its restart. NUnit.Framework.Assert.IsNotNull(NameNodeAdapter.GetFileInfo(cluster.GetNameNode( 1), "/test", true)); NUnit.Framework.Assert.IsNotNull(NameNodeAdapter.GetFileInfo(cluster.GetNameNode( 1), "/test2", true)); } finally { cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> private static void WaitForLogRollInSharedDir(MiniDFSCluster cluster, long startTxId ) { URI sharedUri = cluster.GetSharedEditsDir(0, 1); FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current"); FilePath expectedLog = new FilePath(sharedDir, NNStorage.GetInProgressEditsFileName (startTxId)); GenericTestUtils.WaitFor(new _Supplier_153(expectedLog), 100, 10000); }
public virtual void TestChangeWritersLogsInSync() { QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, false); QJMTestUtil.AssertExistsInQuorum(cluster, NNStorage.GetInProgressEditsFileName(1) ); // Make a new QJM qjm = CloseLater(new QuorumJournalManager(conf, cluster.GetQuorumJournalURI(QJMTestUtil .Jid), QJMTestUtil.FakeNsinfo)); qjm.RecoverUnfinalizedSegments(); CheckRecovery(cluster, 1, 3); }
public virtual void TestOldInProgress() { TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(101), true); RunTest(tc); }
/// <exception cref="System.Exception"/> private void DoOutOfSyncTest(int missingOnRecoveryIdx, long expectedRecoveryTxnId ) { SetupLoggers345(); QJMTestUtil.AssertExistsInQuorum(cluster, NNStorage.GetInProgressEditsFileName(1) ); // Shut down the specified JN, so it's not present during recovery. cluster.GetJournalNode(missingOnRecoveryIdx).StopAndJoin(0); // Make a new QJM qjm = CreateSpyingQJM(); qjm.RecoverUnfinalizedSegments(); CheckRecovery(cluster, 1, expectedRecoveryTxnId); }
/// <exception cref="System.Exception"/> private void TestFailoverFinalizesAndReadsInProgress(bool partialTxAtEnd) { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleHATopology()).NumDataNodes(0).Build(); try { // Create a fake in-progress edit-log in the shared directory URI sharedUri = cluster.GetSharedEditsDir(0, 1); FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current"); FSNamesystem fsn = cluster.GetNamesystem(0); FSImageTestUtil.CreateAbortedLogWithMkdirs(sharedDir, NumDirsInLog, 1, fsn.GetFSDirectory ().GetLastInodeId() + 1); AssertEditFiles(Sharpen.Collections.SingletonList(sharedUri), NNStorage.GetInProgressEditsFileName (1)); if (partialTxAtEnd) { FileOutputStream outs = null; try { FilePath editLogFile = new FilePath(sharedDir, NNStorage.GetInProgressEditsFileName (1)); outs = new FileOutputStream(editLogFile, true); outs.Write(new byte[] { unchecked ((int)(0x18)), unchecked ((int)(0x00)), unchecked ( (int)(0x00)), unchecked ((int)(0x00)) }); Log.Error("editLogFile = " + editLogFile); } finally { IOUtils.Cleanup(Log, outs); } } // Transition one of the NNs to active cluster.TransitionToActive(0); // In the transition to active, it should have read the log -- and // hence see one of the dirs we made in the fake log. string testPath = "/dir" + NumDirsInLog; NUnit.Framework.Assert.IsNotNull(cluster.GetNameNode(0).GetRpcServer().GetFileInfo (testPath)); // It also should have finalized that log in the shared directory and started // writing to a new one at the next txid. AssertEditFiles(Sharpen.Collections.SingletonList(sharedUri), NNStorage.GetFinalizedEditsFileName (1, NumDirsInLog + 1), NNStorage.GetInProgressEditsFileName(NumDirsInLog + 2)); } finally { cluster.Shutdown(); } }
/// <summary>Create an unfinalized edit log for testing purposes</summary> /// <param name="testDir">Directory to create the edit log in</param> /// <param name="numTx">Number of transactions to add to the new edit log</param> /// <param name="offsetToTxId"> /// A map from transaction IDs to offsets in the /// edit log file. /// </param> /// <returns>The new edit log file name.</returns> /// <exception cref="System.IO.IOException"/> private static FilePath PrepareUnfinalizedTestEditLog(FilePath testDir, int numTx , SortedDictionary <long, long> offsetToTxId) { FilePath inProgressFile = new FilePath(testDir, NNStorage.GetInProgressEditsFileName (1)); FSEditLog fsel = null; FSEditLog spyLog = null; try { fsel = FSImageTestUtil.CreateStandaloneEditLog(testDir); spyLog = Org.Mockito.Mockito.Spy(fsel); // Normally, the in-progress edit log would be finalized by // FSEditLog#endCurrentLogSegment. For testing purposes, we // disable that here. Org.Mockito.Mockito.DoNothing().When(spyLog).EndCurrentLogSegment(true); spyLog.OpenForWrite(); NUnit.Framework.Assert.IsTrue("should exist: " + inProgressFile, inProgressFile.Exists ()); for (int i = 0; i < numTx; i++) { long trueOffset = GetNonTrailerLength(inProgressFile); long thisTxId = spyLog.GetLastWrittenTxId() + 1; offsetToTxId[trueOffset] = thisTxId; System.Console.Error.WriteLine("txid " + thisTxId + " at offset " + trueOffset); spyLog.LogDelete("path" + i, i, false); spyLog.LogSync(); } } finally { if (spyLog != null) { spyLog.Close(); } else { if (fsel != null) { fsel.Close(); } } } return(inProgressFile); }
public virtual void TestPurgeEasyCase() { TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), true); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false ); tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false); // Test that other files don't get purged tc.AddLog("/foo1/current/VERSION", false); RunTest(tc); }
public virtual void TestRetainExtraLogs() { conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 50); TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image); tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true); // Since we need 50 extra edits, *do* retain the 201-300 segment tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false ); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false); RunTest(tc); }
/// <exception cref="System.IO.IOException"/> public static EditLogOutputStream WriteSegment(MiniJournalCluster cluster, QuorumJournalManager qjm, long startTxId, int numTxns, bool finalize) { EditLogOutputStream stm = qjm.StartLogSegment(startTxId, NameNodeLayoutVersion.CurrentLayoutVersion ); // Should create in-progress AssertExistsInQuorum(cluster, NNStorage.GetInProgressEditsFileName(startTxId)); WriteTxns(stm, startTxId, numTxns); if (finalize) { stm.Close(); qjm.FinalizeLogSegment(startTxId, startTxId + numTxns - 1); return(null); } else { return(stm); } }
/// <summary>For namenode, Verify that the current and previous directories exist.</summary> /// <remarks> /// For namenode, Verify that the current and previous directories exist. /// Verify that previous hasn't been modified by comparing the checksum of all /// its files with their original checksum. It is assumed that the /// server has recovered and upgraded. /// </remarks> /// <exception cref="System.IO.IOException"/> internal virtual void CheckNameNode(string[] baseDirs, long imageTxId) { foreach (string baseDir in baseDirs) { Log.Info("Checking namenode directory " + baseDir); Log.Info("==== Contents ====:\n " + Joiner.On(" \n").Join(new FilePath(baseDir, "current").List())); Log.Info("=================="); GenericTestUtils.AssertExists(new FilePath(baseDir, "current")); GenericTestUtils.AssertExists(new FilePath(baseDir, "current/VERSION")); GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetInProgressEditsFileName (imageTxId + 1))); GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetImageFileName (imageTxId))); GenericTestUtils.AssertExists(new FilePath(baseDir, "current/seen_txid")); FilePath previous = new FilePath(baseDir, "previous"); GenericTestUtils.AssertExists(previous); NUnit.Framework.Assert.AreEqual(UpgradeUtilities.ChecksumContents(HdfsServerConstants.NodeType .NameNode, previous, false), UpgradeUtilities.ChecksumMasterNameNodeContents()); } }
public virtual void TestRetainExtraLogsLimitedSegments() { conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 150); conf.SetLong(DFSConfigKeys.DfsNamenodeMaxExtraEditsSegmentsRetainedKey, 2); TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image); tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); // Segments containing txns upto txId 250 are extra and should be purged. tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(1, 100), true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 175), true); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(176) + ".empty" , true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(176, 200), true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 225), true); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(226) + ".corrupt" , true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(226, 240), true); // Only retain 2 extra segments. The 301-350 and 351-400 segments are // considered required, not extra. tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(241, 275), false ); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(276, 300), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(301) + ".empty" , false); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 350), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(351) + ".corrupt" , false); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(351, 400), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false); RunTest(tc); }
/// <summary> /// Test cancellation of ongoing checkpoints when failover happens /// mid-checkpoint. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestCheckpointCancellation() { cluster.TransitionToStandby(0); // Create an edit log in the shared edits dir with a lot // of mkdirs operations. This is solely so that the image is // large enough to take a non-trivial amount of time to load. // (only ~15MB) URI sharedUri = cluster.GetSharedEditsDir(0, 1); FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current"); FilePath tmpDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "testCheckpointCancellation-tmp" ); FSNamesystem fsn = cluster.GetNamesystem(0); FSImageTestUtil.CreateAbortedLogWithMkdirs(tmpDir, NumDirsInLog, 3, fsn.GetFSDirectory ().GetLastInodeId() + 1); string fname = NNStorage.GetInProgressEditsFileName(3); new FilePath(tmpDir, fname).RenameTo(new FilePath(sharedDir, fname)); // Checkpoint as fast as we can, in a tight loop. cluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey, 0); cluster.RestartNameNode(1); nn1 = cluster.GetNameNode(1); cluster.TransitionToActive(0); bool canceledOne = false; for (int i = 0; i < 10 && !canceledOne; i++) { DoEdits(i * 10, i * 10 + 10); cluster.TransitionToStandby(0); cluster.TransitionToActive(1); cluster.TransitionToStandby(1); cluster.TransitionToActive(0); canceledOne = StandbyCheckpointer.GetCanceledCount() > 0; } NUnit.Framework.Assert.IsTrue(canceledOne); }
private void CheckNNStorage(NNStorage storage, long imageTxId, long trashEndTxId) { IList <FilePath> finalizedEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits , NNStorage.GetFinalizedEditsFileName(1, imageTxId)); NUnit.Framework.Assert.IsTrue(FileExists(finalizedEdits)); IList <FilePath> inprogressEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits , NNStorage.GetInProgressEditsFileName(imageTxId + 1)); // For rollback case we will have an inprogress file for future transactions NUnit.Framework.Assert.IsTrue(FileExists(inprogressEdits)); if (trashEndTxId > 0) { IList <FilePath> trashedEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits, NNStorage.GetFinalizedEditsFileName(imageTxId + 1, trashEndTxId) + ".trash"); NUnit.Framework.Assert.IsTrue(FileExists(trashedEdits)); } string imageFileName = trashEndTxId > 0 ? NNStorage.GetImageFileName(imageTxId) : NNStorage.GetRollbackImageFileName(imageTxId); IList <FilePath> imageFiles = storage.GetFiles(NNStorage.NameNodeDirType.Image, imageFileName ); NUnit.Framework.Assert.IsTrue(FileExists(imageFiles)); }
public virtual void TestFailureOfSharedDir() { Configuration conf = new Configuration(); conf.SetLong(DFSConfigKeys.DfsNamenodeResourceCheckIntervalKey, 2000); // The shared edits dir will automatically be marked required. MiniDFSCluster cluster = null; FilePath sharedEditsDir = null; try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).CheckExitOnShutdown(false).Build(); cluster.WaitActive(); cluster.TransitionToActive(0); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/test1"))); // Blow away the shared edits dir. URI sharedEditsUri = cluster.GetSharedEditsDir(0, 1); sharedEditsDir = new FilePath(sharedEditsUri); NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(sharedEditsDir.GetAbsolutePath( ), "-w", true)); Sharpen.Thread.Sleep(conf.GetLong(DFSConfigKeys.DfsNamenodeResourceCheckIntervalKey , DFSConfigKeys.DfsNamenodeResourceCheckIntervalDefault) * 2); NameNode nn1 = cluster.GetNameNode(1); NUnit.Framework.Assert.IsTrue(nn1.IsStandbyState()); NUnit.Framework.Assert.IsFalse("StandBy NameNode should not go to SafeMode on resource unavailability" , nn1.IsInSafeMode()); NameNode nn0 = cluster.GetNameNode(0); try { // Make sure that subsequent operations on the NN fail. nn0.GetRpcServer().RollEditLog(); NUnit.Framework.Assert.Fail("Succeeded in rolling edit log despite shared dir being deleted" ); } catch (ExitUtil.ExitException ee) { GenericTestUtils.AssertExceptionContains("finalize log segment 1, 3 failed for required journal" , ee); } // Check that none of the edits dirs rolled, since the shared edits // dir didn't roll. Regression test for HDFS-2874. foreach (URI editsUri in cluster.GetNameEditsDirs(0)) { if (editsUri.Equals(sharedEditsUri)) { continue; } FilePath editsDir = new FilePath(editsUri.GetPath()); FilePath curDir = new FilePath(editsDir, "current"); GenericTestUtils.AssertGlobEquals(curDir, "edits_.*", NNStorage.GetInProgressEditsFileName (1)); } } finally { if (sharedEditsDir != null) { // without this test cleanup will fail FileUtil.Chmod(sharedEditsDir.GetAbsolutePath(), "+w", true); } if (cluster != null) { cluster.Shutdown(); } } }
/// <summary> /// Set up the following tricky edge case state which is used by /// multiple tests: /// Initial writer: /// - Writing to 3 JNs: JN0, JN1, JN2: /// - A log segment with txnid 1 through 100 succeeds. /// </summary> /// <remarks> /// Set up the following tricky edge case state which is used by /// multiple tests: /// Initial writer: /// - Writing to 3 JNs: JN0, JN1, JN2: /// - A log segment with txnid 1 through 100 succeeds. /// - The first transaction in the next segment only goes to JN0 /// before the writer crashes (eg it is partitioned) /// Recovery by another writer: /// - The new NN starts recovery and talks to all three. Thus, it sees /// that the newest log segment which needs recovery is 101. /// - It sends the prepareRecovery(101) call, and decides that the /// recovery length for 101 is only the 1 transaction. /// - It sends acceptRecovery(101-101) to only JN0, before crashing /// This yields the following state: /// - JN0: 1-100 finalized, 101_inprogress, accepted recovery: 101-101 /// - JN1: 1-100 finalized, 101_inprogress.empty /// - JN2: 1-100 finalized, 101_inprogress.empty /// (the .empty files got moved aside during recovery) /// </remarks> /// <exception cref="System.Exception"></exception> private void SetupEdgeCaseOneJnHasSegmentWithAcceptedRecovery() { // Log segment with txns 1-100 succeeds QJMTestUtil.WriteSegment(cluster, qjm, 1, 100, true); // startLogSegment only makes it to one of the three nodes FailLoggerAtTxn(spies[1], 101); FailLoggerAtTxn(spies[2], 101); try { QJMTestUtil.WriteSegment(cluster, qjm, 101, 1, true); NUnit.Framework.Assert.Fail("Should have failed"); } catch (QuorumException qe) { GenericTestUtils.AssertExceptionContains("mock failure", qe); } finally { qjm.Close(); } // Recovery 1: // make acceptRecovery() only make it to the node which has txid 101 // this should fail because only 1/3 accepted the recovery qjm = CreateSpyingQJM(); spies = qjm.GetLoggerSetForTests().GetLoggersForTests(); TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spies [1]).AcceptRecovery(Org.Mockito.Mockito.Any <QJournalProtocolProtos.SegmentStateProto >(), Org.Mockito.Mockito.Any <Uri>()); TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spies [2]).AcceptRecovery(Org.Mockito.Mockito.Any <QJournalProtocolProtos.SegmentStateProto >(), Org.Mockito.Mockito.Any <Uri>()); try { qjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.Fail("Should have failed to recover"); } catch (QuorumException qe) { GenericTestUtils.AssertExceptionContains("mock failure", qe); } finally { qjm.Close(); } // Check that we have entered the expected state as described in the // method javadoc. GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(0, QJMTestUtil.Jid), "edits_.*" , NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName (101)); GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(1, QJMTestUtil.Jid), "edits_.*" , NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName (101) + ".empty"); GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(2, QJMTestUtil.Jid), "edits_.*" , NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName (101) + ".empty"); FilePath paxos0 = new FilePath(cluster.GetCurrentDir(0, QJMTestUtil.Jid), "paxos" ); FilePath paxos1 = new FilePath(cluster.GetCurrentDir(1, QJMTestUtil.Jid), "paxos" ); FilePath paxos2 = new FilePath(cluster.GetCurrentDir(2, QJMTestUtil.Jid), "paxos" ); GenericTestUtils.AssertGlobEquals(paxos0, ".*", "101"); GenericTestUtils.AssertGlobEquals(paxos1, ".*"); GenericTestUtils.AssertGlobEquals(paxos2, ".*"); }
/// <returns> /// the path for an in-progress edits file starting at the given /// transaction ID. This does not verify existence of the file. /// </returns> internal virtual FilePath GetInProgressEditLog(long startTxId) { return(new FilePath(sd.GetCurrentDir(), NNStorage.GetInProgressEditsFileName(startTxId ))); }
/// <param name="segmentTxId">the first txid of the segment</param> /// <param name="epoch"> /// the epoch number of the writer which is coordinating /// recovery /// </param> /// <returns> /// the temporary path in which an edits log should be stored /// while it is being downloaded from a remote JournalNode /// </returns> internal virtual FilePath GetSyncLogTemporaryFile(long segmentTxId, long epoch) { string name = NNStorage.GetInProgressEditsFileName(segmentTxId) + ".epoch=" + epoch; return(new FilePath(sd.GetCurrentDir(), name)); }
/// <summary> /// Test the case where, at the beginning of a segment, transactions /// have been written to one JN but not others. /// </summary> /// <exception cref="System.Exception"/> public virtual void DoTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn) { int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3; int nodeMissingSegment = (nodeWithOneTxn + 2) % 3; QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true); WaitForAllPendingCalls(qjm.GetLoggerSetForTests()); cluster.GetJournalNode(nodeMissingSegment).StopAndJoin(0); // Open segment on 2/3 nodes EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion ); try { WaitForAllPendingCalls(qjm.GetLoggerSetForTests()); // Write transactions to only 1/3 nodes FailLoggerAtTxn(spies[nodeWithEmptySegment], 4); try { QJMTestUtil.WriteTxns(stm, 4, 1); NUnit.Framework.Assert.Fail("Did not fail even though 2/3 failed"); } catch (QuorumException qe) { GenericTestUtils.AssertExceptionContains("mock failure", qe); } } finally { stm.Abort(); } // Bring back the down JN. cluster.RestartJournalNode(nodeMissingSegment); // Make a new QJM. At this point, the state is as follows: // A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty) // B: nodeWithOneTxn: 1-3 finalized, 4_inprogress (1 txn) // C: nodeMissingSegment: 1-3 finalized GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithEmptySegment, QJMTestUtil .Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName (4)); GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithOneTxn, QJMTestUtil .Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName (4)); GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeMissingSegment, QJMTestUtil .Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3)); // Stop one of the nodes. Since we run this test three // times, rotating the roles of the nodes, we'll test // all the permutations. cluster.GetJournalNode(2).StopAndJoin(0); qjm = CreateSpyingQJM(); qjm.RecoverUnfinalizedSegments(); if (nodeWithOneTxn == 0 || nodeWithOneTxn == 1) { // If the node that had the transaction committed was one of the nodes // that responded during recovery, then we should have recovered txid // 4. CheckRecovery(cluster, 4, 4); QJMTestUtil.WriteSegment(cluster, qjm, 5, 3, true); } else { // Otherwise, we should have recovered only 1-3 and should be able to // start a segment at 4. CheckRecovery(cluster, 1, 3); QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true); } }
public virtual void TestSaveRightBeforeSync() { Configuration conf = GetConf(); NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode); DFSTestUtil.FormatNameNode(conf); FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf); try { FSImage fsimage = namesystem.GetFSImage(); FSEditLog editLog = Org.Mockito.Mockito.Spy(fsimage.GetEditLog()); DFSTestUtil.SetEditLogForTesting(namesystem, editLog); AtomicReference <Exception> deferredException = new AtomicReference <Exception>(); CountDownLatch waitToEnterSync = new CountDownLatch(1); Sharpen.Thread doAnEditThread = new _Thread_467(namesystem, deferredException, waitToEnterSync ); Answer <Void> blockingSync = new _Answer_484(doAnEditThread, waitToEnterSync); Org.Mockito.Mockito.DoAnswer(blockingSync).When(editLog).LogSync(); doAnEditThread.Start(); Log.Info("Main thread: waiting to just before logSync..."); waitToEnterSync.Await(); NUnit.Framework.Assert.IsNull(deferredException.Get()); Log.Info("Main thread: detected that logSync about to be called."); Log.Info("Trying to enter safe mode."); Log.Info("This should block for " + BlockTime + "sec, since we have pending edits" ); long st = Time.Now(); namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); long et = Time.Now(); Log.Info("Entered safe mode"); // Make sure we really waited for the flush to complete! NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000); // Once we're in safe mode, save namespace. namesystem.SaveNamespace(); Log.Info("Joining on edit thread..."); doAnEditThread.Join(); NUnit.Framework.Assert.IsNull(deferredException.Get()); // We did 3 edits: begin, txn, and end NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage. GetFinalizedEditsFileName(1, 3), 1)); // after the save, just the one "begin" NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage. GetInProgressEditsFileName(4), 4)); } finally { Log.Info("Closing nn"); if (namesystem != null) { namesystem.Close(); } } }
public virtual void TestSaveImageWhileSyncInProgress() { Configuration conf = GetConf(); NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode); DFSTestUtil.FormatNameNode(conf); FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf); try { FSImage fsimage = namesystem.GetFSImage(); FSEditLog editLog = fsimage.GetEditLog(); JournalSet.JournalAndStream jas = editLog.GetJournals()[0]; EditLogFileOutputStream spyElos = Org.Mockito.Mockito.Spy((EditLogFileOutputStream )jas.GetCurrentStream()); jas.SetCurrentStreamForTests(spyElos); AtomicReference <Exception> deferredException = new AtomicReference <Exception>(); CountDownLatch waitToEnterFlush = new CountDownLatch(1); Sharpen.Thread doAnEditThread = new _Thread_371(namesystem, deferredException, waitToEnterFlush ); Answer <Void> blockingFlush = new _Answer_388(doAnEditThread, waitToEnterFlush); // Signal to main thread that the edit thread is in the racy section Org.Mockito.Mockito.DoAnswer(blockingFlush).When(spyElos).Flush(); doAnEditThread.Start(); // Wait for the edit thread to get to the logsync unsynchronized section Log.Info("Main thread: waiting to enter flush..."); waitToEnterFlush.Await(); NUnit.Framework.Assert.IsNull(deferredException.Get()); Log.Info("Main thread: detected that logSync is in unsynchronized section."); Log.Info("Trying to enter safe mode."); Log.Info("This should block for " + BlockTime + "sec, since flush will sleep that long" ); long st = Time.Now(); namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); long et = Time.Now(); Log.Info("Entered safe mode"); // Make sure we really waited for the flush to complete! NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000); // Once we're in safe mode, save namespace. namesystem.SaveNamespace(); Log.Info("Joining on edit thread..."); doAnEditThread.Join(); NUnit.Framework.Assert.IsNull(deferredException.Get()); // We did 3 edits: begin, txn, and end NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage. GetFinalizedEditsFileName(1, 3), 1)); // after the save, just the one "begin" NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage. GetInProgressEditsFileName(4), 4)); } finally { Log.Info("Closing nn"); if (namesystem != null) { namesystem.Close(); } } }
public virtual void TestSaveNamespace() { // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; FileSystem fileSys = null; AtomicReference <Exception> caughtErr = new AtomicReference <Exception>(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build(); cluster.WaitActive(); fileSys = cluster.GetFileSystem(); FSNamesystem namesystem = cluster.GetNamesystem(); NamenodeProtocols nn = cluster.GetNameNodeRpc(); FSImage fsimage = namesystem.GetFSImage(); FSEditLog editLog = fsimage.GetEditLog(); StartTransactionWorkers(nn, caughtErr); for (int i = 0; i < NumSaveImage && caughtErr.Get() == null; i++) { try { Sharpen.Thread.Sleep(20); } catch (Exception) { } Log.Info("Save " + i + ": entering safe mode"); namesystem.EnterSafeMode(false); // Verify edit logs before the save // They should start with the first edit after the checkpoint long logStartTxId = fsimage.GetStorage().GetMostRecentCheckpointTxId() + 1; VerifyEditLogs(namesystem, fsimage, NNStorage.GetInProgressEditsFileName(logStartTxId ), logStartTxId); Log.Info("Save " + i + ": saving namespace"); namesystem.SaveNamespace(); Log.Info("Save " + i + ": leaving safemode"); long savedImageTxId = fsimage.GetStorage().GetMostRecentCheckpointTxId(); // Verify that edit logs post save got finalized and aren't corrupt VerifyEditLogs(namesystem, fsimage, NNStorage.GetFinalizedEditsFileName(logStartTxId , savedImageTxId), logStartTxId); // The checkpoint id should be 1 less than the last written ID, since // the log roll writes the "BEGIN" transaction to the new log. NUnit.Framework.Assert.AreEqual(fsimage.GetStorage().GetMostRecentCheckpointTxId( ), editLog.GetLastWrittenTxId() - 1); namesystem.LeaveSafeMode(); Log.Info("Save " + i + ": complete"); } } finally { StopTransactionWorkers(); if (caughtErr.Get() != null) { throw new RuntimeException(caughtErr.Get()); } if (fileSys != null) { fileSys.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestGetRemoteEditLog() { Storage.StorageDirectory sd = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType .Edits, false, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetFinalizedEditsFileName (101, 200), NNStorage.GetInProgressEditsFileName(201), NNStorage.GetFinalizedEditsFileName (1001, 1100)); // passing null for NNStorage because this unit test will not use it FileJournalManager fjm = new FileJournalManager(conf, sd, null); NUnit.Framework.Assert.AreEqual("[1,100],[101,200],[1001,1100]", GetLogsAsString( fjm, 1)); NUnit.Framework.Assert.AreEqual("[101,200],[1001,1100]", GetLogsAsString(fjm, 101 )); NUnit.Framework.Assert.AreEqual("[101,200],[1001,1100]", GetLogsAsString(fjm, 150 )); NUnit.Framework.Assert.AreEqual("[1001,1100]", GetLogsAsString(fjm, 201)); NUnit.Framework.Assert.AreEqual("Asking for a newer log than exists should return empty list" , string.Empty, GetLogsAsString(fjm, 9999)); }
public virtual void TestCurrentStorageInspector() { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); Storage.StorageDirectory mockDir = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType .ImageAndEdits, false, "/foo/current/" + NNStorage.GetImageFileName(123), "/foo/current/" + NNStorage.GetFinalizedEditsFileName(123, 456), "/foo/current/" + NNStorage.GetImageFileName (456), "/foo/current/" + NNStorage.GetInProgressEditsFileName(457)); inspector.InspectDirectory(mockDir); NUnit.Framework.Assert.AreEqual(2, inspector.foundImages.Count); FSImageStorageInspector.FSImageFile latestImage = inspector.GetLatestImages()[0]; NUnit.Framework.Assert.AreEqual(456, latestImage.txId); NUnit.Framework.Assert.AreSame(mockDir, latestImage.sd); NUnit.Framework.Assert.IsTrue(inspector.IsUpgradeFinalized()); NUnit.Framework.Assert.AreEqual(new FilePath("/foo/current/" + NNStorage.GetImageFileName (456)), latestImage.GetFile()); }
public virtual void TestPurgingWithNameEditsDirAfterFailure() { MiniDFSCluster cluster = null; Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 0); FilePath sd0 = new FilePath(TestRootDir, "nn0"); FilePath sd1 = new FilePath(TestRootDir, "nn1"); FilePath cd0 = new FilePath(sd0, "current"); FilePath cd1 = new FilePath(sd1, "current"); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Joiner.On(",").Join(sd0, sd1)); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false ).Format(true).Build(); NameNode nn = cluster.GetNameNode(); DoSaveNamespace(nn); Log.Info("After first save, images 0 and 2 should exist in both dirs"); GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName (0), NNStorage.GetImageFileName(2)); GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName (0), NNStorage.GetImageFileName(2)); GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName (1, 2), NNStorage.GetInProgressEditsFileName(3)); GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName (1, 2), NNStorage.GetInProgressEditsFileName(3)); DoSaveNamespace(nn); Log.Info("After second save, image 0 should be purged, " + "and image 4 should exist in both." ); GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName (2), NNStorage.GetImageFileName(4)); GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName (2), NNStorage.GetImageFileName(4)); GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName (3, 4), NNStorage.GetInProgressEditsFileName(5)); GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName (3, 4), NNStorage.GetInProgressEditsFileName(5)); Log.Info("Failing first storage dir by chmodding it"); NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "000")); DoSaveNamespace(nn); Log.Info("Restoring accessibility of first storage dir"); NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "755")); Log.Info("nothing should have been purged in first storage dir"); GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName (2), NNStorage.GetImageFileName(4)); GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName (3, 4), NNStorage.GetInProgressEditsFileName(5)); Log.Info("fsimage_2 should be purged in second storage dir"); GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName (4), NNStorage.GetImageFileName(6)); GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName (5, 6), NNStorage.GetInProgressEditsFileName(7)); Log.Info("On next save, we should purge logs from the failed dir," + " but not images, since the image directory is in failed state." ); DoSaveNamespace(nn); GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName (6), NNStorage.GetImageFileName(8)); GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName (7, 8), NNStorage.GetInProgressEditsFileName(9)); GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName (2), NNStorage.GetImageFileName(4)); GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetInProgressEditsFileName (9)); } finally { FileUtil.Chmod(cd0.GetAbsolutePath(), "755"); Log.Info("Shutting down..."); if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestStorageRestore() { int numDatanodes = 0; cluster = new MiniDFSCluster.Builder(config).NumDataNodes(numDatanodes).ManageNameDfsDirs (false).Build(); cluster.WaitActive(); SecondaryNameNode secondary = new SecondaryNameNode(config); System.Console.Out.WriteLine("****testStorageRestore: Cluster and SNN started"); PrintStorages(cluster.GetNameNode().GetFSImage()); FileSystem fs = cluster.GetFileSystem(); Path path = new Path("/", "test"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path)); System.Console.Out.WriteLine("****testStorageRestore: dir 'test' created, invalidating storage..." ); InvalidateStorage(cluster.GetNameNode().GetFSImage(), ImmutableSet.Of(path2, path3 )); PrintStorages(cluster.GetNameNode().GetFSImage()); System.Console.Out.WriteLine("****testStorageRestore: storage invalidated"); path = new Path("/", "test1"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path)); System.Console.Out.WriteLine("****testStorageRestore: dir 'test1' created"); // We did another edit, so the still-active directory at 'path1' // should now differ from the others FSImageTestUtil.AssertFileContentsDifferent(2, new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName (1)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(1)), new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1))); FSImageTestUtil.AssertFileContentsSame(new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName (1)), new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1))); System.Console.Out.WriteLine("****testStorageRestore: checkfiles(false) run"); secondary.DoCheckpoint(); ///should enable storage.. // We should have a checkpoint through txid 4 in the two image dirs // (txid=4 for BEGIN, mkdir, mkdir, END) FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName (4)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(4))); NUnit.Framework.Assert.IsFalse("Should not have any image in an edits-only directory" , new FilePath(path3, "current/" + NNStorage.GetImageFileName(4)).Exists()); // Should have finalized logs in the directory that didn't fail NUnit.Framework.Assert.IsTrue("Should have finalized logs in the directory that didn't fail" , new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists ()); // Should not have finalized logs in the failed directories NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories" , new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists ()); NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories" , new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists ()); // The new log segment should be in all of the directories. FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName (5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)), new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5))); string md5BeforeEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName(5))); // The original image should still be the previously failed image // directory after it got restored, since it's still useful for // a recovery! FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName (0)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(0))); // Do another edit to verify that all the logs are active. path = new Path("/", "test2"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path)); // Logs should be changed by the edit. string md5AfterEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName(5))); NUnit.Framework.Assert.IsFalse(md5BeforeEdit.Equals(md5AfterEdit)); // And all logs should be changed. FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName (5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)), new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5))); secondary.Shutdown(); cluster.Shutdown(); // All logs should be finalized by clean shutdown FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName (5, 7)), new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(5, 7)), new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(5, 7)) ); }
public virtual void TestAppendRestart() { Configuration conf = new HdfsConfiguration(); // Turn off persistent IPC, so that the DFSClient can survive NN restart conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0); MiniDFSCluster cluster = null; FSDataOutputStream stream = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); FileSystem fs = cluster.GetFileSystem(); FilePath editLog = new FilePath(FSImageTestUtil.GetNameNodeCurrentDirs(cluster, 0 )[0], NNStorage.GetInProgressEditsFileName(1)); EnumMap <FSEditLogOpCodes, Holder <int> > counts; Path p1 = new Path("/block-boundaries"); WriteAndAppend(fs, p1, BlockSize, BlockSize); counts = FSImageTestUtil.CountEditLogOpTypes(editLog); // OP_ADD to create file // OP_ADD_BLOCK for first block // OP_CLOSE to close file // OP_APPEND to reopen file // OP_ADD_BLOCK for second block // OP_CLOSE to close file NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAdd].held); NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAppend].held); NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAddBlock].held); NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpClose].held); Path p2 = new Path("/not-block-boundaries"); WriteAndAppend(fs, p2, BlockSize / 2, BlockSize); counts = FSImageTestUtil.CountEditLogOpTypes(editLog); // OP_ADD to create file // OP_ADD_BLOCK for first block // OP_CLOSE to close file // OP_APPEND to re-establish the lease // OP_UPDATE_BLOCKS from the updatePipeline call (increments genstamp of last block) // OP_ADD_BLOCK at the start of the second block // OP_CLOSE to close file // Total: 2 OP_ADDs, 1 OP_UPDATE_BLOCKS, 2 OP_ADD_BLOCKs, and 2 OP_CLOSEs // in addition to the ones above NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAdd].held); NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAppend].held); NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpUpdateBlocks].held ); NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpAddBlock].held ); NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpClose].held ); cluster.RestartNameNode(); AppendTestUtil.Check(fs, p1, 2 * BlockSize); AppendTestUtil.Check(fs, p2, 3 * BlockSize / 2); } finally { IOUtils.CloseStream(stream); if (cluster != null) { cluster.Shutdown(); } } }