public virtual void TestSaveImageWhileSyncInProgress()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);
try
{
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = fsimage.GetEditLog();
JournalSet.JournalAndStream jas = editLog.GetJournals()[0];
EditLogFileOutputStream spyElos = Org.Mockito.Mockito.Spy((EditLogFileOutputStream
)jas.GetCurrentStream());
jas.SetCurrentStreamForTests(spyElos);
AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
CountDownLatch waitToEnterFlush = new CountDownLatch(1);
Sharpen.Thread doAnEditThread = new _Thread_371(namesystem, deferredException, waitToEnterFlush
);
Answer <Void> blockingFlush = new _Answer_388(doAnEditThread, waitToEnterFlush);
// Signal to main thread that the edit thread is in the racy section
Org.Mockito.Mockito.DoAnswer(blockingFlush).When(spyElos).Flush();
doAnEditThread.Start();
// Wait for the edit thread to get to the logsync unsynchronized section
Log.Info("Main thread: waiting to enter flush...");
waitToEnterFlush.Await();
NUnit.Framework.Assert.IsNull(deferredException.Get());
Log.Info("Main thread: detected that logSync is in unsynchronized section.");
Log.Info("Trying to enter safe mode.");
Log.Info("This should block for " + BlockTime + "sec, since flush will sleep that long"
);
long st = Time.Now();
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
long et = Time.Now();
Log.Info("Entered safe mode");
// Make sure we really waited for the flush to complete!
NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.SaveNamespace();
Log.Info("Joining on edit thread...");
doAnEditThread.Join();
NUnit.Framework.Assert.IsNull(deferredException.Get());
// We did 3 edits: begin, txn, and end
NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetFinalizedEditsFileName(1, 3), 1));
// after the save, just the one "begin"
NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetInProgressEditsFileName(4), 4));
}
finally
{
Log.Info("Closing nn");
if (namesystem != null)
{
namesystem.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestResolveReservedPath()
{
FSNamesystem fsn = SetupFileSystem();
FSEditLog editlog = fsn.GetEditLog();
fsn.GetBlockLocations("dummy", ReservedPath, 0, 1024);
Org.Mockito.Mockito.Verify(editlog).LogTimes(Matchers.Eq(FilePath), Matchers.AnyLong
(), Matchers.AnyLong());
fsn.Close();
}
public virtual void TestSaveRightBeforeSync()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);
try
{
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = Org.Mockito.Mockito.Spy(fsimage.GetEditLog());
DFSTestUtil.SetEditLogForTesting(namesystem, editLog);
AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
CountDownLatch waitToEnterSync = new CountDownLatch(1);
Sharpen.Thread doAnEditThread = new _Thread_467(namesystem, deferredException, waitToEnterSync
);
Answer <Void> blockingSync = new _Answer_484(doAnEditThread, waitToEnterSync);
Org.Mockito.Mockito.DoAnswer(blockingSync).When(editLog).LogSync();
doAnEditThread.Start();
Log.Info("Main thread: waiting to just before logSync...");
waitToEnterSync.Await();
NUnit.Framework.Assert.IsNull(deferredException.Get());
Log.Info("Main thread: detected that logSync about to be called.");
Log.Info("Trying to enter safe mode.");
Log.Info("This should block for " + BlockTime + "sec, since we have pending edits"
);
long st = Time.Now();
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
long et = Time.Now();
Log.Info("Entered safe mode");
// Make sure we really waited for the flush to complete!
NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.SaveNamespace();
Log.Info("Joining on edit thread...");
doAnEditThread.Join();
NUnit.Framework.Assert.IsNull(deferredException.Get());
// We did 3 edits: begin, txn, and end
NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetFinalizedEditsFileName(1, 3), 1));
// after the save, just the one "begin"
NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetInProgressEditsFileName(4), 4));
}
finally
{
Log.Info("Closing nn");
if (namesystem != null)
{
namesystem.Close();
}
}
}
public virtual void TestRetryCacheMetrics()
{
CheckMetrics(0, 0, 0);
// DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY is 2 ,
// so 2 requests are dropped at first.
// After that, 1 request will reach NameNode correctly.
TrySaveNamespace();
CheckMetrics(2, 0, 1);
// RetryCache will be cleared after Namesystem#close()
namesystem.Close();
CheckMetrics(2, 1, 1);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetBlockLocationsRacingWithDelete()
{
FSNamesystem fsn = Org.Mockito.Mockito.Spy(SetupFileSystem());
FSDirectory fsd = fsn.GetFSDirectory();
FSEditLog editlog = fsn.GetEditLog();
Org.Mockito.Mockito.DoAnswer(new _Answer_67(fsd)).When(fsn).WriteLock();
fsn.GetBlockLocations("dummy", ReservedPath, 0, 1024);
Org.Mockito.Mockito.Verify(editlog, Org.Mockito.Mockito.Never()).LogTimes(Matchers.AnyString
(), Matchers.AnyLong(), Matchers.AnyLong());
fsn.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetBlockLocationsRacingWithRename()
{
FSNamesystem fsn = Org.Mockito.Mockito.Spy(SetupFileSystem());
FSDirectory fsd = fsn.GetFSDirectory();
FSEditLog editlog = fsn.GetEditLog();
string DstPath = "/bar";
bool[] renamed = new bool[1];
Org.Mockito.Mockito.DoAnswer(new _Answer_92(renamed, fsd, DstPath)).When(fsn).WriteLock
();
fsn.GetBlockLocations("dummy", ReservedPath, 0, 1024);
Org.Mockito.Mockito.Verify(editlog).LogTimes(Matchers.Eq(DstPath), Matchers.AnyLong
(), Matchers.AnyLong());
fsn.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestTxIdPersistence()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
try
{
// We have a BEGIN_LOG_SEGMENT txn to start
NUnit.Framework.Assert.AreEqual(1, fsn.GetEditLog().GetLastWrittenTxId());
DoAnEdit(fsn, 1);
NUnit.Framework.Assert.AreEqual(2, fsn.GetEditLog().GetLastWrittenTxId());
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fsn.SaveNamespace();
// 2 more txns: END the first segment, BEGIN a new one
NUnit.Framework.Assert.AreEqual(4, fsn.GetEditLog().GetLastWrittenTxId());
// Shut down and restart
fsn.GetFSImage().Close();
fsn.Close();
// 1 more txn to END that segment
NUnit.Framework.Assert.AreEqual(5, fsn.GetEditLog().GetLastWrittenTxId());
fsn = null;
fsn = FSNamesystem.LoadFromDisk(conf);
// 1 more txn to start new segment on restart
NUnit.Framework.Assert.AreEqual(6, fsn.GetEditLog().GetLastWrittenTxId());
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestSaveWhileEditsRolled()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
try
{
DoAnEdit(fsn, 1);
CheckpointSignature sig = fsn.RollEditLog();
Log.Warn("Checkpoint signature: " + sig);
// Do another edit
DoAnEdit(fsn, 2);
// Save namespace
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fsn.SaveNamespace();
// Now shut down and restart the NN
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edits.
CheckEditExists(fsn, 1);
CheckEditExists(fsn, 2);
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}
/// <exception cref="System.Exception"/>
private void SaveNamespaceWithInjectedFault(TestSaveNamespace.Fault fault)
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage originalImage = fsn.GetFSImage();
NNStorage storage = originalImage.GetStorage();
NNStorage spyStorage = Org.Mockito.Mockito.Spy(storage);
originalImage.storage = spyStorage;
FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);
Whitebox.SetInternalState(fsn, "fsImage", spyImage);
bool shouldFail = false;
switch (fault)
{
case TestSaveNamespace.Fault.SaveSecondFsimageRte:
{
// should we expect the save operation to fail
// inject fault
// The spy throws a RuntimeException when writing to the second directory
Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyImage
).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
)Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
shouldFail = false;
break;
}
case TestSaveNamespace.Fault.SaveSecondFsimageIoe:
{
// The spy throws an IOException when writing to the second directory
Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(false)).When(spyImage
).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
)Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
shouldFail = false;
break;
}
case TestSaveNamespace.Fault.SaveAllFsimages:
{
// The spy throws IOException in all directories
Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyImage).SaveFSImage
((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory)Matchers.AnyObject
(), (NNStorage.NameNodeFile)Matchers.AnyObject());
shouldFail = true;
break;
}
case TestSaveNamespace.Fault.WriteStorageAll:
{
// The spy throws an exception before writing any VERSION files
Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyStorage).WriteAll
();
shouldFail = true;
break;
}
case TestSaveNamespace.Fault.WriteStorageOne:
{
// The spy throws on exception on one particular storage directory
Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyStorage
).WriteProperties((Storage.StorageDirectory)Matchers.AnyObject());
// TODO: unfortunately this fails -- should be improved.
// See HDFS-2173.
shouldFail = true;
break;
}
}
try
{
DoAnEdit(fsn, 1);
// Save namespace - this may fail, depending on fault injected
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
fsn.SaveNamespace();
if (shouldFail)
{
NUnit.Framework.Assert.Fail("Did not fail!");
}
}
catch (Exception e)
{
if (!shouldFail)
{
throw;
}
else
{
Log.Info("Test caught expected exception", e);
}
}
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
// Should still be able to perform edits
DoAnEdit(fsn, 2);
// Now shut down and restart the namesystem
originalImage.Close();
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edits.
CheckEditExists(fsn, 1);
CheckEditExists(fsn, 2);
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCancelSaveNamespace()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage image = fsn.GetFSImage();
NNStorage storage = image.GetStorage();
storage.Close();
// unlock any directories that FSNamesystem's initialization may have locked
storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem.GetNamespaceEditsDirs
(conf));
FSNamesystem spyFsn = Org.Mockito.Mockito.Spy(fsn);
FSNamesystem finalFsn = spyFsn;
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
BlockIdManager bid = Org.Mockito.Mockito.Spy(spyFsn.GetBlockIdManager());
Whitebox.SetInternalState(finalFsn, "blockIdManager", bid);
Org.Mockito.Mockito.DoAnswer(delayer).When(bid).GetGenerationStampV2();
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
DoAnEdit(fsn, 1);
Canceler canceler = new Canceler();
// Save namespace
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
Future <Void> saverFuture = pool.Submit(new _Callable_561(image, finalFsn, canceler
));
// Wait until saveNamespace calls getGenerationStamp
delayer.WaitForCall();
// then cancel the saveNamespace
Future <Void> cancelFuture = pool.Submit(new _Callable_572(canceler));
// give the cancel call time to run
Sharpen.Thread.Sleep(500);
// allow saveNamespace to proceed - it should check the cancel flag after
// this point and throw an exception
delayer.Proceed();
cancelFuture.Get();
saverFuture.Get();
NUnit.Framework.Assert.Fail("saveNamespace did not fail even though cancelled!");
}
catch (Exception t)
{
GenericTestUtils.AssertExceptionContains("SaveNamespaceCancelledException", t);
}
Log.Info("Successfully cancelled a saveNamespace");
// Check that we have only the original image and not any
// cruft left over from half-finished images
FSImageTestUtil.LogStorageContents(Log, storage);
foreach (Storage.StorageDirectory sd in storage.DirIterable(null))
{
FilePath curDir = sd.GetCurrentDir();
GenericTestUtils.AssertGlobEquals(curDir, "fsimage_.*", NNStorage.GetImageFileName
(0), NNStorage.GetImageFileName(0) + MD5FileUtils.Md5Suffix);
}
}
finally
{
fsn.Close();
}
}
/// <summary>Injects a failure on all storage directories while saving namespace.</summary>
/// <param name="restoreStorageAfterFailure">
/// if true, will try to save again after
/// clearing the failure injection
/// </param>
/// <exception cref="System.Exception"/>
public virtual void DoTestFailedSaveNamespace(bool restoreStorageAfterFailure)
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage originalImage = fsn.GetFSImage();
NNStorage storage = originalImage.GetStorage();
storage.Close();
// unlock any directories that FSNamesystem's initialization may have locked
NNStorage spyStorage = Org.Mockito.Mockito.Spy(storage);
originalImage.storage = spyStorage;
FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);
Whitebox.SetInternalState(fsn, "fsImage", spyImage);
spyImage.storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem
.GetNamespaceEditsDirs(conf));
Org.Mockito.Mockito.DoThrow(new IOException("Injected fault: saveFSImage")).When(
spyImage).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
)Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
try
{
DoAnEdit(fsn, 1);
// Save namespace
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
fsn.SaveNamespace();
NUnit.Framework.Assert.Fail("saveNamespace did not fail even when all directories failed!"
);
}
catch (IOException ioe)
{
Log.Info("Got expected exception", ioe);
}
// Ensure that, if storage dirs come back online, things work again.
if (restoreStorageAfterFailure)
{
Org.Mockito.Mockito.Reset(spyImage);
spyStorage.SetRestoreFailedStorage(true);
fsn.SaveNamespace();
CheckEditExists(fsn, 1);
}
// Now shut down and restart the NN
originalImage.Close();
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edits.
CheckEditExists(fsn, 1);
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}
/// <summary>
/// Verify that a saveNamespace command brings faulty directories
/// in fs.name.dir and fs.edit.dir back online.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestReinsertnamedirsInSavenamespace()
{
// create a configuration with the key to restore error
// directories in fs.name.dir
Configuration conf = GetConf();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeNameDirRestoreKey, true);
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage originalImage = fsn.GetFSImage();
NNStorage storage = originalImage.GetStorage();
FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);
Whitebox.SetInternalState(fsn, "fsImage", spyImage);
FileSystem fs = FileSystem.GetLocal(conf);
FilePath rootDir = storage.GetStorageDir(0).GetRoot();
Path rootPath = new Path(rootDir.GetPath(), "current");
FsPermission permissionNone = new FsPermission((short)0);
FsPermission permissionAll = new FsPermission(FsAction.All, FsAction.ReadExecute,
FsAction.ReadExecute);
fs.SetPermission(rootPath, permissionNone);
try
{
DoAnEdit(fsn, 1);
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
// Save namespace - should mark the first storage dir as faulty
// since it's not traversable.
Log.Info("Doing the first savenamespace.");
fsn.SaveNamespace();
Log.Info("First savenamespace sucessful.");
NUnit.Framework.Assert.IsTrue("Savenamespace should have marked one directory as bad."
+ " But found " + storage.GetRemovedStorageDirs().Count + " bad directories.",
storage.GetRemovedStorageDirs().Count == 1);
fs.SetPermission(rootPath, permissionAll);
// The next call to savenamespace should try inserting the
// erroneous directory back to fs.name.dir. This command should
// be successful.
Log.Info("Doing the second savenamespace.");
fsn.SaveNamespace();
Log.Warn("Second savenamespace sucessful.");
NUnit.Framework.Assert.IsTrue("Savenamespace should have been successful in removing "
+ " bad directories from Image." + " But found " + storage.GetRemovedStorageDirs
().Count + " bad directories.", storage.GetRemovedStorageDirs().Count == 0);
// Now shut down and restart the namesystem
Log.Info("Shutting down fsimage.");
originalImage.Close();
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
Log.Info("Loading new FSmage from disk.");
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edit.
Log.Info("Checking reloaded image.");
CheckEditExists(fsn, 1);
Log.Info("Reloaded image is good.");
}
finally
{
if (rootDir.Exists())
{
fs.SetPermission(rootPath, permissionAll);
}
if (fsn != null)
{
try
{
fsn.Close();
}
catch (Exception t)
{
Log.Fatal("Failed to shut down", t);
}
}
}
}