/// <exception cref="System.IO.IOException"/>
internal static void DownloadEditsToStorage(Uri fsName, RemoteEditLog log, NNStorage
dstStorage)
{
System.Diagnostics.Debug.Assert(log.GetStartTxId() > 0 && log.GetEndTxId() > 0, "bad log: "
+ log);
string fileid = ImageServlet.GetParamStringForLog(log, dstStorage);
string finalFileName = NNStorage.GetFinalizedEditsFileName(log.GetStartTxId(), log
.GetEndTxId());
IList <FilePath> finalFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits,
finalFileName);
System.Diagnostics.Debug.Assert(!finalFiles.IsEmpty(), "No checkpoint targets.");
foreach (FilePath f in finalFiles)
{
if (f.Exists() && FileUtil.CanRead(f))
{
Log.Info("Skipping download of remote edit log " + log + " since it already is stored locally at "
+ f);
return;
}
else
{
if (Log.IsDebugEnabled())
{
Log.Debug("Dest file: " + f);
}
}
}
long milliTime = Time.MonotonicNow();
string tmpFileName = NNStorage.GetTemporaryEditsFileName(log.GetStartTxId(), log.
GetEndTxId(), milliTime);
IList <FilePath> tmpFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, tmpFileName
);
GetFileClient(fsName, fileid, tmpFiles, dstStorage, false);
Log.Info("Downloaded file " + tmpFiles[0].GetName() + " size " + finalFiles[0].Length
() + " bytes.");
CheckpointFaultInjector.GetInstance().BeforeEditsRename();
foreach (Storage.StorageDirectory sd in dstStorage.DirIterable(NNStorage.NameNodeDirType
.Edits))
{
FilePath tmpFile = NNStorage.GetTemporaryEditsFile(sd, log.GetStartTxId(), log.GetEndTxId
(), milliTime);
FilePath finalizedFile = NNStorage.GetFinalizedEditsFile(sd, log.GetStartTxId(),
log.GetEndTxId());
if (Log.IsDebugEnabled())
{
Log.Debug("Renaming " + tmpFile + " to " + finalizedFile);
}
bool success = tmpFile.RenameTo(finalizedFile);
if (!success)
{
Log.Warn("Unable to rename edits file from " + tmpFile + " to " + finalizedFile);
}
}
}
/// <returns>
/// a List which contains the "current" dir for each storage
/// directory of the given type.
/// </returns>
public static IList <FilePath> GetCurrentDirs(NNStorage storage, NNStorage.NameNodeDirType
type)
{
IList <FilePath> ret = Lists.NewArrayList();
foreach (Storage.StorageDirectory sd in storage.DirIterable(type))
{
ret.AddItem(sd.GetCurrentDir());
}
return(ret);
}
/// <summary>
/// Iterate over all the storage directories, checking if it should be
/// formatted.
/// </summary>
/// <remarks>
/// Iterate over all the storage directories, checking if it should be
/// formatted. Format the storage if necessary and allowed by the user.
/// </remarks>
/// <returns>True if formatting is processed</returns>
/// <exception cref="System.IO.IOException"/>
private bool Format(NNStorage storage, NamespaceInfo nsInfo)
{
// Check with the user before blowing away data.
if (!Storage.ConfirmFormat(storage.DirIterable(null), force, interactive))
{
storage.Close();
return(false);
}
else
{
// Format the storage (writes VERSION file)
storage.Format(nsInfo);
return(true);
}
}
public static void LogStorageContents(Log Log, NNStorage storage)
{
Log.Info("current storages and corresponding sizes:");
foreach (Storage.StorageDirectory sd in storage.DirIterable(null))
{
FilePath curDir = sd.GetCurrentDir();
Log.Info("In directory " + curDir);
FilePath[] files = curDir.ListFiles();
Arrays.Sort(files);
foreach (FilePath f in files)
{
Log.Info(" file " + f.GetAbsolutePath() + "; len = " + f.Length());
}
}
}
public virtual void TestNormalOperation()
{
FilePath f1 = new FilePath(TestEditLog.TestDir + "/normtest0");
FilePath f2 = new FilePath(TestEditLog.TestDir + "/normtest1");
FilePath f3 = new FilePath(TestEditLog.TestDir + "/normtest2");
IList <URI> editUris = ImmutableList.Of(f1.ToURI(), f2.ToURI(), f3.ToURI());
NNStorage storage = TestEditLog.SetupEdits(editUris, 5);
long numJournals = 0;
foreach (Storage.StorageDirectory sd in storage.DirIterable(NNStorage.NameNodeDirType
.Edits))
{
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(6 * TestEditLog.TxnsPerRoll, GetNumberOfTransactions
(jm, 1, true, false));
numJournals++;
}
NUnit.Framework.Assert.AreEqual(3, numJournals);
}
/// <exception cref="System.Exception"/>
public virtual void TestCancelSaveNamespace()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
// Replace the FSImage with a spy
FSImage image = fsn.GetFSImage();
NNStorage storage = image.GetStorage();
storage.Close();
// unlock any directories that FSNamesystem's initialization may have locked
storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem.GetNamespaceEditsDirs
(conf));
FSNamesystem spyFsn = Org.Mockito.Mockito.Spy(fsn);
FSNamesystem finalFsn = spyFsn;
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
BlockIdManager bid = Org.Mockito.Mockito.Spy(spyFsn.GetBlockIdManager());
Whitebox.SetInternalState(finalFsn, "blockIdManager", bid);
Org.Mockito.Mockito.DoAnswer(delayer).When(bid).GetGenerationStampV2();
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
DoAnEdit(fsn, 1);
Canceler canceler = new Canceler();
// Save namespace
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
Future <Void> saverFuture = pool.Submit(new _Callable_561(image, finalFsn, canceler
));
// Wait until saveNamespace calls getGenerationStamp
delayer.WaitForCall();
// then cancel the saveNamespace
Future <Void> cancelFuture = pool.Submit(new _Callable_572(canceler));
// give the cancel call time to run
Sharpen.Thread.Sleep(500);
// allow saveNamespace to proceed - it should check the cancel flag after
// this point and throw an exception
delayer.Proceed();
cancelFuture.Get();
saverFuture.Get();
NUnit.Framework.Assert.Fail("saveNamespace did not fail even though cancelled!");
}
catch (Exception t)
{
GenericTestUtils.AssertExceptionContains("SaveNamespaceCancelledException", t);
}
Log.Info("Successfully cancelled a saveNamespace");
// Check that we have only the original image and not any
// cruft left over from half-finished images
FSImageTestUtil.LogStorageContents(Log, storage);
foreach (Storage.StorageDirectory sd in storage.DirIterable(null))
{
FilePath curDir = sd.GetCurrentDir();
GenericTestUtils.AssertGlobEquals(curDir, "fsimage_.*", NNStorage.GetImageFileName
(0), NNStorage.GetImageFileName(0) + MD5FileUtils.Md5Suffix);
}
}
finally
{
fsn.Close();
}
}