/// <exception cref="System.IO.IOException"/> internal static void DownloadEditsToStorage(Uri fsName, RemoteEditLog log, NNStorage dstStorage) { System.Diagnostics.Debug.Assert(log.GetStartTxId() > 0 && log.GetEndTxId() > 0, "bad log: " + log); string fileid = ImageServlet.GetParamStringForLog(log, dstStorage); string finalFileName = NNStorage.GetFinalizedEditsFileName(log.GetStartTxId(), log .GetEndTxId()); IList <FilePath> finalFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, finalFileName); System.Diagnostics.Debug.Assert(!finalFiles.IsEmpty(), "No checkpoint targets."); foreach (FilePath f in finalFiles) { if (f.Exists() && FileUtil.CanRead(f)) { Log.Info("Skipping download of remote edit log " + log + " since it already is stored locally at " + f); return; } else { if (Log.IsDebugEnabled()) { Log.Debug("Dest file: " + f); } } } long milliTime = Time.MonotonicNow(); string tmpFileName = NNStorage.GetTemporaryEditsFileName(log.GetStartTxId(), log. GetEndTxId(), milliTime); IList <FilePath> tmpFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, tmpFileName ); GetFileClient(fsName, fileid, tmpFiles, dstStorage, false); Log.Info("Downloaded file " + tmpFiles[0].GetName() + " size " + finalFiles[0].Length () + " bytes."); CheckpointFaultInjector.GetInstance().BeforeEditsRename(); foreach (Storage.StorageDirectory sd in dstStorage.DirIterable(NNStorage.NameNodeDirType .Edits)) { FilePath tmpFile = NNStorage.GetTemporaryEditsFile(sd, log.GetStartTxId(), log.GetEndTxId (), milliTime); FilePath finalizedFile = NNStorage.GetFinalizedEditsFile(sd, log.GetStartTxId(), log.GetEndTxId()); if (Log.IsDebugEnabled()) { Log.Debug("Renaming " + tmpFile + " to " + finalizedFile); } bool success = tmpFile.RenameTo(finalizedFile); if (!success) { Log.Warn("Unable to rename edits file from " + tmpFile + " to " + finalizedFile); } } }
/// <returns> /// a List which contains the "current" dir for each storage /// directory of the given type. /// </returns> public static IList <FilePath> GetCurrentDirs(NNStorage storage, NNStorage.NameNodeDirType type) { IList <FilePath> ret = Lists.NewArrayList(); foreach (Storage.StorageDirectory sd in storage.DirIterable(type)) { ret.AddItem(sd.GetCurrentDir()); } return(ret); }
/// <summary> /// Iterate over all the storage directories, checking if it should be /// formatted. /// </summary> /// <remarks> /// Iterate over all the storage directories, checking if it should be /// formatted. Format the storage if necessary and allowed by the user. /// </remarks> /// <returns>True if formatting is processed</returns> /// <exception cref="System.IO.IOException"/> private bool Format(NNStorage storage, NamespaceInfo nsInfo) { // Check with the user before blowing away data. if (!Storage.ConfirmFormat(storage.DirIterable(null), force, interactive)) { storage.Close(); return(false); } else { // Format the storage (writes VERSION file) storage.Format(nsInfo); return(true); } }
public static void LogStorageContents(Log Log, NNStorage storage) { Log.Info("current storages and corresponding sizes:"); foreach (Storage.StorageDirectory sd in storage.DirIterable(null)) { FilePath curDir = sd.GetCurrentDir(); Log.Info("In directory " + curDir); FilePath[] files = curDir.ListFiles(); Arrays.Sort(files); foreach (FilePath f in files) { Log.Info(" file " + f.GetAbsolutePath() + "; len = " + f.Length()); } } }
public virtual void TestNormalOperation() { FilePath f1 = new FilePath(TestEditLog.TestDir + "/normtest0"); FilePath f2 = new FilePath(TestEditLog.TestDir + "/normtest1"); FilePath f3 = new FilePath(TestEditLog.TestDir + "/normtest2"); IList <URI> editUris = ImmutableList.Of(f1.ToURI(), f2.ToURI(), f3.ToURI()); NNStorage storage = TestEditLog.SetupEdits(editUris, 5); long numJournals = 0; foreach (Storage.StorageDirectory sd in storage.DirIterable(NNStorage.NameNodeDirType .Edits)) { FileJournalManager jm = new FileJournalManager(conf, sd, storage); NUnit.Framework.Assert.AreEqual(6 * TestEditLog.TxnsPerRoll, GetNumberOfTransactions (jm, 1, true, false)); numJournals++; } NUnit.Framework.Assert.AreEqual(3, numJournals); }
/// <exception cref="System.Exception"/> public virtual void TestCancelSaveNamespace() { Configuration conf = GetConf(); NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode); DFSTestUtil.FormatNameNode(conf); FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf); // Replace the FSImage with a spy FSImage image = fsn.GetFSImage(); NNStorage storage = image.GetStorage(); storage.Close(); // unlock any directories that FSNamesystem's initialization may have locked storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem.GetNamespaceEditsDirs (conf)); FSNamesystem spyFsn = Org.Mockito.Mockito.Spy(fsn); FSNamesystem finalFsn = spyFsn; GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log); BlockIdManager bid = Org.Mockito.Mockito.Spy(spyFsn.GetBlockIdManager()); Whitebox.SetInternalState(finalFsn, "blockIdManager", bid); Org.Mockito.Mockito.DoAnswer(delayer).When(bid).GetGenerationStampV2(); ExecutorService pool = Executors.NewFixedThreadPool(2); try { DoAnEdit(fsn, 1); Canceler canceler = new Canceler(); // Save namespace fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); try { Future <Void> saverFuture = pool.Submit(new _Callable_561(image, finalFsn, canceler )); // Wait until saveNamespace calls getGenerationStamp delayer.WaitForCall(); // then cancel the saveNamespace Future <Void> cancelFuture = pool.Submit(new _Callable_572(canceler)); // give the cancel call time to run Sharpen.Thread.Sleep(500); // allow saveNamespace to proceed - it should check the cancel flag after // this point and throw an exception delayer.Proceed(); cancelFuture.Get(); saverFuture.Get(); NUnit.Framework.Assert.Fail("saveNamespace did not fail even though cancelled!"); } catch (Exception t) { GenericTestUtils.AssertExceptionContains("SaveNamespaceCancelledException", t); } Log.Info("Successfully cancelled a saveNamespace"); // Check that we have only the original image and not any // cruft left over from half-finished images FSImageTestUtil.LogStorageContents(Log, storage); foreach (Storage.StorageDirectory sd in storage.DirIterable(null)) { FilePath curDir = sd.GetCurrentDir(); GenericTestUtils.AssertGlobEquals(curDir, "fsimage_.*", NNStorage.GetImageFileName (0), NNStorage.GetImageFileName(0) + MD5FileUtils.Md5Suffix); } } finally { fsn.Close(); } }