/// <summary> /// Assert that all of the given directories have the same newest filename /// for fsimage that they hold the same data. /// </summary> /// <exception cref="System.Exception"/> public static void AssertSameNewestImage(IList <FilePath> dirs) { if (dirs.Count < 2) { return; } long imageTxId = -1; IList <FilePath> imageFiles = new AList <FilePath>(); foreach (FilePath dir in dirs) { FSImageTransactionalStorageInspector inspector = InspectStorageDirectory(dir, NNStorage.NameNodeDirType .Image); IList <FSImageStorageInspector.FSImageFile> latestImages = inspector.GetLatestImages (); System.Diagnostics.Debug.Assert((!latestImages.IsEmpty())); long thisTxId = latestImages[0].GetCheckpointTxId(); if (imageTxId != -1 && thisTxId != imageTxId) { NUnit.Framework.Assert.Fail("Storage directory " + dir + " does not have the same " + "last image index " + imageTxId + " as another"); } imageTxId = thisTxId; imageFiles.AddItem(inspector.GetLatestImages()[0].GetFile()); } AssertFileContentsSame(Sharpen.Collections.ToArray(imageFiles, new FilePath[0])); }
/// <exception cref="System.IO.IOException"/> internal virtual void CheckImageAndEditsFilesExistence(FilePath dir, bool shouldHaveImages , bool shouldHaveEdits) { FSImageTransactionalStorageInspector ins = Inspect(dir); if (shouldHaveImages) { NUnit.Framework.Assert.IsTrue("Expect images in " + dir, ins.foundImages.Count > 0); } else { NUnit.Framework.Assert.IsTrue("Expect no images in " + dir, ins.foundImages.IsEmpty ()); } IList <FileJournalManager.EditLogFile> editlogs = FileJournalManager.MatchEditLogs (new FilePath(dir, "current").ListFiles()); if (shouldHaveEdits) { NUnit.Framework.Assert.IsTrue("Expect edits in " + dir, editlogs.Count > 0); } else { NUnit.Framework.Assert.IsTrue("Expect no edits in " + dir, editlogs.IsEmpty()); } }
/// <returns> /// the fsimage file with the most recent transaction ID in the /// given storage directory. /// </returns> /// <exception cref="System.IO.IOException"/> public static FilePath FindLatestImageFile(Storage.StorageDirectory sd) { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); inspector.InspectDirectory(sd); return(inspector.GetLatestImages()[0].GetFile()); }
/// <exception cref="System.IO.IOException"/> public static FSImageTransactionalStorageInspector InspectStorageDirectory(FilePath dir, NNStorage.NameNodeDirType dirType) { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); inspector.InspectDirectory(MockStorageDirectory(dir, dirType)); return(inspector); }
private void PurgeCheckpointsOlderThan(FSImageTransactionalStorageInspector inspector , long minTxId) { foreach (FSImageStorageInspector.FSImageFile image in inspector.GetFoundImages()) { if (image.GetCheckpointTxId() < minTxId) { purger.PurgeImage(image); } } }
/// <returns> /// the fsimage file with the most recent transaction ID in the /// given 'current/' directory. /// </returns> /// <exception cref="System.IO.IOException"/> public static FilePath FindNewestImageFile(string currentDirPath) { Storage.StorageDirectory sd = FSImageTestUtil.MockStorageDirectory(new FilePath(currentDirPath ), NNStorage.NameNodeDirType.Image); FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); inspector.InspectDirectory(sd); IList <FSImageStorageInspector.FSImageFile> latestImages = inspector.GetLatestImages (); return((latestImages.IsEmpty()) ? null : latestImages[0].GetFile()); }
/// <exception cref="System.IO.IOException"/> internal virtual void PurgeOldStorage(NNStorage.NameNodeFile nnf) { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (EnumSet.Of(nnf)); storage.InspectStorageDirs(inspector); long minImageTxId = GetImageTxIdToRetain(inspector); PurgeCheckpointsOlderThan(inspector, minImageTxId); if (nnf == NNStorage.NameNodeFile.ImageRollback) { // do not purge edits for IMAGE_ROLLBACK. return; } // If fsimage_N is the image we want to keep, then we need to keep // all txns > N. We can remove anything < N+1, since fsimage_N // reflects the state up to and including N. However, we also // provide a "cushion" of older txns that we keep, which is // handy for HA, where a remote node may not have as many // new images. // // First, determine the target number of extra transactions to retain based // on the configured amount. long minimumRequiredTxId = minImageTxId + 1; long purgeLogsFrom = Math.Max(0, minimumRequiredTxId - numExtraEditsToRetain); AList <EditLogInputStream> editLogs = new AList <EditLogInputStream>(); purgeableLogs.SelectInputStreams(editLogs, purgeLogsFrom, false); editLogs.Sort(new _IComparer_138()); // Remove from consideration any edit logs that are in fact required. while (editLogs.Count > 0 && editLogs[editLogs.Count - 1].GetFirstTxId() >= minimumRequiredTxId ) { editLogs.Remove(editLogs.Count - 1); } // Next, adjust the number of transactions to retain if doing so would mean // keeping too many segments around. while (editLogs.Count > maxExtraEditsSegmentsToRetain) { purgeLogsFrom = editLogs[0].GetLastTxId() + 1; editLogs.Remove(0); } // Finally, ensure that we're not trying to purge any transactions that we // actually need. if (purgeLogsFrom > minimumRequiredTxId) { throw new Exception("Should not purge more edits than required to " + "restore: " + purgeLogsFrom + " should be <= " + minimumRequiredTxId); } purgeableLogs.PurgeLogsOlderThan(purgeLogsFrom); }
/// <exception cref="System.IO.IOException"/> internal virtual void PurgeCheckpoinsAfter(NNStorage.NameNodeFile nnf, long fromTxId ) { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (EnumSet.Of(nnf)); storage.InspectStorageDirs(inspector); foreach (FSImageStorageInspector.FSImageFile image in inspector.GetFoundImages()) { if (image.GetCheckpointTxId() > fromTxId) { purger.PurgeImage(image); } } }
public virtual void TestCurrentStorageInspector() { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); Storage.StorageDirectory mockDir = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType .ImageAndEdits, false, "/foo/current/" + NNStorage.GetImageFileName(123), "/foo/current/" + NNStorage.GetFinalizedEditsFileName(123, 456), "/foo/current/" + NNStorage.GetImageFileName (456), "/foo/current/" + NNStorage.GetInProgressEditsFileName(457)); inspector.InspectDirectory(mockDir); NUnit.Framework.Assert.AreEqual(2, inspector.foundImages.Count); FSImageStorageInspector.FSImageFile latestImage = inspector.GetLatestImages()[0]; NUnit.Framework.Assert.AreEqual(456, latestImage.txId); NUnit.Framework.Assert.AreSame(mockDir, latestImage.sd); NUnit.Framework.Assert.IsTrue(inspector.IsUpgradeFinalized()); NUnit.Framework.Assert.AreEqual(new FilePath("/foo/current/" + NNStorage.GetImageFileName (456)), latestImage.GetFile()); }
/// <param name="inspector">inspector that has already inspected all storage dirs</param> /// <returns> /// the transaction ID corresponding to the oldest checkpoint /// that should be retained. /// </returns> private long GetImageTxIdToRetain(FSImageTransactionalStorageInspector inspector) { IList <FSImageStorageInspector.FSImageFile> images = inspector.GetFoundImages(); TreeSet <long> imageTxIds = Sets.NewTreeSet(); foreach (FSImageStorageInspector.FSImageFile image in images) { imageTxIds.AddItem(image.GetCheckpointTxId()); } IList <long> imageTxIdsList = Lists.NewArrayList(imageTxIds); if (imageTxIdsList.IsEmpty()) { return(0); } Sharpen.Collections.Reverse(imageTxIdsList); int toRetain = Math.Min(numCheckpointsToRetain, imageTxIdsList.Count); long minTxId = imageTxIdsList[toRetain - 1]; Log.Info("Going to retain " + toRetain + " images with txid >= " + minTxId); return(minTxId); }
/// <exception cref="System.IO.IOException"/> private bool TryConvergeJournalSpool() { Preconditions.CheckState(bnState == BackupImage.BNState.JournalOnly, "bad state: %s" , bnState); // This section is unsynchronized so we can continue to apply // ahead of where we're reading, concurrently. Since the state // is JOURNAL_ONLY at this point, we know that lastAppliedTxId // doesn't change, and curSegmentTxId only increases while (lastAppliedTxId < editLog.GetCurSegmentTxId() - 1) { long target = editLog.GetCurSegmentTxId(); Log.Info("Loading edits into backupnode to try to catch up from txid " + lastAppliedTxId + " to " + target); FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); storage.InspectStorageDirs(inspector); editLog.RecoverUnclosedStreams(); IEnumerable <EditLogInputStream> editStreamsAll = editLog.SelectInputStreams(lastAppliedTxId , target - 1); // remove inprogress IList <EditLogInputStream> editStreams = Lists.NewArrayList(); foreach (EditLogInputStream s in editStreamsAll) { if (s.GetFirstTxId() != editLog.GetCurSegmentTxId()) { editStreams.AddItem(s); } } LoadEdits(editStreams, GetNamesystem()); } // now, need to load the in-progress file lock (this) { if (lastAppliedTxId != editLog.GetCurSegmentTxId() - 1) { Log.Debug("Logs rolled while catching up to current segment"); return(false); } // drop lock and try again to load local logs EditLogInputStream stream = null; ICollection <EditLogInputStream> editStreams = GetEditLog().SelectInputStreams(GetEditLog ().GetCurSegmentTxId(), GetEditLog().GetCurSegmentTxId()); foreach (EditLogInputStream s in editStreams) { if (s.GetFirstTxId() == GetEditLog().GetCurSegmentTxId()) { stream = s; } break; } if (stream == null) { Log.Warn("Unable to find stream starting with " + editLog.GetCurSegmentTxId() + ". This indicates that there is an error in synchronization in BackupImage" ); return(false); } try { long remainingTxns = GetEditLog().GetLastWrittenTxId() - lastAppliedTxId; Log.Info("Going to finish converging with remaining " + remainingTxns + " txns from in-progress stream " + stream); FSEditLogLoader loader = new FSEditLogLoader(GetNamesystem(), lastAppliedTxId); loader.LoadFSEdits(stream, lastAppliedTxId + 1); lastAppliedTxId = loader.GetLastAppliedTxId(); System.Diagnostics.Debug.Assert(lastAppliedTxId == GetEditLog().GetLastWrittenTxId ()); } finally { FSEditLog.CloseAllStreams(editStreams); } Log.Info("Successfully synced BackupNode with NameNode at txnid " + lastAppliedTxId ); SetState(BackupImage.BNState.InSync); } return(true); }