/// <returns> /// the image files that have the most recent associated /// transaction IDs. If there are multiple storage directories which /// contain equal images, we'll return them all. /// </returns> /// <exception cref="System.IO.FileNotFoundException">if not images are found.</exception> /// <exception cref="System.IO.IOException"/> internal override IList <FSImageStorageInspector.FSImageFile> GetLatestImages() { List <FSImageStorageInspector.FSImageFile> ret = new List <FSImageStorageInspector.FSImageFile >(); foreach (FSImageStorageInspector.FSImageFile img in foundImages) { if (ret.IsEmpty()) { ret.AddItem(img); } else { FSImageStorageInspector.FSImageFile cur = ret.GetFirst(); if (cur.txId == img.txId) { ret.AddItem(img); } else { if (cur.txId < img.txId) { ret.Clear(); ret.AddItem(img); } } } } if (ret.IsEmpty()) { throw new FileNotFoundException("No valid image files found"); } return(ret); }
/// <exception cref="System.IO.IOException"/> internal override IList <FSImageStorageInspector.FSImageFile> GetLatestImages() { // We should have at least one image and one edits dirs if (latestNameSD == null) { throw new IOException("Image file is not found in " + imageDirs); } if (latestEditsSD == null) { throw new IOException("Edits file is not found in " + editsDirs); } // Make sure we are loading image and edits from same checkpoint if (latestNameCheckpointTime > latestEditsCheckpointTime && latestNameSD != latestEditsSD && latestNameSD.GetStorageDirType() == NNStorage.NameNodeDirType.Image && latestEditsSD .GetStorageDirType() == NNStorage.NameNodeDirType.Edits) { // This is a rare failure when NN has image-only and edits-only // storage directories, and fails right after saving images, // in some of the storage directories, but before purging edits. // See -NOTE- in saveNamespace(). Log.Error("This is a rare failure scenario!!!"); Log.Error("Image checkpoint time " + latestNameCheckpointTime + " > edits checkpoint time " + latestEditsCheckpointTime); Log.Error("Name-node will treat the image as the latest state of " + "the namespace. Old edits will be discarded." ); } else { if (latestNameCheckpointTime != latestEditsCheckpointTime) { throw new IOException("Inconsistent storage detected, " + "image and edits checkpoint times do not match. " + "image checkpoint time = " + latestNameCheckpointTime + "edits checkpoint time = " + latestEditsCheckpointTime); } } needToSaveAfterRecovery = DoRecovery(); FSImageStorageInspector.FSImageFile file = new FSImageStorageInspector.FSImageFile (latestNameSD, NNStorage.GetStorageFile(latestNameSD, NNStorage.NameNodeFile.Image ), HdfsConstants.InvalidTxid); List <FSImageStorageInspector.FSImageFile> ret = new List <FSImageStorageInspector.FSImageFile >(); ret.AddItem(file); return(ret); }
public virtual void TestCurrentStorageInspector() { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); Storage.StorageDirectory mockDir = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType .ImageAndEdits, false, "/foo/current/" + NNStorage.GetImageFileName(123), "/foo/current/" + NNStorage.GetFinalizedEditsFileName(123, 456), "/foo/current/" + NNStorage.GetImageFileName (456), "/foo/current/" + NNStorage.GetInProgressEditsFileName(457)); inspector.InspectDirectory(mockDir); NUnit.Framework.Assert.AreEqual(2, inspector.foundImages.Count); FSImageStorageInspector.FSImageFile latestImage = inspector.GetLatestImages()[0]; NUnit.Framework.Assert.AreEqual(456, latestImage.txId); NUnit.Framework.Assert.AreSame(mockDir, latestImage.sd); NUnit.Framework.Assert.IsTrue(inspector.IsUpgradeFinalized()); NUnit.Framework.Assert.AreEqual(new FilePath("/foo/current/" + NNStorage.GetImageFileName (456)), latestImage.GetFile()); }
public virtual void PurgeImage(FSImageStorageInspector.FSImageFile image) { Log.Info("Purging old image " + image); DeleteOrWarn(image.GetFile()); DeleteOrWarn(MD5FileUtils.GetDigestFileForFile(image.GetFile())); }