示例#1
0
        /// <summary>
        /// Assert that all of the given directories have the same newest filename
        /// for fsimage that they hold the same data.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public static void AssertSameNewestImage(IList <FilePath> dirs)
        {
            if (dirs.Count < 2)
            {
                return;
            }
            long             imageTxId  = -1;
            IList <FilePath> imageFiles = new AList <FilePath>();

            foreach (FilePath dir in dirs)
            {
                FSImageTransactionalStorageInspector inspector = InspectStorageDirectory(dir, NNStorage.NameNodeDirType
                                                                                         .Image);
                IList <FSImageStorageInspector.FSImageFile> latestImages = inspector.GetLatestImages
                                                                               ();
                System.Diagnostics.Debug.Assert((!latestImages.IsEmpty()));
                long thisTxId = latestImages[0].GetCheckpointTxId();
                if (imageTxId != -1 && thisTxId != imageTxId)
                {
                    NUnit.Framework.Assert.Fail("Storage directory " + dir + " does not have the same "
                                                + "last image index " + imageTxId + " as another");
                }
                imageTxId = thisTxId;
                imageFiles.AddItem(inspector.GetLatestImages()[0].GetFile());
            }
            AssertFileContentsSame(Sharpen.Collections.ToArray(imageFiles, new FilePath[0]));
        }
        /// <exception cref="System.IO.IOException"/>
        internal virtual void CheckImageAndEditsFilesExistence(FilePath dir, bool shouldHaveImages
                                                               , bool shouldHaveEdits)
        {
            FSImageTransactionalStorageInspector ins = Inspect(dir);

            if (shouldHaveImages)
            {
                NUnit.Framework.Assert.IsTrue("Expect images in " + dir, ins.foundImages.Count >
                                              0);
            }
            else
            {
                NUnit.Framework.Assert.IsTrue("Expect no images in " + dir, ins.foundImages.IsEmpty
                                                  ());
            }
            IList <FileJournalManager.EditLogFile> editlogs = FileJournalManager.MatchEditLogs
                                                                  (new FilePath(dir, "current").ListFiles());

            if (shouldHaveEdits)
            {
                NUnit.Framework.Assert.IsTrue("Expect edits in " + dir, editlogs.Count > 0);
            }
            else
            {
                NUnit.Framework.Assert.IsTrue("Expect no edits in " + dir, editlogs.IsEmpty());
            }
        }
示例#3
0
        /// <returns>
        /// the fsimage file with the most recent transaction ID in the
        /// given storage directory.
        /// </returns>
        /// <exception cref="System.IO.IOException"/>
        public static FilePath FindLatestImageFile(Storage.StorageDirectory sd)
        {
            FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                                 ();

            inspector.InspectDirectory(sd);
            return(inspector.GetLatestImages()[0].GetFile());
        }
示例#4
0
        /// <exception cref="System.IO.IOException"/>
        public static FSImageTransactionalStorageInspector InspectStorageDirectory(FilePath
                                                                                   dir, NNStorage.NameNodeDirType dirType)
        {
            FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                                 ();

            inspector.InspectDirectory(MockStorageDirectory(dir, dirType));
            return(inspector);
        }
示例#5
0
 private void PurgeCheckpointsOlderThan(FSImageTransactionalStorageInspector inspector
                                        , long minTxId)
 {
     foreach (FSImageStorageInspector.FSImageFile image in inspector.GetFoundImages())
     {
         if (image.GetCheckpointTxId() < minTxId)
         {
             purger.PurgeImage(image);
         }
     }
 }
示例#6
0
        /// <returns>
        /// the fsimage file with the most recent transaction ID in the
        /// given 'current/' directory.
        /// </returns>
        /// <exception cref="System.IO.IOException"/>
        public static FilePath FindNewestImageFile(string currentDirPath)
        {
            Storage.StorageDirectory sd = FSImageTestUtil.MockStorageDirectory(new FilePath(currentDirPath
                                                                                            ), NNStorage.NameNodeDirType.Image);
            FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                                 ();

            inspector.InspectDirectory(sd);
            IList <FSImageStorageInspector.FSImageFile> latestImages = inspector.GetLatestImages
                                                                           ();

            return((latestImages.IsEmpty()) ? null : latestImages[0].GetFile());
        }
示例#7
0
        /// <exception cref="System.IO.IOException"/>
        internal virtual void PurgeOldStorage(NNStorage.NameNodeFile nnf)
        {
            FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                                 (EnumSet.Of(nnf));

            storage.InspectStorageDirs(inspector);
            long minImageTxId = GetImageTxIdToRetain(inspector);

            PurgeCheckpointsOlderThan(inspector, minImageTxId);
            if (nnf == NNStorage.NameNodeFile.ImageRollback)
            {
                // do not purge edits for IMAGE_ROLLBACK.
                return;
            }
            // If fsimage_N is the image we want to keep, then we need to keep
            // all txns > N. We can remove anything < N+1, since fsimage_N
            // reflects the state up to and including N. However, we also
            // provide a "cushion" of older txns that we keep, which is
            // handy for HA, where a remote node may not have as many
            // new images.
            //
            // First, determine the target number of extra transactions to retain based
            // on the configured amount.
            long minimumRequiredTxId            = minImageTxId + 1;
            long purgeLogsFrom                  = Math.Max(0, minimumRequiredTxId - numExtraEditsToRetain);
            AList <EditLogInputStream> editLogs = new AList <EditLogInputStream>();

            purgeableLogs.SelectInputStreams(editLogs, purgeLogsFrom, false);
            editLogs.Sort(new _IComparer_138());
            // Remove from consideration any edit logs that are in fact required.
            while (editLogs.Count > 0 && editLogs[editLogs.Count - 1].GetFirstTxId() >= minimumRequiredTxId
                   )
            {
                editLogs.Remove(editLogs.Count - 1);
            }
            // Next, adjust the number of transactions to retain if doing so would mean
            // keeping too many segments around.
            while (editLogs.Count > maxExtraEditsSegmentsToRetain)
            {
                purgeLogsFrom = editLogs[0].GetLastTxId() + 1;
                editLogs.Remove(0);
            }
            // Finally, ensure that we're not trying to purge any transactions that we
            // actually need.
            if (purgeLogsFrom > minimumRequiredTxId)
            {
                throw new Exception("Should not purge more edits than required to " + "restore: "
                                    + purgeLogsFrom + " should be <= " + minimumRequiredTxId);
            }
            purgeableLogs.PurgeLogsOlderThan(purgeLogsFrom);
        }
示例#8
0
        /// <exception cref="System.IO.IOException"/>
        internal virtual void PurgeCheckpoinsAfter(NNStorage.NameNodeFile nnf, long fromTxId
                                                   )
        {
            FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                                 (EnumSet.Of(nnf));

            storage.InspectStorageDirs(inspector);
            foreach (FSImageStorageInspector.FSImageFile image in inspector.GetFoundImages())
            {
                if (image.GetCheckpointTxId() > fromTxId)
                {
                    purger.PurgeImage(image);
                }
            }
        }
示例#9
0
        public virtual void TestCurrentStorageInspector()
        {
            FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                                 ();

            Storage.StorageDirectory mockDir = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType
                                                                                    .ImageAndEdits, false, "/foo/current/" + NNStorage.GetImageFileName(123), "/foo/current/"
                                                                                    + NNStorage.GetFinalizedEditsFileName(123, 456), "/foo/current/" + NNStorage.GetImageFileName
                                                                                        (456), "/foo/current/" + NNStorage.GetInProgressEditsFileName(457));
            inspector.InspectDirectory(mockDir);
            NUnit.Framework.Assert.AreEqual(2, inspector.foundImages.Count);
            FSImageStorageInspector.FSImageFile latestImage = inspector.GetLatestImages()[0];
            NUnit.Framework.Assert.AreEqual(456, latestImage.txId);
            NUnit.Framework.Assert.AreSame(mockDir, latestImage.sd);
            NUnit.Framework.Assert.IsTrue(inspector.IsUpgradeFinalized());
            NUnit.Framework.Assert.AreEqual(new FilePath("/foo/current/" + NNStorage.GetImageFileName
                                                             (456)), latestImage.GetFile());
        }
示例#10
0
        /// <param name="inspector">inspector that has already inspected all storage dirs</param>
        /// <returns>
        /// the transaction ID corresponding to the oldest checkpoint
        /// that should be retained.
        /// </returns>
        private long GetImageTxIdToRetain(FSImageTransactionalStorageInspector inspector)
        {
            IList <FSImageStorageInspector.FSImageFile> images = inspector.GetFoundImages();
            TreeSet <long> imageTxIds = Sets.NewTreeSet();

            foreach (FSImageStorageInspector.FSImageFile image in images)
            {
                imageTxIds.AddItem(image.GetCheckpointTxId());
            }
            IList <long> imageTxIdsList = Lists.NewArrayList(imageTxIds);

            if (imageTxIdsList.IsEmpty())
            {
                return(0);
            }
            Sharpen.Collections.Reverse(imageTxIdsList);
            int  toRetain = Math.Min(numCheckpointsToRetain, imageTxIdsList.Count);
            long minTxId  = imageTxIdsList[toRetain - 1];

            Log.Info("Going to retain " + toRetain + " images with txid >= " + minTxId);
            return(minTxId);
        }
示例#11
0
 /// <exception cref="System.IO.IOException"/>
 private bool TryConvergeJournalSpool()
 {
     Preconditions.CheckState(bnState == BackupImage.BNState.JournalOnly, "bad state: %s"
                              , bnState);
     // This section is unsynchronized so we can continue to apply
     // ahead of where we're reading, concurrently. Since the state
     // is JOURNAL_ONLY at this point, we know that lastAppliedTxId
     // doesn't change, and curSegmentTxId only increases
     while (lastAppliedTxId < editLog.GetCurSegmentTxId() - 1)
     {
         long target = editLog.GetCurSegmentTxId();
         Log.Info("Loading edits into backupnode to try to catch up from txid " + lastAppliedTxId
                  + " to " + target);
         FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                              ();
         storage.InspectStorageDirs(inspector);
         editLog.RecoverUnclosedStreams();
         IEnumerable <EditLogInputStream> editStreamsAll = editLog.SelectInputStreams(lastAppliedTxId
                                                                                      , target - 1);
         // remove inprogress
         IList <EditLogInputStream> editStreams = Lists.NewArrayList();
         foreach (EditLogInputStream s in editStreamsAll)
         {
             if (s.GetFirstTxId() != editLog.GetCurSegmentTxId())
             {
                 editStreams.AddItem(s);
             }
         }
         LoadEdits(editStreams, GetNamesystem());
     }
     // now, need to load the in-progress file
     lock (this)
     {
         if (lastAppliedTxId != editLog.GetCurSegmentTxId() - 1)
         {
             Log.Debug("Logs rolled while catching up to current segment");
             return(false);
         }
         // drop lock and try again to load local logs
         EditLogInputStream stream = null;
         ICollection <EditLogInputStream> editStreams = GetEditLog().SelectInputStreams(GetEditLog
                                                                                            ().GetCurSegmentTxId(), GetEditLog().GetCurSegmentTxId());
         foreach (EditLogInputStream s in editStreams)
         {
             if (s.GetFirstTxId() == GetEditLog().GetCurSegmentTxId())
             {
                 stream = s;
             }
             break;
         }
         if (stream == null)
         {
             Log.Warn("Unable to find stream starting with " + editLog.GetCurSegmentTxId() + ". This indicates that there is an error in synchronization in BackupImage"
                      );
             return(false);
         }
         try
         {
             long remainingTxns = GetEditLog().GetLastWrittenTxId() - lastAppliedTxId;
             Log.Info("Going to finish converging with remaining " + remainingTxns + " txns from in-progress stream "
                      + stream);
             FSEditLogLoader loader = new FSEditLogLoader(GetNamesystem(), lastAppliedTxId);
             loader.LoadFSEdits(stream, lastAppliedTxId + 1);
             lastAppliedTxId = loader.GetLastAppliedTxId();
             System.Diagnostics.Debug.Assert(lastAppliedTxId == GetEditLog().GetLastWrittenTxId
                                                 ());
         }
         finally
         {
             FSEditLog.CloseAllStreams(editStreams);
         }
         Log.Info("Successfully synced BackupNode with NameNode at txnid " + lastAppliedTxId
                  );
         SetState(BackupImage.BNState.InSync);
     }
     return(true);
 }