public virtual void TestDoPreUpgradeIOError() { FilePath storageDir = new FilePath(TestEditLog.TestDir, "preupgradeioerror"); IList <URI> editUris = Sharpen.Collections.SingletonList(storageDir.ToURI()); NNStorage storage = TestEditLog.SetupEdits(editUris, 5); Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits ).Next(); NUnit.Framework.Assert.IsNotNull(sd); // Change storage directory so that renaming current to previous.tmp fails. FileUtil.SetWritable(storageDir, false); FileJournalManager jm = null; try { jm = new FileJournalManager(conf, sd, storage); exception.Expect(typeof(IOException)); if (NativeCodeLoader.IsNativeCodeLoaded()) { exception.ExpectMessage("failure in native rename"); } jm.DoPreUpgrade(); } finally { IOUtils.Cleanup(Log, jm); // Restore permissions on storage directory and make sure we can delete. FileUtil.SetWritable(storageDir, true); FileUtil.FullyDelete(storageDir); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.TimeoutException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/> public virtual void TestAddBackRemovedVolume() { StartDFSCluster(1, 2); // Create some data on every volume. CreateFile(new Path("/test"), 32); DataNode dn = cluster.GetDataNodes()[0]; Configuration conf = dn.GetConf(); string oldDataDir = conf.Get(DFSConfigKeys.DfsDatanodeDataDirKey); string keepDataDir = oldDataDir.Split(",")[0]; string removeDataDir = oldDataDir.Split(",")[1]; dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, keepDataDir); for (int i = 0; i < cluster.GetNumNameNodes(); i++) { string bpid = cluster.GetNamesystem(i).GetBlockPoolId(); BlockPoolSliceStorage bpsStorage = dn.GetStorage().GetBPStorage(bpid); // Make sure that there is no block pool level storage under removeDataDir. for (int j = 0; j < bpsStorage.GetNumStorageDirs(); j++) { Storage.StorageDirectory sd = bpsStorage.GetStorageDir(j); NUnit.Framework.Assert.IsFalse(sd.GetRoot().GetAbsolutePath().StartsWith(new FilePath (removeDataDir).GetAbsolutePath())); } NUnit.Framework.Assert.AreEqual(dn.GetStorage().GetBPStorage(bpid).GetNumStorageDirs (), 1); } // Bring the removed directory back. It only successes if all metadata about // this directory were removed from the previous step. dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, oldDataDir); }
/// <summary>Format a block pool slice storage.</summary> /// <param name="dnCurDir">DataStorage current directory</param> /// <param name="nsInfo">the name space info</param> /// <exception cref="System.IO.IOException">Signals that an I/O exception has occurred. /// </exception> internal virtual void Format(FilePath dnCurDir, NamespaceInfo nsInfo) { FilePath curBpDir = GetBpRoot(nsInfo.GetBlockPoolID(), dnCurDir); Storage.StorageDirectory bpSdir = new Storage.StorageDirectory(curBpDir); Format(bpSdir, nsInfo); }
/// <summary>Check whether the path is a valid DataNode data directory.</summary> private static void CheckDir(FilePath dataDir) { Storage.StorageDirectory sd = new Storage.StorageDirectory(dataDir); NUnit.Framework.Assert.IsTrue(sd.GetRoot().IsDirectory()); NUnit.Framework.Assert.IsTrue(sd.GetCurrentDir().IsDirectory()); NUnit.Framework.Assert.IsTrue(sd.GetVersionFile().IsFile()); }
/// <summary>invalidate storage by removing the second and third storage directories</summary> /// <exception cref="System.IO.IOException"/> public virtual void InvalidateStorage(FSImage fi, ICollection <FilePath> filesToInvalidate ) { AList <Storage.StorageDirectory> al = new AList <Storage.StorageDirectory>(2); IEnumerator <Storage.StorageDirectory> it = fi.GetStorage().DirIterator(); while (it.HasNext()) { Storage.StorageDirectory sd = it.Next(); if (filesToInvalidate.Contains(sd.GetRoot())) { Log.Info("causing IO error on " + sd.GetRoot()); al.AddItem(sd); } } // simulate an error fi.GetStorage().ReportErrorsOnDirectories(al); foreach (JournalSet.JournalAndStream j in fi.GetEditLog().GetJournals()) { if (j.GetManager() is FileJournalManager) { FileJournalManager fm = (FileJournalManager)j.GetManager(); if (fm.GetStorageDirectory().GetRoot().Equals(path2) || fm.GetStorageDirectory(). GetRoot().Equals(path3)) { EditLogOutputStream mockStream = Org.Mockito.Mockito.Spy(j.GetCurrentStream()); j.SetCurrentStreamForTests(mockStream); Org.Mockito.Mockito.DoThrow(new IOException("Injected fault: write")).When(mockStream ).Write(Org.Mockito.Mockito.AnyObject <FSEditLogOp>()); } } } }
public FileJournalManager(Configuration conf, Storage.StorageDirectory sd, StorageErrorReporter errorReporter) { this.conf = conf; this.sd = sd; this.errorReporter = errorReporter; }
/// <summary>verify that edits log and fsimage are in different directories and of a correct size /// </summary> private void VerifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize ) { Storage.StorageDirectory sd = null; for (IEnumerator <Storage.StorageDirectory> it = img.GetStorage().DirIterator(); it .HasNext();) { sd = it.Next(); if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Image)) { img.GetStorage(); FilePath imf = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Image, 0); Log.Info("--image file " + imf.GetAbsolutePath() + "; len = " + imf.Length() + "; expected = " + expectedImgSize); NUnit.Framework.Assert.AreEqual(expectedImgSize, imf.Length()); } else { if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Edits)) { img.GetStorage(); FilePath edf = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Edits, 0); Log.Info("-- edits file " + edf.GetAbsolutePath() + "; len = " + edf.Length() + "; expected = " + expectedEditsSize); NUnit.Framework.Assert.AreEqual(expectedEditsSize, edf.Length()); } else { NUnit.Framework.Assert.Fail("Image/Edits directories are not different"); } } } }
/// <summary> /// Confirm that FSImage files in all StorageDirectory are the same, /// and non-empty, and there are the expected number of them. /// </summary> /// <param name="fsn">- the FSNamesystem being checked.</param> /// <param name="numImageDirs">- the configured number of StorageDirectory of type IMAGE. /// </param> /// <returns>- the md5 hash of the most recent FSImage files, which must all be the same. /// </returns> /// <exception cref="System.Exception"> /// if image files are empty or different, /// if less than two StorageDirectory are provided, or if the /// actual number of StorageDirectory is less than configured. /// </exception> public static string CheckImages(FSNamesystem fsn, int numImageDirs) { NNStorage stg = fsn.GetFSImage().GetStorage(); //any failed StorageDirectory is removed from the storageDirs list NUnit.Framework.Assert.AreEqual("Some StorageDirectories failed Upgrade", numImageDirs , stg.GetNumStorageDirs(NNStorage.NameNodeDirType.Image)); NUnit.Framework.Assert.IsTrue("Not enough fsimage copies in MiniDFSCluster " + "to test parallel write" , numImageDirs > 1); // List of "current/" directory from each SD IList <FilePath> dirs = FSImageTestUtil.GetCurrentDirs(stg, NNStorage.NameNodeDirType .Image); // across directories, all files with same names should be identical hashes FSImageTestUtil.AssertParallelFilesAreIdentical(dirs, Sharpen.Collections.EmptySet <string>()); FSImageTestUtil.AssertSameNewestImage(dirs); // Return the hash of the newest image file Storage.StorageDirectory firstSd = stg.DirIterator(NNStorage.NameNodeDirType.Image ).Next(); FilePath latestImage = FSImageTestUtil.FindLatestImageFile(firstSd); string md5 = FSImageTestUtil.GetImageFileMD5IgnoringTxId(latestImage); System.Console.Error.WriteLine("md5 of " + latestImage + ": " + md5); return(md5); }
public virtual void TestAddVolumeFailures() { StartDFSCluster(1, 1); string dataDir = cluster.GetDataDirectory(); DataNode dn = cluster.GetDataNodes()[0]; IList <string> newDirs = Lists.NewArrayList(); int NumNewDirs = 4; for (int i = 0; i < NumNewDirs; i++) { FilePath newVolume = new FilePath(dataDir, "new_vol" + i); newDirs.AddItem(newVolume.ToString()); if (i % 2 == 0) { // Make addVolume() fail. newVolume.CreateNewFile(); } } string newValue = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey) + "," + Joiner .On(",").Join(newDirs); try { dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newValue); NUnit.Framework.Assert.Fail("Expect to throw IOException."); } catch (ReconfigurationException e) { string errorMessage = e.InnerException.Message; string[] messages = errorMessage.Split("\\r?\\n"); NUnit.Framework.Assert.AreEqual(2, messages.Length); Assert.AssertThat(messages[0], CoreMatchers.ContainsString("new_vol0")); Assert.AssertThat(messages[1], CoreMatchers.ContainsString("new_vol2")); } // Make sure that vol0 and vol2's metadata are not left in memory. FsDatasetSpi <object> dataset = dn.GetFSDataset(); foreach (FsVolumeSpi volume in dataset.GetVolumes()) { Assert.AssertThat(volume.GetBasePath(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf (IS.Is(newDirs[0]), IS.Is(newDirs[2]))))); } DataStorage storage = dn.GetStorage(); for (int i_1 = 0; i_1 < storage.GetNumStorageDirs(); i_1++) { Storage.StorageDirectory sd = storage.GetStorageDir(i_1); Assert.AssertThat(sd.GetRoot().ToString(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf (IS.Is(newDirs[0]), IS.Is(newDirs[2]))))); } // The newly effective conf does not have vol0 and vol2. string[] effectiveVolumes = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey) .Split(","); NUnit.Framework.Assert.AreEqual(4, effectiveVolumes.Length); foreach (string ev in effectiveVolumes) { Assert.AssertThat(StorageLocation.Parse(ev).GetFile().GetCanonicalPath(), IS.Is(CoreMatchers.Not (CoreMatchers.AnyOf(IS.Is(newDirs[0]), IS.Is(newDirs[2]))))); } }
/// <summary>Analyze and load storage directories.</summary> /// <remarks> /// Analyze and load storage directories. Recover from previous transitions if /// required. /// The block pool storages are either all analyzed or none of them is loaded. /// Therefore, a failure on loading any block pool storage results a faulty /// data volume. /// </remarks> /// <param name="datanode">Datanode to which this storage belongs to</param> /// <param name="nsInfo">namespace information</param> /// <param name="dataDirs">storage directories of block pool</param> /// <param name="startOpt">startup option</param> /// <returns>an array of loaded block pool directories.</returns> /// <exception cref="System.IO.IOException">on error</exception> internal virtual IList <Storage.StorageDirectory> LoadBpStorageDirectories(DataNode datanode, NamespaceInfo nsInfo, ICollection <FilePath> dataDirs, HdfsServerConstants.StartupOption startOpt) { IList <Storage.StorageDirectory> succeedDirs = Lists.NewArrayList(); try { foreach (FilePath dataDir in dataDirs) { if (ContainsStorageDir(dataDir)) { throw new IOException("BlockPoolSliceStorage.recoverTransitionRead: " + "attempt to load an used block storage: " + dataDir); } Storage.StorageDirectory sd = LoadStorageDirectory(datanode, nsInfo, dataDir, startOpt ); succeedDirs.AddItem(sd); } } catch (IOException e) { Log.Warn("Failed to analyze storage directories for block pool " + nsInfo.GetBlockPoolID (), e); throw; } return(succeedDirs); }
/// <exception cref="System.IO.IOException"/> public virtual void TestRemoveNewlyAddedVolume() { int numExistingVolumes = dataset.GetVolumes().Count; IList <NamespaceInfo> nsInfos = new AList <NamespaceInfo>(); foreach (string bpid in BlockPoolIds) { nsInfos.AddItem(new NamespaceInfo(0, ClusterId, bpid, 1)); } string newVolumePath = BaseDir + "/newVolumeToRemoveLater"; StorageLocation loc = StorageLocation.Parse(newVolumePath); Storage.StorageDirectory sd = CreateStorageDirectory(new FilePath(newVolumePath)); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); Org.Mockito.Mockito.When(storage.PrepareVolume(Matchers.Eq(datanode), Matchers.Eq (loc.GetFile()), Matchers.AnyListOf <NamespaceInfo>())).ThenReturn(builder); dataset.AddVolume(loc, nsInfos); NUnit.Framework.Assert.AreEqual(numExistingVolumes + 1, dataset.GetVolumes().Count ); Org.Mockito.Mockito.When(storage.GetNumStorageDirs()).ThenReturn(numExistingVolumes + 1); Org.Mockito.Mockito.When(storage.GetStorageDir(numExistingVolumes)).ThenReturn(sd ); ICollection <FilePath> volumesToRemove = new HashSet <FilePath>(); volumesToRemove.AddItem(loc.GetFile()); dataset.RemoveVolumes(volumesToRemove, true); NUnit.Framework.Assert.AreEqual(numExistingVolumes, dataset.GetVolumes().Count); }
public virtual void TestExcludeInProgressStreams() { FilePath f = new FilePath(TestEditLog.TestDir + "/excludeinprogressstreams"); // Don't close the edit log once the files have been set up. NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI> (f.ToURI()), 10, false); Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits ).Next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); // If we exclude the in-progess stream, we should only have 100 tx. NUnit.Framework.Assert.AreEqual(100, GetNumberOfTransactions(jm, 1, false, false) ); EditLogInputStream elis = GetJournalInputStream(jm, 90, false); try { FSEditLogOp lastReadOp = null; while ((lastReadOp = elis.ReadOp()) != null) { NUnit.Framework.Assert.IsTrue(lastReadOp.GetTransactionId() <= 100); } } finally { IOUtils.Cleanup(Log, elis); } }
/// <exception cref="System.Exception"/> public virtual void TestJournalLocking() { Assume.AssumeTrue(journal.GetStorage().GetStorageDir(0).IsLockSupported()); Storage.StorageDirectory sd = journal.GetStorage().GetStorageDir(0); FilePath lockFile = new FilePath(sd.GetRoot(), Storage.StorageFileLock); // Journal should be locked, since the format() call locks it. GenericTestUtils.AssertExists(lockFile); journal.NewEpoch(FakeNsinfo, 1); try { new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular, mockErrorReporter ); NUnit.Framework.Assert.Fail("Did not fail to create another journal in same dir"); } catch (IOException ioe) { GenericTestUtils.AssertExceptionContains("Cannot lock storage", ioe); } journal.Close(); // Journal should no longer be locked after the close() call. // Hence, should be able to create a new Journal in the same dir. Journal journal2 = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption .Regular, mockErrorReporter); journal2.NewEpoch(FakeNsinfo, 2); journal2.Close(); }
public virtual void TestManyLogsWithGaps() { FilePath f = new FilePath(TestEditLog.TestDir + "/manylogswithgaps"); NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI> (f.ToURI()), 10); Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits ).Next(); long startGapTxId = 3 * TestEditLog.TxnsPerRoll + 1; long endGapTxId = 4 * TestEditLog.TxnsPerRoll; FilePath[] files = new FilePath(f, "current").ListFiles(new _FilenameFilter_324(startGapTxId , endGapTxId)); NUnit.Framework.Assert.AreEqual(1, files.Length); NUnit.Framework.Assert.IsTrue(files[0].Delete()); FileJournalManager jm = new FileJournalManager(conf, sd, storage); NUnit.Framework.Assert.AreEqual(startGapTxId - 1, GetNumberOfTransactions(jm, 1, true, true)); NUnit.Framework.Assert.AreEqual(0, GetNumberOfTransactions(jm, startGapTxId, true , true)); // rolled 10 times so there should be 11 files. NUnit.Framework.Assert.AreEqual(11 * TestEditLog.TxnsPerRoll - endGapTxId, GetNumberOfTransactions (jm, endGapTxId + 1, true, true)); }
public virtual void TestInprogressRecoveryAll() { FilePath f1 = new FilePath(TestEditLog.TestDir + "/failalltest0"); FilePath f2 = new FilePath(TestEditLog.TestDir + "/failalltest1"); FilePath f3 = new FilePath(TestEditLog.TestDir + "/failalltest2"); IList <URI> editUris = ImmutableList.Of(f1.ToURI(), f2.ToURI(), f3.ToURI()); // abort after the 5th roll NNStorage storage = TestEditLog.SetupEdits(editUris, 5, new TestEditLog.AbortSpec (5, 0), new TestEditLog.AbortSpec(5, 1), new TestEditLog.AbortSpec(5, 2)); IEnumerator <Storage.StorageDirectory> dirs = storage.DirIterator(NNStorage.NameNodeDirType .Edits); Storage.StorageDirectory sd = dirs.Next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail , GetNumberOfTransactions(jm, 1, true, false)); sd = dirs.Next(); jm = new FileJournalManager(conf, sd, storage); NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail , GetNumberOfTransactions(jm, 1, true, false)); sd = dirs.Next(); jm = new FileJournalManager(conf, sd, storage); NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail , GetNumberOfTransactions(jm, 1, true, false)); }
/* * Finalize the block pool storage by deleting <BP>/previous directory * that holds the snapshot. */ /// <exception cref="System.IO.IOException"/> internal virtual void DoFinalize(FilePath dnCurDir) { FilePath bpRoot = GetBpRoot(blockpoolID, dnCurDir); Storage.StorageDirectory bpSd = new Storage.StorageDirectory(bpRoot); // block pool level previous directory FilePath prevDir = bpSd.GetPreviousDir(); if (!prevDir.Exists()) { return; } // already finalized string dataDirPath = bpSd.GetRoot().GetCanonicalPath(); Log.Info("Finalizing upgrade for storage directory " + dataDirPath + ".\n cur LV = " + this.GetLayoutVersion() + "; cur CTime = " + this.GetCTime()); System.Diagnostics.Debug.Assert(bpSd.GetCurrentDir().Exists(), "Current directory must exist." ); // rename previous to finalized.tmp FilePath tmpDir = bpSd.GetFinalizedTmp(); Rename(prevDir, tmpDir); // delete finalized.tmp dir in a separate thread new Daemon(new _Runnable_618(tmpDir, dataDirPath)).Start(); }
public virtual void TestAddVolumeFailureReleasesInUseLock() { FsDatasetImpl spyDataset = Org.Mockito.Mockito.Spy(dataset); FsVolumeImpl mockVolume = Org.Mockito.Mockito.Mock <FsVolumeImpl>(); FilePath badDir = new FilePath(BaseDir, "bad"); badDir.Mkdirs(); Org.Mockito.Mockito.DoReturn(mockVolume).When(spyDataset).CreateFsVolume(Matchers.AnyString (), Matchers.Any <FilePath>(), Matchers.Any <StorageType>()); Org.Mockito.Mockito.DoThrow(new IOException("Failed to getVolumeMap()")).When(mockVolume ).GetVolumeMap(Matchers.AnyString(), Matchers.Any <ReplicaMap>(), Matchers.Any <RamDiskReplicaLruTracker >()); Storage.StorageDirectory sd = CreateStorageDirectory(badDir); sd.Lock(); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); Org.Mockito.Mockito.When(storage.PrepareVolume(Matchers.Eq(datanode), Matchers.Eq (badDir.GetAbsoluteFile()), Matchers.Any <IList <NamespaceInfo> >())).ThenReturn(builder ); StorageLocation location = StorageLocation.Parse(badDir.ToString()); IList <NamespaceInfo> nsInfos = Lists.NewArrayList(); foreach (string bpid in BlockPoolIds) { nsInfos.AddItem(new NamespaceInfo(0, ClusterId, bpid, 1)); } try { spyDataset.AddVolume(location, nsInfos); NUnit.Framework.Assert.Fail("Expect to throw MultipleIOException"); } catch (MultipleIOException) { } FsDatasetTestUtil.AssertFileLockReleased(badDir.ToString()); }
/// <returns> /// the fsimage file with the most recent transaction ID in the /// given storage directory. /// </returns> /// <exception cref="System.IO.IOException"/> public static FilePath FindLatestImageFile(Storage.StorageDirectory sd) { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); inspector.InspectDirectory(sd); return(inspector.GetLatestImages()[0].GetFile()); }
internal FSImageFile(Storage.StorageDirectory sd, FilePath file, long txId) { System.Diagnostics.Debug.Assert(txId >= 0 || txId == HdfsConstants.InvalidTxid, "Invalid txid on " + file + ": " + txId); this.sd = sd; this.txId = txId; this.file = file; }
/// <summary> /// Set layoutVersion, namespaceID and blockpoolID into block pool storage /// VERSION file /// </summary> /// <exception cref="System.IO.IOException"/> protected internal override void SetPropertiesFromFields(Properties props, Storage.StorageDirectory sd) { props.SetProperty("layoutVersion", layoutVersion.ToString()); props.SetProperty("namespaceID", namespaceID.ToString()); props.SetProperty("blockpoolID", blockpoolID); props.SetProperty("cTime", cTime.ToString()); }
/// <summary>Load one storage directory.</summary> /// <remarks>Load one storage directory. Recover from previous transitions if required. /// </remarks> /// <param name="datanode">datanode instance</param> /// <param name="nsInfo">namespace information</param> /// <param name="dataDir">the root path of the storage directory</param> /// <param name="startOpt">startup option</param> /// <returns>the StorageDirectory successfully loaded.</returns> /// <exception cref="System.IO.IOException"/> private Storage.StorageDirectory LoadStorageDirectory(DataNode datanode, NamespaceInfo nsInfo, FilePath dataDir, HdfsServerConstants.StartupOption startOpt) { Storage.StorageDirectory sd = new Storage.StorageDirectory(dataDir, null, true); try { Storage.StorageState curState = sd.AnalyzeStorage(startOpt, this); switch (curState) { case Storage.StorageState.Normal: { // sd is locked but not opened break; } case Storage.StorageState.NonExistent: { Log.Info("Block pool storage directory " + dataDir + " does not exist"); throw new IOException("Storage directory " + dataDir + " does not exist"); } case Storage.StorageState.NotFormatted: { // format Log.Info("Block pool storage directory " + dataDir + " is not formatted for " + nsInfo .GetBlockPoolID()); Log.Info("Formatting ..."); Format(sd, nsInfo); break; } default: { // recovery part is common sd.DoRecover(curState); break; } } // 2. Do transitions // Each storage directory is treated individually. // During startup some of them can upgrade or roll back // while others could be up-to-date for the regular startup. DoTransition(datanode, sd, nsInfo, startOpt); if (GetCTime() != nsInfo.GetCTime()) { throw new IOException("Data-node and name-node CTimes must be the same."); } // 3. Update successfully loaded storage. SetServiceLayoutVersion(GetServiceLayoutVersion()); WriteProperties(sd); return(sd); } catch (IOException ioe) { sd.Unlock(); throw; } }
/// <summary>Get common storage fields.</summary> /// <remarks> /// Get common storage fields. /// Should be overloaded if additional fields need to be get. /// </remarks> /// <param name="props">properties</param> /// <exception cref="System.IO.IOException">on error</exception> protected internal virtual void SetFieldsFromProperties(Properties props, Storage.StorageDirectory sd) { SetLayoutVersion(props, sd); SetNamespaceID(props, sd); SetcTime(props, sd); SetClusterId(props, layoutVersion, sd); CheckStorageType(props, sd); }
/// <param name="conf">Configuration object</param> /// <param name="logDir">the path to the directory in which data will be stored</param> /// <param name="errorReporter">a callback to report errors</param> /// <exception cref="System.IO.IOException"></exception> protected internal JNStorage(Configuration conf, FilePath logDir, HdfsServerConstants.StartupOption startOpt, StorageErrorReporter errorReporter) : base(HdfsServerConstants.NodeType.JournalNode) { sd = new Storage.StorageDirectory(logDir); this.AddStorageDir(sd); this.fjm = new FileJournalManager(conf, sd, errorReporter); AnalyzeAndRecoverStorage(startOpt); }
/// <exception cref="System.IO.IOException"/> private void DoUpgrade(NNStorage storage) { for (IEnumerator <Storage.StorageDirectory> it = storage.DirIterator(false); it.HasNext ();) { Storage.StorageDirectory sd = it.Next(); NNUpgradeUtil.DoUpgrade(sd, storage); } }
/// <summary>This is called when using bootstrapStandby for HA upgrade.</summary> /// <remarks> /// This is called when using bootstrapStandby for HA upgrade. The SBN should /// also create previous directory so that later when it starts, it understands /// that the cluster is in the upgrade state. This function renames the old /// current directory to previous.tmp. /// </remarks> /// <exception cref="System.IO.IOException"/> private bool DoPreUpgrade(NNStorage storage, NamespaceInfo nsInfo) { bool isFormatted = false; IDictionary <Storage.StorageDirectory, Storage.StorageState> dataDirStates = new Dictionary <Storage.StorageDirectory, Storage.StorageState>(); try { isFormatted = FSImage.RecoverStorageDirs(HdfsServerConstants.StartupOption.Upgrade , storage, dataDirStates); if (dataDirStates.Values.Contains(Storage.StorageState.NotFormatted)) { // recoverStorageDirs returns true if there is a formatted directory isFormatted = false; System.Console.Error.WriteLine("The original storage directory is not formatted." ); } } catch (InconsistentFSStateException e) { // if the storage is in a bad state, Log.Warn("The storage directory is in an inconsistent state", e); } finally { storage.UnlockAll(); } // if there is InconsistentFSStateException or the storage is not formatted, // format the storage. Although this format is done through the new // software, since in HA setup the SBN is rolled back through // "-bootstrapStandby", we should still be fine. if (!isFormatted && !Format(storage, nsInfo)) { return(false); } // make sure there is no previous directory FSImage.CheckUpgrade(storage); // Do preUpgrade for each directory for (IEnumerator <Storage.StorageDirectory> it = storage.DirIterator(false); it.HasNext ();) { Storage.StorageDirectory sd = it.Next(); try { NNUpgradeUtil.RenameCurToTmp(sd); } catch (IOException e) { Log.Error("Failed to move aside pre-upgrade storage " + "in image directory " + sd .GetRoot(), e); throw; } } storage.SetStorageInfo(nsInfo); storage.SetBlockPoolID(nsInfo.GetBlockPoolID()); return(true); }
public virtual void TestEditLogRolling() { // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; FileSystem fileSys = null; AtomicReference <Exception> caughtErr = new AtomicReference <Exception>(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build(); cluster.WaitActive(); fileSys = cluster.GetFileSystem(); NamenodeProtocols nn = cluster.GetNameNode().GetRpcServer(); FSImage fsimage = cluster.GetNamesystem().GetFSImage(); Storage.StorageDirectory sd = fsimage.GetStorage().GetStorageDir(0); StartTransactionWorkers(nn, caughtErr); long previousLogTxId = 1; for (int i = 0; i < NumRolls && caughtErr.Get() == null; i++) { try { Sharpen.Thread.Sleep(20); } catch (Exception) { } Log.Info("Starting roll " + i + "."); CheckpointSignature sig = nn.RollEditLog(); long nextLog = sig.curSegmentTxId; string logFileName = NNStorage.GetFinalizedEditsFileName(previousLogTxId, nextLog - 1); previousLogTxId += VerifyEditLogs(cluster.GetNamesystem(), fsimage, logFileName, previousLogTxId); NUnit.Framework.Assert.AreEqual(previousLogTxId, nextLog); FilePath expectedLog = NNStorage.GetInProgressEditsFile(sd, previousLogTxId); NUnit.Framework.Assert.IsTrue("Expect " + expectedLog + " to exist", expectedLog. Exists()); } } finally { StopTransactionWorkers(); if (caughtErr.Get() != null) { throw new RuntimeException(caughtErr.Get()); } if (fileSys != null) { fileSys.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Common.IncorrectVersionException"/ /// > /// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Common.InconsistentFSStateException /// "/> protected internal override void SetLayoutVersion(Properties props, Storage.StorageDirectory sd) { int lv = System.Convert.ToInt32(GetProperty(props, sd, "layoutVersion")); // For journal node, since it now does not decode but just scan through the // edits, it can handle edits with future version in most of the cases. // Thus currently we may skip the layoutVersion check here. layoutVersion = lv; }
/// <exception cref="System.IO.IOException"/> protected internal override void SetFieldsFromProperties(Properties props, Storage.StorageDirectory sd) { SetLayoutVersion(props, sd); SetNamespaceID(props, sd); SetcTime(props, sd); string sbpid = props.GetProperty("blockpoolID"); SetBlockPoolID(sd.GetRoot(), sbpid); }
/// <returns> /// the latest edits log, finalized or otherwise, from the given /// storage directory. /// </returns> /// <exception cref="System.IO.IOException"/> public static FileJournalManager.EditLogFile FindLatestEditsLog(Storage.StorageDirectory sd) { FilePath currentDir = sd.GetCurrentDir(); IList <FileJournalManager.EditLogFile> foundEditLogs = Lists.NewArrayList(FileJournalManager .MatchEditLogs(currentDir)); return(Sharpen.Collections.Max(foundEditLogs, FileJournalManager.EditLogFile.CompareByStartTxid )); }
/// <summary>Check whether the root is a valid BlockPoolSlice storage.</summary> private static void CheckDir(FilePath root, string bpid) { Storage.StorageDirectory sd = new Storage.StorageDirectory(root); FilePath bpRoot = new FilePath(sd.GetCurrentDir(), bpid); Storage.StorageDirectory bpSd = new Storage.StorageDirectory(bpRoot); NUnit.Framework.Assert.IsTrue(bpSd.GetRoot().IsDirectory()); NUnit.Framework.Assert.IsTrue(bpSd.GetCurrentDir().IsDirectory()); NUnit.Framework.Assert.IsTrue(bpSd.GetVersionFile().IsFile()); }