/// <exception cref="System.IO.IOException"/> public override void FinalizeLogSegment(long firstTxId, long lastTxId) { lock (this) { FilePath inprogressFile = NNStorage.GetInProgressEditsFile(sd, firstTxId); FilePath dstFile = NNStorage.GetFinalizedEditsFile(sd, firstTxId, lastTxId); Log.Info("Finalizing edits file " + inprogressFile + " -> " + dstFile); Preconditions.CheckState(!dstFile.Exists(), "Can't finalize edits file " + inprogressFile + " since finalized file " + "already exists"); try { NativeIO.RenameTo(inprogressFile, dstFile); } catch (IOException e) { errorReporter.ReportErrorOnFile(dstFile); throw new InvalidOperationException("Unable to finalize edits file " + inprogressFile , e); } if (inprogressFile.Equals(currentInProgress)) { currentInProgress = null; } } }
/// <exception cref="System.IO.IOException"/> internal static void DownloadEditsToStorage(Uri fsName, RemoteEditLog log, NNStorage dstStorage) { System.Diagnostics.Debug.Assert(log.GetStartTxId() > 0 && log.GetEndTxId() > 0, "bad log: " + log); string fileid = ImageServlet.GetParamStringForLog(log, dstStorage); string finalFileName = NNStorage.GetFinalizedEditsFileName(log.GetStartTxId(), log .GetEndTxId()); IList <FilePath> finalFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, finalFileName); System.Diagnostics.Debug.Assert(!finalFiles.IsEmpty(), "No checkpoint targets."); foreach (FilePath f in finalFiles) { if (f.Exists() && FileUtil.CanRead(f)) { Log.Info("Skipping download of remote edit log " + log + " since it already is stored locally at " + f); return; } else { if (Log.IsDebugEnabled()) { Log.Debug("Dest file: " + f); } } } long milliTime = Time.MonotonicNow(); string tmpFileName = NNStorage.GetTemporaryEditsFileName(log.GetStartTxId(), log. GetEndTxId(), milliTime); IList <FilePath> tmpFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, tmpFileName ); GetFileClient(fsName, fileid, tmpFiles, dstStorage, false); Log.Info("Downloaded file " + tmpFiles[0].GetName() + " size " + finalFiles[0].Length () + " bytes."); CheckpointFaultInjector.GetInstance().BeforeEditsRename(); foreach (Storage.StorageDirectory sd in dstStorage.DirIterable(NNStorage.NameNodeDirType .Edits)) { FilePath tmpFile = NNStorage.GetTemporaryEditsFile(sd, log.GetStartTxId(), log.GetEndTxId (), milliTime); FilePath finalizedFile = NNStorage.GetFinalizedEditsFile(sd, log.GetStartTxId(), log.GetEndTxId()); if (Log.IsDebugEnabled()) { Log.Debug("Renaming " + tmpFile + " to " + finalizedFile); } bool success = tmpFile.RenameTo(finalizedFile); if (!success) { Log.Warn("Unable to rename edits file from " + tmpFile + " to " + finalizedFile); } } }
/// <summary>Get edits filename</summary> /// <returns>edits file name for cluster</returns> /// <exception cref="System.IO.IOException"/> private string GetEditsFilename(CheckpointSignature sig) { FSImage image = cluster.GetNameNode().GetFSImage(); // it was set up to only have ONE StorageDirectory IEnumerator <Storage.StorageDirectory> it = image.GetStorage().DirIterator(NNStorage.NameNodeDirType .Edits); Storage.StorageDirectory sd = it.Next(); FilePath ret = NNStorage.GetFinalizedEditsFile(sd, 1, sig.curSegmentTxId - 1); System.Diagnostics.Debug.Assert(ret.Exists(), "expected " + ret + " exists"); return(ret.GetAbsolutePath()); }
public virtual void TestEditLog() { // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; FileSystem fileSys = null; try { conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build(); cluster.WaitActive(); fileSys = cluster.GetFileSystem(); FSNamesystem namesystem = cluster.GetNamesystem(); for (IEnumerator <URI> it = cluster.GetNameDirs(0).GetEnumerator(); it.HasNext();) { FilePath dir = new FilePath(it.Next().GetPath()); System.Console.Out.WriteLine(dir); } FSImage fsimage = namesystem.GetFSImage(); FSEditLog editLog = fsimage.GetEditLog(); // set small size of flush buffer editLog.SetOutputBufferCapacity(2048); // Create threads and make them run transactions concurrently. Sharpen.Thread[] threadId = new Sharpen.Thread[NumThreads]; for (int i = 0; i < NumThreads; i++) { TestSecurityTokenEditLog.Transactions trans = new TestSecurityTokenEditLog.Transactions (namesystem, NumTransactions); threadId[i] = new Sharpen.Thread(trans, "TransactionThread-" + i); threadId[i].Start(); } // wait for all transactions to get over for (int i_1 = 0; i_1 < NumThreads; i_1++) { try { threadId[i_1].Join(); } catch (Exception) { i_1--; } } // retry editLog.Close(); // Verify that we can read in all the transactions that we have written. // If there were any corruptions, it is likely that the reading in // of these transactions will throw an exception. // namesystem.GetDelegationTokenSecretManager().StopThreads(); int numKeys = namesystem.GetDelegationTokenSecretManager().GetNumberOfKeys(); int expectedTransactions = NumThreads * opsPerTrans * NumTransactions + numKeys + 2; // + 2 for BEGIN and END txns foreach (Storage.StorageDirectory sd in fsimage.GetStorage().DirIterable(NNStorage.NameNodeDirType .Edits)) { FilePath editFile = NNStorage.GetFinalizedEditsFile(sd, 1, 1 + expectedTransactions - 1); System.Console.Out.WriteLine("Verifying file: " + editFile); FSEditLogLoader loader = new FSEditLogLoader(namesystem, 0); long numEdits = loader.LoadFSEdits(new EditLogFileInputStream(editFile), 1); NUnit.Framework.Assert.AreEqual("Verification for " + editFile, expectedTransactions , numEdits); } } finally { if (fileSys != null) { fileSys.Close(); } if (cluster != null) { cluster.Shutdown(); } } }