/// <exception cref="System.IO.IOException"/> public override void FinalizeLogSegment(long firstTxId, long lastTxId) { lock (this) { FilePath inprogressFile = NNStorage.GetInProgressEditsFile(sd, firstTxId); FilePath dstFile = NNStorage.GetFinalizedEditsFile(sd, firstTxId, lastTxId); Log.Info("Finalizing edits file " + inprogressFile + " -> " + dstFile); Preconditions.CheckState(!dstFile.Exists(), "Can't finalize edits file " + inprogressFile + " since finalized file " + "already exists"); try { NativeIO.RenameTo(inprogressFile, dstFile); } catch (IOException e) { errorReporter.ReportErrorOnFile(dstFile); throw new InvalidOperationException("Unable to finalize edits file " + inprogressFile , e); } if (inprogressFile.Equals(currentInProgress)) { currentInProgress = null; } } }
public virtual void TestEditLogRolling() { // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; FileSystem fileSys = null; AtomicReference <Exception> caughtErr = new AtomicReference <Exception>(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build(); cluster.WaitActive(); fileSys = cluster.GetFileSystem(); NamenodeProtocols nn = cluster.GetNameNode().GetRpcServer(); FSImage fsimage = cluster.GetNamesystem().GetFSImage(); Storage.StorageDirectory sd = fsimage.GetStorage().GetStorageDir(0); StartTransactionWorkers(nn, caughtErr); long previousLogTxId = 1; for (int i = 0; i < NumRolls && caughtErr.Get() == null; i++) { try { Sharpen.Thread.Sleep(20); } catch (Exception) { } Log.Info("Starting roll " + i + "."); CheckpointSignature sig = nn.RollEditLog(); long nextLog = sig.curSegmentTxId; string logFileName = NNStorage.GetFinalizedEditsFileName(previousLogTxId, nextLog - 1); previousLogTxId += VerifyEditLogs(cluster.GetNamesystem(), fsimage, logFileName, previousLogTxId); NUnit.Framework.Assert.AreEqual(previousLogTxId, nextLog); FilePath expectedLog = NNStorage.GetInProgressEditsFile(sd, previousLogTxId); NUnit.Framework.Assert.IsTrue("Expect " + expectedLog + " to exist", expectedLog. Exists()); } } finally { StopTransactionWorkers(); if (caughtErr.Get() != null) { throw new RuntimeException(caughtErr.Get()); } if (fileSys != null) { fileSys.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public override EditLogOutputStream StartLogSegment(long txid, int layoutVersion) { lock (this) { try { currentInProgress = NNStorage.GetInProgressEditsFile(sd, txid); EditLogOutputStream stm = new EditLogFileOutputStream(conf, currentInProgress, outputBufferCapacity ); stm.Create(layoutVersion); return(stm); } catch (IOException e) { Log.Warn("Unable to start log segment " + txid + " at " + currentInProgress + ": " + e.GetLocalizedMessage()); errorReporter.ReportErrorOnFile(currentInProgress); throw; } } }
public static FilePath GetInProgressEditsFile(Storage.StorageDirectory sd, long startTxId ) { return(NNStorage.GetInProgressEditsFile(sd, startTxId)); }