/// <exception cref="System.IO.IOException"/>
public override void FinalizeLogSegment(long firstTxId, long lastTxId)
{
lock (this)
{
FilePath inprogressFile = NNStorage.GetInProgressEditsFile(sd, firstTxId);
FilePath dstFile = NNStorage.GetFinalizedEditsFile(sd, firstTxId, lastTxId);
Log.Info("Finalizing edits file " + inprogressFile + " -> " + dstFile);
Preconditions.CheckState(!dstFile.Exists(), "Can't finalize edits file " + inprogressFile
+ " since finalized file " + "already exists");
try
{
NativeIO.RenameTo(inprogressFile, dstFile);
}
catch (IOException e)
{
errorReporter.ReportErrorOnFile(dstFile);
throw new InvalidOperationException("Unable to finalize edits file " + inprogressFile
, e);
}
if (inprogressFile.Equals(currentInProgress))
{
currentInProgress = null;
}
}
}
/// <exception cref="System.IO.IOException"/>
internal static void DownloadEditsToStorage(Uri fsName, RemoteEditLog log, NNStorage
dstStorage)
{
System.Diagnostics.Debug.Assert(log.GetStartTxId() > 0 && log.GetEndTxId() > 0, "bad log: "
+ log);
string fileid = ImageServlet.GetParamStringForLog(log, dstStorage);
string finalFileName = NNStorage.GetFinalizedEditsFileName(log.GetStartTxId(), log
.GetEndTxId());
IList <FilePath> finalFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits,
finalFileName);
System.Diagnostics.Debug.Assert(!finalFiles.IsEmpty(), "No checkpoint targets.");
foreach (FilePath f in finalFiles)
{
if (f.Exists() && FileUtil.CanRead(f))
{
Log.Info("Skipping download of remote edit log " + log + " since it already is stored locally at "
+ f);
return;
}
else
{
if (Log.IsDebugEnabled())
{
Log.Debug("Dest file: " + f);
}
}
}
long milliTime = Time.MonotonicNow();
string tmpFileName = NNStorage.GetTemporaryEditsFileName(log.GetStartTxId(), log.
GetEndTxId(), milliTime);
IList <FilePath> tmpFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, tmpFileName
);
GetFileClient(fsName, fileid, tmpFiles, dstStorage, false);
Log.Info("Downloaded file " + tmpFiles[0].GetName() + " size " + finalFiles[0].Length
() + " bytes.");
CheckpointFaultInjector.GetInstance().BeforeEditsRename();
foreach (Storage.StorageDirectory sd in dstStorage.DirIterable(NNStorage.NameNodeDirType
.Edits))
{
FilePath tmpFile = NNStorage.GetTemporaryEditsFile(sd, log.GetStartTxId(), log.GetEndTxId
(), milliTime);
FilePath finalizedFile = NNStorage.GetFinalizedEditsFile(sd, log.GetStartTxId(),
log.GetEndTxId());
if (Log.IsDebugEnabled())
{
Log.Debug("Renaming " + tmpFile + " to " + finalizedFile);
}
bool success = tmpFile.RenameTo(finalizedFile);
if (!success)
{
Log.Warn("Unable to rename edits file from " + tmpFile + " to " + finalizedFile);
}
}
}
/// <summary>Get edits filename</summary>
/// <returns>edits file name for cluster</returns>
/// <exception cref="System.IO.IOException"/>
private string GetEditsFilename(CheckpointSignature sig)
{
FSImage image = cluster.GetNameNode().GetFSImage();
// it was set up to only have ONE StorageDirectory
IEnumerator <Storage.StorageDirectory> it = image.GetStorage().DirIterator(NNStorage.NameNodeDirType
.Edits);
Storage.StorageDirectory sd = it.Next();
FilePath ret = NNStorage.GetFinalizedEditsFile(sd, 1, sig.curSegmentTxId - 1);
System.Diagnostics.Debug.Assert(ret.Exists(), "expected " + ret + " exists");
return(ret.GetAbsolutePath());
}
public virtual void TestEditLog()
{
// start a cluster
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
try
{
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build();
cluster.WaitActive();
fileSys = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
for (IEnumerator <URI> it = cluster.GetNameDirs(0).GetEnumerator(); it.HasNext();)
{
FilePath dir = new FilePath(it.Next().GetPath());
System.Console.Out.WriteLine(dir);
}
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = fsimage.GetEditLog();
// set small size of flush buffer
editLog.SetOutputBufferCapacity(2048);
// Create threads and make them run transactions concurrently.
Sharpen.Thread[] threadId = new Sharpen.Thread[NumThreads];
for (int i = 0; i < NumThreads; i++)
{
TestSecurityTokenEditLog.Transactions trans = new TestSecurityTokenEditLog.Transactions
(namesystem, NumTransactions);
threadId[i] = new Sharpen.Thread(trans, "TransactionThread-" + i);
threadId[i].Start();
}
// wait for all transactions to get over
for (int i_1 = 0; i_1 < NumThreads; i_1++)
{
try
{
threadId[i_1].Join();
}
catch (Exception)
{
i_1--;
}
}
// retry
editLog.Close();
// Verify that we can read in all the transactions that we have written.
// If there were any corruptions, it is likely that the reading in
// of these transactions will throw an exception.
//
namesystem.GetDelegationTokenSecretManager().StopThreads();
int numKeys = namesystem.GetDelegationTokenSecretManager().GetNumberOfKeys();
int expectedTransactions = NumThreads * opsPerTrans * NumTransactions + numKeys +
2;
// + 2 for BEGIN and END txns
foreach (Storage.StorageDirectory sd in fsimage.GetStorage().DirIterable(NNStorage.NameNodeDirType
.Edits))
{
FilePath editFile = NNStorage.GetFinalizedEditsFile(sd, 1, 1 + expectedTransactions
- 1);
System.Console.Out.WriteLine("Verifying file: " + editFile);
FSEditLogLoader loader = new FSEditLogLoader(namesystem, 0);
long numEdits = loader.LoadFSEdits(new EditLogFileInputStream(editFile), 1);
NUnit.Framework.Assert.AreEqual("Verification for " + editFile, expectedTransactions
, numEdits);
}
}
finally
{
if (fileSys != null)
{
fileSys.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}