public virtual void TestDoPreUpgradeIOError()
{
FilePath storageDir = new FilePath(TestEditLog.TestDir, "preupgradeioerror");
IList <URI> editUris = Sharpen.Collections.SingletonList(storageDir.ToURI());
NNStorage storage = TestEditLog.SetupEdits(editUris, 5);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
NUnit.Framework.Assert.IsNotNull(sd);
// Change storage directory so that renaming current to previous.tmp fails.
FileUtil.SetWritable(storageDir, false);
FileJournalManager jm = null;
try
{
jm = new FileJournalManager(conf, sd, storage);
exception.Expect(typeof(IOException));
if (NativeCodeLoader.IsNativeCodeLoaded())
{
exception.ExpectMessage("failure in native rename");
}
jm.DoPreUpgrade();
}
finally
{
IOUtils.Cleanup(Log, jm);
// Restore permissions on storage directory and make sure we can delete.
FileUtil.SetWritable(storageDir, true);
FileUtil.FullyDelete(storageDir);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual FSEditLog MockEditLog(NNStorageRetentionManager.StoragePurger purger
)
{
IList <JournalManager> jms = Lists.NewArrayList();
JournalSet journalSet = new JournalSet(0);
foreach (TestNNStorageRetentionManager.TestCaseDescription.FakeRoot root in this.
dirRoots.Values)
{
if (!root.type.IsOfType(NNStorage.NameNodeDirType.Edits))
{
continue;
}
// passing null NNStorage for unit test because it does not use it
FileJournalManager fjm = new FileJournalManager(this._enclosing.conf, root.MockStorageDir
(), null);
fjm.purger = purger;
jms.AddItem(fjm);
journalSet.Add(fjm, false);
}
FSEditLog mockLog = Org.Mockito.Mockito.Mock <FSEditLog>();
Org.Mockito.Mockito.DoAnswer(new _Answer_350(jms)).When(mockLog).PurgeLogsOlderThan
(Org.Mockito.Mockito.AnyLong());
Org.Mockito.Mockito.DoAnswer(new _Answer_365(journalSet)).When(mockLog).SelectInputStreams
(Org.Mockito.Mockito.AnyCollection(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.AnyBoolean());
return(mockLog);
}
public virtual void TestManyLogsWithGaps()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/manylogswithgaps");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
long startGapTxId = 3 * TestEditLog.TxnsPerRoll + 1;
long endGapTxId = 4 * TestEditLog.TxnsPerRoll;
FilePath[] files = new FilePath(f, "current").ListFiles(new _FilenameFilter_324(startGapTxId
, endGapTxId));
NUnit.Framework.Assert.AreEqual(1, files.Length);
NUnit.Framework.Assert.IsTrue(files[0].Delete());
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(startGapTxId - 1, GetNumberOfTransactions(jm, 1,
true, true));
NUnit.Framework.Assert.AreEqual(0, GetNumberOfTransactions(jm, startGapTxId, true
, true));
// rolled 10 times so there should be 11 files.
NUnit.Framework.Assert.AreEqual(11 * TestEditLog.TxnsPerRoll - endGapTxId, GetNumberOfTransactions
(jm, endGapTxId + 1, true, true));
}
public virtual void TestExcludeInProgressStreams()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/excludeinprogressstreams");
// Don't close the edit log once the files have been set up.
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, false);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
// If we exclude the in-progess stream, we should only have 100 tx.
NUnit.Framework.Assert.AreEqual(100, GetNumberOfTransactions(jm, 1, false, false)
);
EditLogInputStream elis = GetJournalInputStream(jm, 90, false);
try
{
FSEditLogOp lastReadOp = null;
while ((lastReadOp = elis.ReadOp()) != null)
{
NUnit.Framework.Assert.IsTrue(lastReadOp.GetTransactionId() <= 100);
}
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
public virtual void TestInprogressRecoveryAll()
{
FilePath f1 = new FilePath(TestEditLog.TestDir + "/failalltest0");
FilePath f2 = new FilePath(TestEditLog.TestDir + "/failalltest1");
FilePath f3 = new FilePath(TestEditLog.TestDir + "/failalltest2");
IList <URI> editUris = ImmutableList.Of(f1.ToURI(), f2.ToURI(), f3.ToURI());
// abort after the 5th roll
NNStorage storage = TestEditLog.SetupEdits(editUris, 5, new TestEditLog.AbortSpec
(5, 0), new TestEditLog.AbortSpec(5, 1), new TestEditLog.AbortSpec(5, 2));
IEnumerator <Storage.StorageDirectory> dirs = storage.DirIterator(NNStorage.NameNodeDirType
.Edits);
Storage.StorageDirectory sd = dirs.Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
sd = dirs.Next();
jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
sd = dirs.Next();
jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
}
/// <exception cref="System.IO.IOException"/>
internal virtual void CheckImageAndEditsFilesExistence(FilePath dir, bool shouldHaveImages
, bool shouldHaveEdits)
{
FSImageTransactionalStorageInspector ins = Inspect(dir);
if (shouldHaveImages)
{
NUnit.Framework.Assert.IsTrue("Expect images in " + dir, ins.foundImages.Count >
0);
}
else
{
NUnit.Framework.Assert.IsTrue("Expect no images in " + dir, ins.foundImages.IsEmpty
());
}
IList <FileJournalManager.EditLogFile> editlogs = FileJournalManager.MatchEditLogs
(new FilePath(dir, "current").ListFiles());
if (shouldHaveEdits)
{
NUnit.Framework.Assert.IsTrue("Expect edits in " + dir, editlogs.Count > 0);
}
else
{
NUnit.Framework.Assert.IsTrue("Expect no edits in " + dir, editlogs.IsEmpty());
}
}
/// <summary>invalidate storage by removing the second and third storage directories</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void InvalidateStorage(FSImage fi, ICollection <FilePath> filesToInvalidate
)
{
AList <Storage.StorageDirectory> al = new AList <Storage.StorageDirectory>(2);
IEnumerator <Storage.StorageDirectory> it = fi.GetStorage().DirIterator();
while (it.HasNext())
{
Storage.StorageDirectory sd = it.Next();
if (filesToInvalidate.Contains(sd.GetRoot()))
{
Log.Info("causing IO error on " + sd.GetRoot());
al.AddItem(sd);
}
}
// simulate an error
fi.GetStorage().ReportErrorsOnDirectories(al);
foreach (JournalSet.JournalAndStream j in fi.GetEditLog().GetJournals())
{
if (j.GetManager() is FileJournalManager)
{
FileJournalManager fm = (FileJournalManager)j.GetManager();
if (fm.GetStorageDirectory().GetRoot().Equals(path2) || fm.GetStorageDirectory().
GetRoot().Equals(path3))
{
EditLogOutputStream mockStream = Org.Mockito.Mockito.Spy(j.GetCurrentStream());
j.SetCurrentStreamForTests(mockStream);
Org.Mockito.Mockito.DoThrow(new IOException("Injected fault: write")).When(mockStream
).Write(Org.Mockito.Mockito.AnyObject <FSEditLogOp>());
}
}
}
}
public virtual void TestInprogressRecovery()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/inprogressrecovery");
// abort after the 5th roll
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 5, new TestEditLog.AbortSpec(5, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
}
/// <summary>
/// Find out how many transactions we can read from a
/// FileJournalManager, starting at a given transaction ID.
/// </summary>
/// <param name="jm">The journal manager</param>
/// <param name="fromTxId">Transaction ID to start at</param>
/// <param name="inProgressOk">Should we consider edit logs that are not finalized?</param>
/// <returns>The number of transactions</returns>
/// <exception cref="System.IO.IOException"/>
internal static long GetNumberOfTransactions(FileJournalManager jm, long fromTxId
, bool inProgressOk, bool abortOnGap)
{
long numTransactions = 0;
long txId = fromTxId;
PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
>(64, JournalSet.EditLogInputStreamComparator);
jm.SelectInputStreams(allStreams, fromTxId, inProgressOk);
EditLogInputStream elis = null;
try
{
while ((elis = allStreams.Poll()) != null)
{
try
{
elis.SkipUntil(txId);
while (true)
{
FSEditLogOp op = elis.ReadOp();
if (op == null)
{
break;
}
if (abortOnGap && (op.GetTransactionId() != txId))
{
Log.Info("getNumberOfTransactions: detected gap at txId " + fromTxId);
return(numTransactions);
}
txId = op.GetTransactionId() + 1;
numTransactions++;
}
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream
[0]));
}
return(numTransactions);
}
public virtual void TestManyLogsWithCorruptInprogress()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/manylogswithcorruptinprogress");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, new TestEditLog.AbortSpec(10, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FilePath[] files = new FilePath(f, "current").ListFiles(new _FilenameFilter_356()
);
NUnit.Framework.Assert.AreEqual(files.Length, 1);
CorruptAfterStartSegment(files[0]);
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(10 * TestEditLog.TxnsPerRoll + 1, GetNumberOfTransactions
(jm, 1, true, false));
}
public virtual void TestAskForTransactionsMidfile()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/askfortransactionsmidfile");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
// 10 rolls, so 11 rolled files, 110 txids total.
int TotalTxids = 10 * 11;
for (int txid = 1; txid <= TotalTxids; txid++)
{
NUnit.Framework.Assert.AreEqual((TotalTxids - txid) + 1, GetNumberOfTransactions(
jm, txid, true, false));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void PurgeLogsOlderThan(long minTxIdToKeep)
{
IEnumerator <Storage.StorageDirectory> iter = storage.DirIterator();
while (iter.HasNext())
{
Storage.StorageDirectory dir = iter.Next();
IList <FileJournalManager.EditLogFile> editFiles = FileJournalManager.MatchEditLogs
(dir.GetCurrentDir());
foreach (FileJournalManager.EditLogFile f in editFiles)
{
if (f.GetLastTxId() < minTxIdToKeep)
{
purger.PurgeLog(f);
}
}
}
}
public virtual void TestGetRemoteEditLog()
{
Storage.StorageDirectory sd = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType
.Edits, false, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetFinalizedEditsFileName
(101, 200), NNStorage.GetInProgressEditsFileName(201), NNStorage.GetFinalizedEditsFileName
(1001, 1100));
// passing null for NNStorage because this unit test will not use it
FileJournalManager fjm = new FileJournalManager(conf, sd, null);
NUnit.Framework.Assert.AreEqual("[1,100],[101,200],[1001,1100]", GetLogsAsString(
fjm, 1));
NUnit.Framework.Assert.AreEqual("[101,200],[1001,1100]", GetLogsAsString(fjm, 101
));
NUnit.Framework.Assert.AreEqual("[101,200],[1001,1100]", GetLogsAsString(fjm, 150
));
NUnit.Framework.Assert.AreEqual("[1001,1100]", GetLogsAsString(fjm, 201));
NUnit.Framework.Assert.AreEqual("Asking for a newer log than exists should return empty list"
, string.Empty, GetLogsAsString(fjm, 9999));
}
public virtual void TestNormalOperation()
{
FilePath f1 = new FilePath(TestEditLog.TestDir + "/normtest0");
FilePath f2 = new FilePath(TestEditLog.TestDir + "/normtest1");
FilePath f3 = new FilePath(TestEditLog.TestDir + "/normtest2");
IList <URI> editUris = ImmutableList.Of(f1.ToURI(), f2.ToURI(), f3.ToURI());
NNStorage storage = TestEditLog.SetupEdits(editUris, 5);
long numJournals = 0;
foreach (Storage.StorageDirectory sd in storage.DirIterable(NNStorage.NameNodeDirType
.Edits))
{
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(6 * TestEditLog.TxnsPerRoll, GetNumberOfTransactions
(jm, 1, true, false));
numJournals++;
}
NUnit.Framework.Assert.AreEqual(3, numJournals);
}
public virtual void TestReadFromStream()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/readfromstream");
// abort after 10th roll
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, new TestEditLog.AbortSpec(10, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
long expectedTotalTxnCount = TestEditLog.TxnsPerRoll * 10 + TestEditLog.TxnsPerFail;
NUnit.Framework.Assert.AreEqual(expectedTotalTxnCount, GetNumberOfTransactions(jm
, 1, true, false));
long skippedTxns = (3 * TestEditLog.TxnsPerRoll);
// skip first 3 files
long startingTxId = skippedTxns + 1;
long numLoadable = GetNumberOfTransactions(jm, startingTxId, true, false);
NUnit.Framework.Assert.AreEqual(expectedTotalTxnCount - skippedTxns, numLoadable);
}
public virtual void TestReadFromMiddleOfEditLog()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/readfrommiddleofeditlog");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
EditLogInputStream elis = GetJournalInputStream(jm, 5, true);
try
{
FSEditLogOp op = elis.ReadOp();
NUnit.Framework.Assert.AreEqual("read unexpected op", op.GetTransactionId(), 5);
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
public virtual void SelectInputStreams(ICollection <EditLogInputStream> streams, long
fromTxId, bool inProgressOk)
{
IEnumerator <Storage.StorageDirectory> iter = storage.DirIterator();
while (iter.HasNext())
{
Storage.StorageDirectory dir = iter.Next();
IList <FileJournalManager.EditLogFile> editFiles;
try
{
editFiles = FileJournalManager.MatchEditLogs(dir.GetCurrentDir());
}
catch (IOException ioe)
{
throw new RuntimeException(ioe);
}
FileJournalManager.AddStreamsToCollectionFromFiles(editFiles, streams, fromTxId,
inProgressOk);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestFinalizeErrorReportedToNNStorage()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/filejournaltestError");
// abort after 10th roll
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, new TestEditLog.AbortSpec(10, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
string sdRootPath = sd.GetRoot().GetAbsolutePath();
FileUtil.Chmod(sdRootPath, "-w", true);
try
{
jm.FinalizeLogSegment(0, 1);
}
finally
{
FileUtil.Chmod(sdRootPath, "+w", true);
NUnit.Framework.Assert.IsTrue(storage.GetRemovedStorageDirs().Contains(sd));
}
}
/// <returns>
/// the latest edits log, finalized or otherwise, from the given
/// storage directory.
/// </returns>
/// <exception cref="System.IO.IOException"/>
public static FileJournalManager.EditLogFile FindLatestEditsLog(Storage.StorageDirectory
sd)
{
FilePath currentDir = sd.GetCurrentDir();
IList <FileJournalManager.EditLogFile> foundEditLogs = Lists.NewArrayList(FileJournalManager
.MatchEditLogs(currentDir));
return(Sharpen.Collections.Max(foundEditLogs, FileJournalManager.EditLogFile.CompareByStartTxid
));
}
/// <exception cref="System.IO.IOException"/>
private static string GetLogsAsString(FileJournalManager fjm, long firstTxId)
{
return(Joiner.On(",").Join(fjm.GetRemoteEditLogs(firstTxId, false)));
}
/// <summary>Return a manifest of what finalized edit logs are available.</summary>
/// <remarks>
/// Return a manifest of what finalized edit logs are available. All available
/// edit logs are returned starting from the transaction id passed. If
/// 'fromTxId' falls in the middle of a log, that log is returned as well.
/// </remarks>
/// <param name="fromTxId">Starting transaction id to read the logs.</param>
/// <returns>RemoteEditLogManifest object.</returns>
public virtual RemoteEditLogManifest GetEditLogManifest(long fromTxId)
{
lock (this)
{
// Collect RemoteEditLogs available from each FileJournalManager
IList <RemoteEditLog> allLogs = Lists.NewArrayList();
foreach (JournalSet.JournalAndStream j in journals)
{
if (j.GetManager() is FileJournalManager)
{
FileJournalManager fjm = (FileJournalManager)j.GetManager();
try
{
Sharpen.Collections.AddAll(allLogs, fjm.GetRemoteEditLogs(fromTxId, false));
}
catch (Exception t)
{
Log.Warn("Cannot list edit logs in " + fjm, t);
}
}
}
// Group logs by their starting txid
ImmutableListMultimap <long, RemoteEditLog> logsByStartTxId = Multimaps.Index(allLogs
, RemoteEditLog.GetStartTxid);
long curStartTxId = fromTxId;
IList <RemoteEditLog> logs = Lists.NewArrayList();
while (true)
{
ImmutableList <RemoteEditLog> logGroup = ((ImmutableList <RemoteEditLog>)logsByStartTxId
.Get(curStartTxId));
if (logGroup.IsEmpty())
{
// we have a gap in logs - for example because we recovered some old
// storage directory with ancient logs. Clear out any logs we've
// accumulated so far, and then skip to the next segment of logs
// after the gap.
ICollection <long> startTxIds = Sets.NewTreeSet(logsByStartTxId.KeySet());
startTxIds = startTxIds.TailSet(curStartTxId);
if (startTxIds.IsEmpty())
{
break;
}
else
{
if (Log.IsDebugEnabled())
{
Log.Debug("Found gap in logs at " + curStartTxId + ": " + "not returning previous logs in manifest."
);
}
logs.Clear();
curStartTxId = startTxIds.First();
continue;
}
}
// Find the one that extends the farthest forward
RemoteEditLog bestLog = Sharpen.Collections.Max(logGroup);
logs.AddItem(bestLog);
// And then start looking from after that point
curStartTxId = bestLog.GetEndTxId() + 1;
}
RemoteEditLogManifest ret = new RemoteEditLogManifest(logs);
if (Log.IsDebugEnabled())
{
Log.Debug("Generated manifest for logs since " + fromTxId + ":" + ret);
}
return(ret);
}
}
/// <summary>tests that passing an invalid dir to matchEditLogs throws IOException</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestMatchEditLogInvalidDirThrowsIOException()
{
FilePath badDir = new FilePath("does not exist");
FileJournalManager.MatchEditLogs(badDir);
}