public virtual void TestReadURL()
{
HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>();
Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(FakeLogData)).When(conn).GetInputStream
();
Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode
();
Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ToString(FakeLogData.Length)).When
(conn).GetHeaderField("Content-Length");
URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>();
Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito
.Any <Uri>(), Matchers.AnyBoolean());
Uri url = new Uri("http://localhost/fakeLog");
EditLogInputStream elis = EditLogFileInputStream.FromUrl(factory, url, HdfsConstants
.InvalidTxid, HdfsConstants.InvalidTxid, false);
// Read the edit log and verify that we got all of the data.
EnumMap <FSEditLogOpCodes, Holder <int> > counts = FSImageTestUtil.CountEditLogOpTypes
(elis);
Assert.AssertThat(counts[FSEditLogOpCodes.OpAdd].held, CoreMatchers.Is(1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpSetGenstampV1].held, CoreMatchers.Is(
1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpClose].held, CoreMatchers.Is(1));
// Check that length header was picked up.
NUnit.Framework.Assert.AreEqual(FakeLogData.Length, elis.Length());
elis.Close();
}
/// <summary>Find out where the edit log ends.</summary>
/// <remarks>
/// Find out where the edit log ends.
/// This will update the lastTxId of the EditLogFile or
/// mark it as corrupt if it is.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void ValidateLog()
{
FSEditLogLoader.EditLogValidation val = EditLogFileInputStream.ValidateEditLog(file
);
this.lastTxId = val.GetEndTxId();
this.hasCorruptHeader = val.HasCorruptHeader();
}
/// <param name="editLog">a path of an edit log file</param>
/// <returns>the count of each type of operation in the log file</returns>
/// <exception cref="System.Exception">if there is an error reading it</exception>
public static EnumMap <FSEditLogOpCodes, Holder <int> > CountEditLogOpTypes(FilePath
editLog)
{
EditLogInputStream elis = new EditLogFileInputStream(editLog);
try
{
return(CountEditLogOpTypes(elis));
}
finally
{
IOUtils.CloseStream(elis);
}
}
public virtual void TestValidateEmptyEditLog()
{
FilePath testDir = new FilePath(TestDir, "testValidateEmptyEditLog");
SortedDictionary <long, long> offsetToTxId = Maps.NewTreeMap();
FilePath logFile = PrepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId);
// Truncate the file so that there is nothing except the header and
// layout flags section.
TruncateFile(logFile, 8);
FSEditLogLoader.EditLogValidation validation = EditLogFileInputStream.ValidateEditLog
(logFile);
NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader());
NUnit.Framework.Assert.AreEqual(HdfsConstants.InvalidTxid, validation.GetEndTxId(
));
}
public virtual void TestValidateEditLogWithCorruptBody()
{
FilePath testDir = new FilePath(TestDir, "testValidateEditLogWithCorruptBody");
SortedDictionary <long, long> offsetToTxId = Maps.NewTreeMap();
int NumTxns = 20;
FilePath logFile = PrepareUnfinalizedTestEditLog(testDir, NumTxns, offsetToTxId);
// Back up the uncorrupted log
FilePath logFileBak = new FilePath(testDir, logFile.GetName() + ".bak");
Files.Copy(logFile, logFileBak);
FSEditLogLoader.EditLogValidation validation = EditLogFileInputStream.ValidateEditLog
(logFile);
NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader());
// We expect that there will be an OP_START_LOG_SEGMENT, followed by
// NUM_TXNS opcodes, followed by an OP_END_LOG_SEGMENT.
NUnit.Framework.Assert.AreEqual(NumTxns + 1, validation.GetEndTxId());
// Corrupt each edit and verify that validation continues to work
foreach (KeyValuePair <long, long> entry in offsetToTxId)
{
long txOffset = entry.Key;
long txId = entry.Value;
// Restore backup, corrupt the txn opcode
Files.Copy(logFileBak, logFile);
CorruptByteInFile(logFile, txOffset);
validation = EditLogFileInputStream.ValidateEditLog(logFile);
long expectedEndTxId = (txId == (NumTxns + 1)) ? NumTxns : (NumTxns + 1);
NUnit.Framework.Assert.AreEqual("Failed when corrupting txn opcode at " + txOffset
, expectedEndTxId, validation.GetEndTxId());
NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader());
}
// Truncate right before each edit and verify that validation continues
// to work
foreach (KeyValuePair <long, long> entry_1 in offsetToTxId)
{
long txOffset = entry_1.Key;
long txId = entry_1.Value;
// Restore backup, corrupt the txn opcode
Files.Copy(logFileBak, logFile);
TruncateFile(logFile, txOffset);
validation = EditLogFileInputStream.ValidateEditLog(logFile);
long expectedEndTxId = (txId == 0) ? HdfsConstants.InvalidTxid : (txId - 1);
NUnit.Framework.Assert.AreEqual("Failed when corrupting txid " + txId + " txn opcode "
+ "at " + txOffset, expectedEndTxId, validation.GetEndTxId());
NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader());
}
}
public virtual void TestValidateEditLogWithCorruptHeader()
{
FilePath testDir = new FilePath(TestDir, "testValidateEditLogWithCorruptHeader");
SortedDictionary <long, long> offsetToTxId = Maps.NewTreeMap();
FilePath logFile = PrepareUnfinalizedTestEditLog(testDir, 2, offsetToTxId);
RandomAccessFile rwf = new RandomAccessFile(logFile, "rw");
try
{
rwf.Seek(0);
rwf.WriteLong(42);
}
finally
{
// corrupt header
rwf.Close();
}
FSEditLogLoader.EditLogValidation validation = EditLogFileInputStream.ValidateEditLog
(logFile);
NUnit.Framework.Assert.IsTrue(validation.HasCorruptHeader());
}
internal static void AddStreamsToCollectionFromFiles(ICollection <FileJournalManager.EditLogFile
> elfs, ICollection <EditLogInputStream> streams, long fromTxId, bool inProgressOk
)
{
foreach (FileJournalManager.EditLogFile elf in elfs)
{
if (elf.IsInProgress())
{
if (!inProgressOk)
{
Log.Debug("passing over " + elf + " because it is in progress " + "and we are ignoring in-progress logs."
);
continue;
}
try
{
elf.ValidateLog();
}
catch (IOException e)
{
Log.Error("got IOException while trying to validate header of " + elf + ". Skipping."
, e);
continue;
}
}
if (elf.lastTxId < fromTxId)
{
System.Diagnostics.Debug.Assert(elf.lastTxId != HdfsConstants.InvalidTxid);
Log.Debug("passing over " + elf + " because it ends at " + elf.lastTxId + ", but we only care about transactions "
+ "as new as " + fromTxId);
continue;
}
EditLogFileInputStream elfis = new EditLogFileInputStream(elf.GetFile(), elf.GetFirstTxId
(), elf.GetLastTxId(), elf.IsInProgress());
Log.Debug("selecting edit log stream " + elf);
streams.AddItem(elfis);
}
}
/// <exception cref="System.IO.IOException"/>
internal static void RunEditLogTest(TestNameNodeRecovery.EditLogTestSetup elts)
{
FilePath TestLogName = new FilePath(TestDir, "test_edit_log");
FSEditLogOp.OpInstanceCache cache = new FSEditLogOp.OpInstanceCache();
EditLogFileOutputStream elfos = null;
EditLogFileInputStream elfis = null;
try
{
elfos = new EditLogFileOutputStream(new Configuration(), TestLogName, 0);
elfos.Create(NameNodeLayoutVersion.CurrentLayoutVersion);
elts.AddTransactionsToLog(elfos, cache);
elfos.SetReadyToFlush();
elfos.FlushAndSync(true);
elfos.Close();
elfos = null;
elfis = new EditLogFileInputStream(TestLogName);
elfis.SetMaxOpSize(elts.GetMaxOpSize());
// reading through normally will get you an exception
ICollection <long> validTxIds = elts.GetValidTxIds();
FSEditLogOp op = null;
long prevTxId = 0;
try
{
while (true)
{
op = elfis.NextOp();
if (op == null)
{
break;
}
Log.Debug("read txid " + op.txid);
if (!validTxIds.Contains(op.GetTransactionId()))
{
NUnit.Framework.Assert.Fail("read txid " + op.GetTransactionId() + ", which we did not expect to find."
);
}
validTxIds.Remove(op.GetTransactionId());
prevTxId = op.GetTransactionId();
}
if (elts.GetLastValidTxId() != -1)
{
NUnit.Framework.Assert.Fail("failed to throw IoException as expected");
}
}
catch (IOException)
{
if (elts.GetLastValidTxId() == -1)
{
NUnit.Framework.Assert.Fail("expected all transactions to be valid, but got exception "
+ "on txid " + prevTxId);
}
else
{
NUnit.Framework.Assert.AreEqual(prevTxId, elts.GetLastValidTxId());
}
}
if (elts.GetLastValidTxId() != -1)
{
// let's skip over the bad transaction
op = null;
prevTxId = 0;
try
{
while (true)
{
op = elfis.NextValidOp();
if (op == null)
{
break;
}
prevTxId = op.GetTransactionId();
NUnit.Framework.Assert.IsTrue(validTxIds.Remove(op.GetTransactionId()));
}
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("caught IOException while trying to skip over bad " +
"transaction. message was " + e.Message + "\nstack trace\n" + StringUtils.StringifyException
(e));
}
}
// We should have read every valid transaction.
NUnit.Framework.Assert.IsTrue(validTxIds.IsEmpty());
}
finally
{
IOUtils.Cleanup(Log, elfos, elfis);
}
}