public virtual void TestReadURL() { HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>(); Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(FakeLogData)).When(conn).GetInputStream (); Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode (); Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ToString(FakeLogData.Length)).When (conn).GetHeaderField("Content-Length"); URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>(); Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito .Any <Uri>(), Matchers.AnyBoolean()); Uri url = new Uri("http://localhost/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.FromUrl(factory, url, HdfsConstants .InvalidTxid, HdfsConstants.InvalidTxid, false); // Read the edit log and verify that we got all of the data. EnumMap <FSEditLogOpCodes, Holder <int> > counts = FSImageTestUtil.CountEditLogOpTypes (elis); Assert.AssertThat(counts[FSEditLogOpCodes.OpAdd].held, CoreMatchers.Is(1)); Assert.AssertThat(counts[FSEditLogOpCodes.OpSetGenstampV1].held, CoreMatchers.Is( 1)); Assert.AssertThat(counts[FSEditLogOpCodes.OpClose].held, CoreMatchers.Is(1)); // Check that length header was picked up. NUnit.Framework.Assert.AreEqual(FakeLogData.Length, elis.Length()); elis.Close(); }
/// <summary>Find out where the edit log ends.</summary> /// <remarks> /// Find out where the edit log ends. /// This will update the lastTxId of the EditLogFile or /// mark it as corrupt if it is. /// </remarks> /// <exception cref="System.IO.IOException"/> public virtual void ValidateLog() { FSEditLogLoader.EditLogValidation val = EditLogFileInputStream.ValidateEditLog(file ); this.lastTxId = val.GetEndTxId(); this.hasCorruptHeader = val.HasCorruptHeader(); }
/// <param name="editLog">a path of an edit log file</param> /// <returns>the count of each type of operation in the log file</returns> /// <exception cref="System.Exception">if there is an error reading it</exception> public static EnumMap <FSEditLogOpCodes, Holder <int> > CountEditLogOpTypes(FilePath editLog) { EditLogInputStream elis = new EditLogFileInputStream(editLog); try { return(CountEditLogOpTypes(elis)); } finally { IOUtils.CloseStream(elis); } }
public virtual void TestValidateEmptyEditLog() { FilePath testDir = new FilePath(TestDir, "testValidateEmptyEditLog"); SortedDictionary <long, long> offsetToTxId = Maps.NewTreeMap(); FilePath logFile = PrepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId); // Truncate the file so that there is nothing except the header and // layout flags section. TruncateFile(logFile, 8); FSEditLogLoader.EditLogValidation validation = EditLogFileInputStream.ValidateEditLog (logFile); NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader()); NUnit.Framework.Assert.AreEqual(HdfsConstants.InvalidTxid, validation.GetEndTxId( )); }
public virtual void TestValidateEditLogWithCorruptBody() { FilePath testDir = new FilePath(TestDir, "testValidateEditLogWithCorruptBody"); SortedDictionary <long, long> offsetToTxId = Maps.NewTreeMap(); int NumTxns = 20; FilePath logFile = PrepareUnfinalizedTestEditLog(testDir, NumTxns, offsetToTxId); // Back up the uncorrupted log FilePath logFileBak = new FilePath(testDir, logFile.GetName() + ".bak"); Files.Copy(logFile, logFileBak); FSEditLogLoader.EditLogValidation validation = EditLogFileInputStream.ValidateEditLog (logFile); NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader()); // We expect that there will be an OP_START_LOG_SEGMENT, followed by // NUM_TXNS opcodes, followed by an OP_END_LOG_SEGMENT. NUnit.Framework.Assert.AreEqual(NumTxns + 1, validation.GetEndTxId()); // Corrupt each edit and verify that validation continues to work foreach (KeyValuePair <long, long> entry in offsetToTxId) { long txOffset = entry.Key; long txId = entry.Value; // Restore backup, corrupt the txn opcode Files.Copy(logFileBak, logFile); CorruptByteInFile(logFile, txOffset); validation = EditLogFileInputStream.ValidateEditLog(logFile); long expectedEndTxId = (txId == (NumTxns + 1)) ? NumTxns : (NumTxns + 1); NUnit.Framework.Assert.AreEqual("Failed when corrupting txn opcode at " + txOffset , expectedEndTxId, validation.GetEndTxId()); NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader()); } // Truncate right before each edit and verify that validation continues // to work foreach (KeyValuePair <long, long> entry_1 in offsetToTxId) { long txOffset = entry_1.Key; long txId = entry_1.Value; // Restore backup, corrupt the txn opcode Files.Copy(logFileBak, logFile); TruncateFile(logFile, txOffset); validation = EditLogFileInputStream.ValidateEditLog(logFile); long expectedEndTxId = (txId == 0) ? HdfsConstants.InvalidTxid : (txId - 1); NUnit.Framework.Assert.AreEqual("Failed when corrupting txid " + txId + " txn opcode " + "at " + txOffset, expectedEndTxId, validation.GetEndTxId()); NUnit.Framework.Assert.IsTrue(!validation.HasCorruptHeader()); } }
public virtual void TestValidateEditLogWithCorruptHeader() { FilePath testDir = new FilePath(TestDir, "testValidateEditLogWithCorruptHeader"); SortedDictionary <long, long> offsetToTxId = Maps.NewTreeMap(); FilePath logFile = PrepareUnfinalizedTestEditLog(testDir, 2, offsetToTxId); RandomAccessFile rwf = new RandomAccessFile(logFile, "rw"); try { rwf.Seek(0); rwf.WriteLong(42); } finally { // corrupt header rwf.Close(); } FSEditLogLoader.EditLogValidation validation = EditLogFileInputStream.ValidateEditLog (logFile); NUnit.Framework.Assert.IsTrue(validation.HasCorruptHeader()); }
internal static void AddStreamsToCollectionFromFiles(ICollection <FileJournalManager.EditLogFile > elfs, ICollection <EditLogInputStream> streams, long fromTxId, bool inProgressOk ) { foreach (FileJournalManager.EditLogFile elf in elfs) { if (elf.IsInProgress()) { if (!inProgressOk) { Log.Debug("passing over " + elf + " because it is in progress " + "and we are ignoring in-progress logs." ); continue; } try { elf.ValidateLog(); } catch (IOException e) { Log.Error("got IOException while trying to validate header of " + elf + ". Skipping." , e); continue; } } if (elf.lastTxId < fromTxId) { System.Diagnostics.Debug.Assert(elf.lastTxId != HdfsConstants.InvalidTxid); Log.Debug("passing over " + elf + " because it ends at " + elf.lastTxId + ", but we only care about transactions " + "as new as " + fromTxId); continue; } EditLogFileInputStream elfis = new EditLogFileInputStream(elf.GetFile(), elf.GetFirstTxId (), elf.GetLastTxId(), elf.IsInProgress()); Log.Debug("selecting edit log stream " + elf); streams.AddItem(elfis); } }
/// <exception cref="System.IO.IOException"/> internal static void RunEditLogTest(TestNameNodeRecovery.EditLogTestSetup elts) { FilePath TestLogName = new FilePath(TestDir, "test_edit_log"); FSEditLogOp.OpInstanceCache cache = new FSEditLogOp.OpInstanceCache(); EditLogFileOutputStream elfos = null; EditLogFileInputStream elfis = null; try { elfos = new EditLogFileOutputStream(new Configuration(), TestLogName, 0); elfos.Create(NameNodeLayoutVersion.CurrentLayoutVersion); elts.AddTransactionsToLog(elfos, cache); elfos.SetReadyToFlush(); elfos.FlushAndSync(true); elfos.Close(); elfos = null; elfis = new EditLogFileInputStream(TestLogName); elfis.SetMaxOpSize(elts.GetMaxOpSize()); // reading through normally will get you an exception ICollection <long> validTxIds = elts.GetValidTxIds(); FSEditLogOp op = null; long prevTxId = 0; try { while (true) { op = elfis.NextOp(); if (op == null) { break; } Log.Debug("read txid " + op.txid); if (!validTxIds.Contains(op.GetTransactionId())) { NUnit.Framework.Assert.Fail("read txid " + op.GetTransactionId() + ", which we did not expect to find." ); } validTxIds.Remove(op.GetTransactionId()); prevTxId = op.GetTransactionId(); } if (elts.GetLastValidTxId() != -1) { NUnit.Framework.Assert.Fail("failed to throw IoException as expected"); } } catch (IOException) { if (elts.GetLastValidTxId() == -1) { NUnit.Framework.Assert.Fail("expected all transactions to be valid, but got exception " + "on txid " + prevTxId); } else { NUnit.Framework.Assert.AreEqual(prevTxId, elts.GetLastValidTxId()); } } if (elts.GetLastValidTxId() != -1) { // let's skip over the bad transaction op = null; prevTxId = 0; try { while (true) { op = elfis.NextValidOp(); if (op == null) { break; } prevTxId = op.GetTransactionId(); NUnit.Framework.Assert.IsTrue(validTxIds.Remove(op.GetTransactionId())); } } catch (Exception e) { NUnit.Framework.Assert.Fail("caught IOException while trying to skip over bad " + "transaction. message was " + e.Message + "\nstack trace\n" + StringUtils.StringifyException (e)); } } // We should have read every valid transaction. NUnit.Framework.Assert.IsTrue(validTxIds.IsEmpty()); } finally { IOUtils.Cleanup(Log, elfos, elfis); } }