public virtual void TestRawWrites() { EditLogFileOutputStream elos = new EditLogFileOutputStream(conf, TestEdits, 0); try { byte[] small = new byte[] { 1, 2, 3, 4, 5, 8, 7 }; elos.Create(NameNodeLayoutVersion.CurrentLayoutVersion); // The first (small) write we make extends the file by 1 MB due to // preallocation. elos.WriteRaw(small, 0, small.Length); FlushAndCheckLength(elos, MinPreallocationLength); // The next small write we make goes into the area that was already // preallocated. elos.WriteRaw(small, 0, small.Length); FlushAndCheckLength(elos, MinPreallocationLength); // Now we write enough bytes so that we exceed the minimum preallocated // length. int BigWriteLength = 3 * MinPreallocationLength; byte[] buf = new byte[4096]; for (int i = 0; i < buf.Length; i++) { buf[i] = 0; } int total = BigWriteLength; while (total > 0) { int toWrite = (total > buf.Length) ? buf.Length : total; elos.WriteRaw(buf, 0, toWrite); total -= toWrite; } FlushAndCheckLength(elos, 4 * MinPreallocationLength); } finally { if (elos != null) { elos.Close(); } } }
/// <exception cref="System.IO.IOException"/> public override EditLogOutputStream StartLogSegment(long txid, int layoutVersion) { lock (this) { try { currentInProgress = NNStorage.GetInProgressEditsFile(sd, txid); EditLogOutputStream stm = new EditLogFileOutputStream(conf, currentInProgress, outputBufferCapacity ); stm.Create(layoutVersion); return(stm); } catch (IOException e) { Log.Warn("Unable to start log segment " + txid + " at " + currentInProgress + ": " + e.GetLocalizedMessage()); errorReporter.ReportErrorOnFile(currentInProgress); throw; } } }
/// <exception cref="System.IO.IOException"/> internal static void RunEditLogTest(TestNameNodeRecovery.EditLogTestSetup elts) { FilePath TestLogName = new FilePath(TestDir, "test_edit_log"); FSEditLogOp.OpInstanceCache cache = new FSEditLogOp.OpInstanceCache(); EditLogFileOutputStream elfos = null; EditLogFileInputStream elfis = null; try { elfos = new EditLogFileOutputStream(new Configuration(), TestLogName, 0); elfos.Create(NameNodeLayoutVersion.CurrentLayoutVersion); elts.AddTransactionsToLog(elfos, cache); elfos.SetReadyToFlush(); elfos.FlushAndSync(true); elfos.Close(); elfos = null; elfis = new EditLogFileInputStream(TestLogName); elfis.SetMaxOpSize(elts.GetMaxOpSize()); // reading through normally will get you an exception ICollection <long> validTxIds = elts.GetValidTxIds(); FSEditLogOp op = null; long prevTxId = 0; try { while (true) { op = elfis.NextOp(); if (op == null) { break; } Log.Debug("read txid " + op.txid); if (!validTxIds.Contains(op.GetTransactionId())) { NUnit.Framework.Assert.Fail("read txid " + op.GetTransactionId() + ", which we did not expect to find." ); } validTxIds.Remove(op.GetTransactionId()); prevTxId = op.GetTransactionId(); } if (elts.GetLastValidTxId() != -1) { NUnit.Framework.Assert.Fail("failed to throw IoException as expected"); } } catch (IOException) { if (elts.GetLastValidTxId() == -1) { NUnit.Framework.Assert.Fail("expected all transactions to be valid, but got exception " + "on txid " + prevTxId); } else { NUnit.Framework.Assert.AreEqual(prevTxId, elts.GetLastValidTxId()); } } if (elts.GetLastValidTxId() != -1) { // let's skip over the bad transaction op = null; prevTxId = 0; try { while (true) { op = elfis.NextValidOp(); if (op == null) { break; } prevTxId = op.GetTransactionId(); NUnit.Framework.Assert.IsTrue(validTxIds.Remove(op.GetTransactionId())); } } catch (Exception e) { NUnit.Framework.Assert.Fail("caught IOException while trying to skip over bad " + "transaction. message was " + e.Message + "\nstack trace\n" + StringUtils.StringifyException (e)); } } // We should have read every valid transaction. NUnit.Framework.Assert.IsTrue(validTxIds.IsEmpty()); } finally { IOUtils.Cleanup(Log, elfos, elfis); } }