public virtual void TestReaderWhileAnotherWrites() { QuorumJournalManager readerQjm = CloseLater(CreateSpyingQJM()); IList <EditLogInputStream> streams = Lists.NewArrayList(); readerQjm.SelectInputStreams(streams, 0, false); NUnit.Framework.Assert.AreEqual(0, streams.Count); QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true); readerQjm.SelectInputStreams(streams, 0, false); try { NUnit.Framework.Assert.AreEqual(1, streams.Count); // Validate the actual stream contents. EditLogInputStream stream = streams[0]; NUnit.Framework.Assert.AreEqual(1, stream.GetFirstTxId()); NUnit.Framework.Assert.AreEqual(3, stream.GetLastTxId()); QJMTestUtil.VerifyEdits(streams, 1, 3); NUnit.Framework.Assert.IsNull(stream.ReadOp()); } finally { IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0])); streams.Clear(); } // Ensure correct results when there is a stream in-progress, but we don't // ask for in-progress. QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, false); readerQjm.SelectInputStreams(streams, 0, false); try { NUnit.Framework.Assert.AreEqual(1, streams.Count); EditLogInputStream stream = streams[0]; NUnit.Framework.Assert.AreEqual(1, stream.GetFirstTxId()); NUnit.Framework.Assert.AreEqual(3, stream.GetLastTxId()); QJMTestUtil.VerifyEdits(streams, 1, 3); } finally { IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0])); streams.Clear(); } // TODO: check results for selectInputStreams with inProgressOK = true. // This doesn't currently work, due to a bug where RedundantEditInputStream // throws an exception if there are any unvalidated in-progress edits in the list! // But, it shouldn't be necessary for current use cases. qjm.FinalizeLogSegment(4, 6); readerQjm.SelectInputStreams(streams, 0, false); try { NUnit.Framework.Assert.AreEqual(2, streams.Count); NUnit.Framework.Assert.AreEqual(4, streams[1].GetFirstTxId()); NUnit.Framework.Assert.AreEqual(6, streams[1].GetLastTxId()); QJMTestUtil.VerifyEdits(streams, 1, 6); } finally { IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0])); streams.Clear(); } }
/// <exception cref="System.IO.IOException"/> public virtual void SelectInputStreams(ICollection <EditLogInputStream> streams, long fromTxnId, bool inProgressOk) { QuorumCall <AsyncLogger, RemoteEditLogManifest> q = loggers.GetEditLogManifest(fromTxnId , inProgressOk); IDictionary <AsyncLogger, RemoteEditLogManifest> resps = loggers.WaitForWriteQuorum (q, selectInputStreamsTimeoutMs, "selectInputStreams"); Log.Debug("selectInputStream manifests:\n" + Joiner.On("\n").WithKeyValueSeparator (": ").Join(resps)); PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream >(64, JournalSet.EditLogInputStreamComparator); foreach (KeyValuePair <AsyncLogger, RemoteEditLogManifest> e in resps) { AsyncLogger logger = e.Key; RemoteEditLogManifest manifest = e.Value; foreach (RemoteEditLog remoteLog in manifest.GetLogs()) { Uri url = logger.BuildURLToFetchLogs(remoteLog.GetStartTxId()); EditLogInputStream elis = EditLogFileInputStream.FromUrl(connectionFactory, url, remoteLog.GetStartTxId(), remoteLog.GetEndTxId(), remoteLog.IsInProgress()); allStreams.AddItem(elis); } } JournalSet.ChainAndMakeRedundantStreams(streams, allStreams, fromTxnId); }
/// <summary> /// Verify that the given list of streams contains exactly the range of /// transactions specified, inclusive. /// </summary> /// <exception cref="System.IO.IOException"/> public static void VerifyEdits(IList <EditLogInputStream> streams, int firstTxnId, int lastTxnId) { IEnumerator <EditLogInputStream> iter = streams.GetEnumerator(); NUnit.Framework.Assert.IsTrue(iter.HasNext()); EditLogInputStream stream = iter.Next(); for (int expected = firstTxnId; expected <= lastTxnId; expected++) { FSEditLogOp op = stream.ReadOp(); while (op == null) { NUnit.Framework.Assert.IsTrue("Expected to find txid " + expected + ", " + "but no more streams available to read from" , iter.HasNext()); stream = iter.Next(); op = stream.ReadOp(); } NUnit.Framework.Assert.AreEqual(FSEditLogOpCodes.OpMkdir, op.opCode); NUnit.Framework.Assert.AreEqual(expected, op.GetTransactionId()); } NUnit.Framework.Assert.IsNull(stream.ReadOp()); NUnit.Framework.Assert.IsFalse("Expected no more txns after " + lastTxnId + " but more streams are available" , iter.HasNext()); }
/// <exception cref="System.IO.IOException"/> internal static OfflineEditsLoader CreateLoader(OfflineEditsVisitor visitor, string inputFileName, bool xmlInput, OfflineEditsViewer.Flags flags) { if (xmlInput) { return(new OfflineEditsXmlLoader(visitor, new FilePath(inputFileName), flags)); } else { FilePath file = null; EditLogInputStream elis = null; OfflineEditsLoader loader = null; try { file = new FilePath(inputFileName); elis = new EditLogFileInputStream(file, HdfsConstants.InvalidTxid, HdfsConstants. InvalidTxid, false); loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.Close(); } } return(loader); } }
/// <summary>Constructor</summary> public OfflineEditsBinaryLoader(OfflineEditsVisitor visitor, EditLogInputStream inputStream , OfflineEditsViewer.Flags flags) { this.visitor = visitor; this.inputStream = inputStream; this.fixTxIds = flags.GetFixTxIds(); this.recoveryMode = flags.GetRecoveryMode(); this.nextTxId = -1; }