public virtual void TestReaderWhileAnotherWrites()
{
QuorumJournalManager readerQjm = CloseLater(CreateSpyingQJM());
IList <EditLogInputStream> streams = Lists.NewArrayList();
readerQjm.SelectInputStreams(streams, 0, false);
NUnit.Framework.Assert.AreEqual(0, streams.Count);
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
readerQjm.SelectInputStreams(streams, 0, false);
try
{
NUnit.Framework.Assert.AreEqual(1, streams.Count);
// Validate the actual stream contents.
EditLogInputStream stream = streams[0];
NUnit.Framework.Assert.AreEqual(1, stream.GetFirstTxId());
NUnit.Framework.Assert.AreEqual(3, stream.GetLastTxId());
QJMTestUtil.VerifyEdits(streams, 1, 3);
NUnit.Framework.Assert.IsNull(stream.ReadOp());
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
streams.Clear();
}
// Ensure correct results when there is a stream in-progress, but we don't
// ask for in-progress.
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, false);
readerQjm.SelectInputStreams(streams, 0, false);
try
{
NUnit.Framework.Assert.AreEqual(1, streams.Count);
EditLogInputStream stream = streams[0];
NUnit.Framework.Assert.AreEqual(1, stream.GetFirstTxId());
NUnit.Framework.Assert.AreEqual(3, stream.GetLastTxId());
QJMTestUtil.VerifyEdits(streams, 1, 3);
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
streams.Clear();
}
// TODO: check results for selectInputStreams with inProgressOK = true.
// This doesn't currently work, due to a bug where RedundantEditInputStream
// throws an exception if there are any unvalidated in-progress edits in the list!
// But, it shouldn't be necessary for current use cases.
qjm.FinalizeLogSegment(4, 6);
readerQjm.SelectInputStreams(streams, 0, false);
try
{
NUnit.Framework.Assert.AreEqual(2, streams.Count);
NUnit.Framework.Assert.AreEqual(4, streams[1].GetFirstTxId());
NUnit.Framework.Assert.AreEqual(6, streams[1].GetLastTxId());
QJMTestUtil.VerifyEdits(streams, 1, 6);
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
streams.Clear();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void SelectInputStreams(ICollection <EditLogInputStream> streams, long
fromTxnId, bool inProgressOk)
{
QuorumCall <AsyncLogger, RemoteEditLogManifest> q = loggers.GetEditLogManifest(fromTxnId
, inProgressOk);
IDictionary <AsyncLogger, RemoteEditLogManifest> resps = loggers.WaitForWriteQuorum
(q, selectInputStreamsTimeoutMs, "selectInputStreams");
Log.Debug("selectInputStream manifests:\n" + Joiner.On("\n").WithKeyValueSeparator
(": ").Join(resps));
PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
>(64, JournalSet.EditLogInputStreamComparator);
foreach (KeyValuePair <AsyncLogger, RemoteEditLogManifest> e in resps)
{
AsyncLogger logger = e.Key;
RemoteEditLogManifest manifest = e.Value;
foreach (RemoteEditLog remoteLog in manifest.GetLogs())
{
Uri url = logger.BuildURLToFetchLogs(remoteLog.GetStartTxId());
EditLogInputStream elis = EditLogFileInputStream.FromUrl(connectionFactory, url,
remoteLog.GetStartTxId(), remoteLog.GetEndTxId(), remoteLog.IsInProgress());
allStreams.AddItem(elis);
}
}
JournalSet.ChainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}
/// <summary>
/// Verify that the given list of streams contains exactly the range of
/// transactions specified, inclusive.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public static void VerifyEdits(IList <EditLogInputStream> streams, int firstTxnId,
int lastTxnId)
{
IEnumerator <EditLogInputStream> iter = streams.GetEnumerator();
NUnit.Framework.Assert.IsTrue(iter.HasNext());
EditLogInputStream stream = iter.Next();
for (int expected = firstTxnId; expected <= lastTxnId; expected++)
{
FSEditLogOp op = stream.ReadOp();
while (op == null)
{
NUnit.Framework.Assert.IsTrue("Expected to find txid " + expected + ", " + "but no more streams available to read from"
, iter.HasNext());
stream = iter.Next();
op = stream.ReadOp();
}
NUnit.Framework.Assert.AreEqual(FSEditLogOpCodes.OpMkdir, op.opCode);
NUnit.Framework.Assert.AreEqual(expected, op.GetTransactionId());
}
NUnit.Framework.Assert.IsNull(stream.ReadOp());
NUnit.Framework.Assert.IsFalse("Expected no more txns after " + lastTxnId + " but more streams are available"
, iter.HasNext());
}
/// <exception cref="System.IO.IOException"/>
internal static OfflineEditsLoader CreateLoader(OfflineEditsVisitor visitor, string
inputFileName, bool xmlInput, OfflineEditsViewer.Flags flags)
{
if (xmlInput)
{
return(new OfflineEditsXmlLoader(visitor, new FilePath(inputFileName), flags));
}
else
{
FilePath file = null;
EditLogInputStream elis = null;
OfflineEditsLoader loader = null;
try
{
file = new FilePath(inputFileName);
elis = new EditLogFileInputStream(file, HdfsConstants.InvalidTxid, HdfsConstants.
InvalidTxid, false);
loader = new OfflineEditsBinaryLoader(visitor, elis, flags);
}
finally
{
if ((loader == null) && (elis != null))
{
elis.Close();
}
}
return(loader);
}
}
/// <summary>Constructor</summary>
public OfflineEditsBinaryLoader(OfflineEditsVisitor visitor, EditLogInputStream inputStream
, OfflineEditsViewer.Flags flags)
{
this.visitor = visitor;
this.inputStream = inputStream;
this.fixTxIds = flags.GetFixTxIds();
this.recoveryMode = flags.GetRecoveryMode();
this.nextTxId = -1;
}
