public static async Task <long?> FindPrevMessagePosition(IPositionedMessagesReader reader,
long originalMessagePos)
{
long nextMessagePos = 0;
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(originalMessagePos, null,
MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed,
MessagesParserDirection.Forward)), async p =>
{
var msgAtOriginalPos = await p.ReadNext();
if (msgAtOriginalPos != null)
{
nextMessagePos = msgAtOriginalPos.Position;
}
else
{
nextMessagePos = reader.EndPosition;
}
});
return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(nextMessagePos, null,
MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed,
MessagesParserDirection.Backward)), async p =>
{
IMessage msg = await p.ReadNext();
if (msg != null)
{
return msg.Position;
}
return (long?)null;
}));
}
/// <summary>
/// Finds the first and the last available messages in the reader.
/// </summary>
/// <param name="reader">Messages reader to read from</param>
/// <param name="cachedFirstMessage">When the caller passes non-null value
/// the function doesn't search for the first message in the reader and return the
/// value precalculated by the client instead. That can be user for optimization:
/// if the client is sure that the first message didn't change then it can
/// pass the value calculated before. If <paramref name="firstMessage"/> is <value>null</value>
/// the function will search for the first message in the reader.</param>
/// <param name="firstMessage">When the function returns <paramref name="firstMessage"/> receives
/// the message with the smallest available position.</param>
/// <param name="lastMessage">When the function returns
/// <paramref name="lastMessage"/> receives the message with the largest available position.</param>
public static async Task <(IMessage firstMessage, IMessage lastMessage)> GetBoundaryMessages(
IPositionedMessagesReader reader,
IMessage cachedFirstMessage)
{
IMessage firstMessage, lastMessage;
if (cachedFirstMessage == null)
{
firstMessage = await ReadNearestMessage(reader, reader.BeginPosition);
}
else
{
firstMessage = cachedFirstMessage;
}
lastMessage = firstMessage;
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.EndPosition,
null, MessagesParserFlag.Default, MessagesParserDirection.Backward)), async parser =>
{
IMessage tmp = await parser.ReadNext();
if (tmp != null)
{
lastMessage = tmp;
}
});
return(firstMessage, lastMessage);
}
public static async Task <long?> FindNextMessagePosition(IPositionedMessagesReader reader,
long originalMessagePos)
{
// Validate the input.
if (originalMessagePos < reader.BeginPosition)
{
return(null);
}
if (originalMessagePos >= reader.EndPosition)
{
return(null);
}
return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(originalMessagePos,
null, MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageTimeIsNotNeeded,
MessagesParserDirection.Forward, null)), async parser =>
{
if (parser.ReadNext() == null)
{
return (long?)null;
}
IMessage p = await parser.ReadNext();
if (p == null)
{
return null;
}
return p.Position;
}));
}
static public async Task <IMessage> ReadNearestMessage(IPositionedMessagesReader reader, long position, MessagesParserFlag flags)
{
return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(position, null, flags, MessagesParserDirection.Forward)), async parser =>
{
IMessage ret = await parser.ReadNext();
return ret;
}));
}
async Task CreateUnderlyingParserAndInitJitterBuffer(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory)
{
CreateParserParams reversedParserParams = originalParams;
reversedParserParams.Range = null;
reversedParserParams.Direction = GetOppositeDirection(originalParams.Direction);
reversedParserParams.Flags |= MessagesParserFlag.DisableMultithreading;
int reversedMessagesQueued = 0;
await DisposableAsync.Using(await underlyingParserFactory(reversedParserParams), async reversedParser =>
{
var tmp = new List <PostprocessedMessage>();
for (int i = 0; i < jitterBufferSize; ++i)
{
var tmpMsg = await reversedParser.ReadNextAndPostprocess();
if (tmpMsg.Message == null)
{
break;
}
tmp.Add(tmpMsg);
}
tmp.Reverse();
foreach (var tmpMsg in tmp)
{
jitterBuffer.Enqueue(new Entry()
{
data = tmpMsg, index = currentIndex++
});
positionsBuffer.Push(new MessagesPositions(tmpMsg.Message));
++reversedMessagesQueued;
}
});
enumerator = await ReadAddMessagesFromRangeCompleteJitterBuffer(underlyingParserFactory).GetEnumerator();
for (int i = 0; i < jitterBufferSize; ++i)
{
var tmp = await LoadNextMessage();
reversedMessagesQueued -= tmp.DequeuedMessages;
if (tmp.LoadedMessage == null)
{
break;
}
}
for (int i = 0; i < reversedMessagesQueued && jitterBuffer.Count > 0; ++i)
{
jitterBuffer.Dequeue();
positionsBuffer.Pop();
}
}
IEnumerableAsync <PostprocessedMessage> ReadAddMessagesFromRangeCompleteJitterBuffer(
Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory)
{
return(EnumerableAsync.Produce <PostprocessedMessage>(async yieldAsync =>
{
CreateParserParams mainParserParams = originalParams;
//mainParserParams.Range = null;
await DisposableAsync.Using(await underlyingParserFactory(mainParserParams), async mainParser =>
{
for (; ;)
{
var msg = await mainParser.ReadNextAndPostprocess();
if (msg.Message == null)
{
break;
}
if (!await yieldAsync.YieldAsync(msg))
{
break;
}
}
});
CreateParserParams jitterBufferCompletionParams = originalParams;
jitterBufferCompletionParams.Flags |= MessagesParserFlag.DisableMultithreading;
jitterBufferCompletionParams.Range = null;
jitterBufferCompletionParams.StartPosition = originalParams.Direction == MessagesParserDirection.Forward ? originalParams.Range.Value.End : originalParams.Range.Value.Begin;
await DisposableAsync.Using(await underlyingParserFactory(jitterBufferCompletionParams), async completionParser =>
{
for (int i = 0; i < jitterBufferSize; ++i)
{
var msg = await completionParser.ReadNextAndPostprocess();
if (msg.Message == null)
{
break;
}
if (!await yieldAsync.YieldAsync(msg))
{
break;
}
}
});
}));
}
async Task IAsyncLogProviderCommandHandler.ContinueAsynchronously(CommandContext ctx)
{
var parserFlags = (flags & EnumMessagesFlag.IsSequentialScanningHint) != 0 ? MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading : MessagesParserFlag.None;
await DisposableAsync.Using(await ctx.Reader.CreateParser(
new CreateParserParams(positionToContinueAsync, null, parserFlags, direction)), async parser =>
{
for (; ;)
{
ctx.Cancellation.ThrowIfCancellationRequested();
ctx.Preemption.ThrowIfCancellationRequested();
var m = await parser.ReadNext();
if (m == null)
{
break;
}
if (!callback(m))
{
break;
}
}
});
}
async Task FillCacheRanges(CancellationToken cancellationToken)
{
using (tracer.NewFrame)
using (var perfop = new Profiling.Operation(tracer, "FillRanges"))
{
bool updateStarted = false;
// Iterate through the ranges
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
currentRange = buffer.GetNextRangeToFill();
if (currentRange == null) // Nothing to fill
{
break;
}
currentMessagesContainer = buffer;
tracer.Info("currentRange={0}", currentRange);
bool failed = false;
try
{
if (!updateStarted)
{
tracer.Info("Starting to update the messages.");
updateStarted = true;
}
long messagesRead = 0;
ResetFlags();
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(
currentRange.GetPositionToStartReadingFrom(), currentRange.DesirableRange,
MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading,
MessagesParserDirection.Forward,
postprocessor: null,
cancellation: cancellationToken)), async parser =>
{
tracer.Info("parser created");
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
ResetFlags();
if (!await ReadNextMessage(parser))
{
cancellationToken.ThrowIfCancellationRequested();
break;
}
ProcessLastReadMessageAndFlush();
++messagesRead;
}
});
tracer.Info("reading finished");
FlushBuffer();
}
catch
{
failed = true;
throw;
}
finally
{
if (!failed)
{
tracer.Info("Loading of the range finished successfully. Completing the range.");
currentRange.Complete();
tracer.Info("Disposing the range.");
}
else
{
tracer.Info("Loading failed. Disposing the range without completion.");
}
currentRange.Dispose();
currentRange = null;
currentMessagesContainer = null;
perfop.Milestone("range completed");
}
}
UpdateLoadedTimeStats(reader);
if (updateStarted)
{
perfop.Milestone("great success");
owner.SetMessagesCache(new AsyncLogProviderDataCache()
{
Messages = new MessagesContainers.ListBasedCollection(
buffer.Forward(0, int.MaxValue).Select(m => m.Message)),
MessagesRange = buffer.ActiveRange,
});
}
}
}
public static MessageFilteringResult GetFilteringResultFromPostprocessorResult(object obj, IFilter dummyFilter)
{
var f = (IFilter)obj;
if (f == null)
{
return new MessageFilteringResult {
Action = FilterAction.Exclude
}
}
;
if (f == dummyFilter)
{
return new MessageFilteringResult {
Action = FilterAction.Include
}
}
;
return(new MessageFilteringResult
{
Action = f.Action,
Filter = f
});
}
};
IEnumerableAsync <SearchResultMessage> Enum()
{
return(EnumerableAsync.Produce <SearchResultMessage>(async yieldAsync =>
{
Func <IMessagesPostprocessor> postprocessor =
() => new MessagesPostprocessor(parserParams.SearchParams, trace, dummyFilter);
long searchableRangesLength = 0;
int searchableRangesCount = 0;
long totalMessagesCount = 0;
long totalHitsCount = 0;
await EnumSearchableRanges().ForEach(async currentSearchableRange =>
{
searchableRangesLength += currentSearchableRange.Length;
++searchableRangesCount;
await DisposableAsync.Using(await CreateParserForSearchableRange(currentSearchableRange, postprocessor), async parser =>
{
long messagesCount = 0;
long hitsCount = 0;
for (; ;)
{
var tmp = await parser.ReadNextAndPostprocess();
if (tmp.Message == null)
{
break;
}
++messagesCount;
var msg = tmp.Message;
var filteringResult = MessagesPostprocessor.GetFilteringResultFromPostprocessorResult(
tmp.PostprocessingResult, dummyFilter);
if (filteringResult.Action != FilterAction.Exclude)
{
++hitsCount;
await yieldAsync.YieldAsync(new SearchResultMessage(msg, filteringResult));
}
progressAndCancellation.HandleMessageReadingProgress(msg.Position);
progressAndCancellation.continuationToken.NextPosition = msg.EndPosition;
progressAndCancellation.CheckTextIterationCancellation();
}
PrintPctStats(string.Format("hits pct in range {0}", currentSearchableRange),
hitsCount, messagesCount);
totalMessagesCount += messagesCount;
totalHitsCount += hitsCount;
});
return true;
});
trace.Info("Stats: searchable ranges count: {0}", searchableRangesCount);
trace.Info("Stats: ave searchable range len: {0}",
searchableRangesCount != 0 ? searchableRangesLength / searchableRangesCount : 0);
PrintPctStats("searchable ranges coverage pct", searchableRangesLength, requestedRange.Length);
PrintPctStats("hits pct overall", totalHitsCount, totalMessagesCount);
await yieldAsync.YieldAsync(new SearchResultMessage(null, new MessageFilteringResult()));
}));
}
void PrintPctStats(string name, long num, long denum)
{
trace.Info("Stats: {0}: {1:F4}%", name, denum != 0 ? num * 100d / denum : 0d);
}
IEnumerableAsync <FileRange.Range> EnumSearchableRanges()
{
return(EnumerableAsync.Produce <FileRange.Range>(async yieldAsync =>
{
var matcher = new PlainTextMatcher(parserParams, textStreamPositioningParams, plainTextSearchOptimizationAllowed, regexFactory);
if (!matcher.PlainTextSearchOptimizationPossible)
{
await yieldAsync.YieldAsync(requestedRange);
return;
}
long?skipRangesDownThisPosition = null;
await EnumSearchableRangesCore(matcher).ForEach(async currentRange =>
{
if (skipRangesDownThisPosition == null)
{
await yieldAsync.YieldAsync(currentRange);
}
else
{
long skipRangesDownThisPositionVal = skipRangesDownThisPosition.Value;
if (currentRange.End < skipRangesDownThisPositionVal) // todo: < or <= ?
{
return true;
}
skipRangesDownThisPosition = null;
if (currentRange.Begin < skipRangesDownThisPositionVal) // todo: < or <= ?
{
await yieldAsync.YieldAsync(new FileRange.Range(skipRangesDownThisPositionVal, currentRange.End));
}
else
{
await yieldAsync.YieldAsync(currentRange);
}
}
return true;
});
}));
}
async Task <IPositionedMessagesParser> CreateParserForSearchableRange(
FileRange.Range searchableRange,
Func <IMessagesPostprocessor> messagesPostprocessor)
{
bool disableMultithreading = false;
return(await owner.CreateParser(new CreateParserParams(
searchableRange.Begin, searchableRange,
MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading
| (disableMultithreading ? MessagesParserFlag.DisableMultithreading : MessagesParserFlag.None),
MessagesParserDirection.Forward,
messagesPostprocessor)));
}
IEnumerableAsync <FileRange.Range> EnumSearchableRangesCore(PlainTextMatcher matcher)
{
return(EnumerableAsync.Produce <FileRange.Range>(async yieldAsync =>
{
ITextAccess ta = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams);
using (var tai = await ta.OpenIterator(requestedRange.Begin, TextAccessDirection.Forward))
{
var lastRange = new FileRange.Range();
await IterateMatchRanges(
EnumCheckpoints(tai, matcher, progressAndCancellation, trace),
// todo: tune next parameter to find the value giving max performance.
// On one sample log bigger block was better than many small ones.
// Hence quite big threshold.
textStreamPositioningParams.AlignmentBlockSize * 8,
progressAndCancellation
).ForEach(async r =>
{
var postprocessedRange = await PostprocessHintRange(r, lastRange);
lastRange = postprocessedRange;
await yieldAsync.YieldAsync(postprocessedRange);
return true;
});
}
}));
}
async Task <FileRange.Range> PostprocessHintRange(FileRange.Range r, FileRange.Range lastRange)
{
long fixedBegin = r.Begin;
long fixedEnd = r.End;
int?inflateRangeBy = null;
if (dejitteringParams != null && (parserParams.Flags & MessagesParserFlag.DisableDejitter) == 0)
{
inflateRangeBy = dejitteringParams.Value.JitterBufferSize;
}
await aligmentSplitter.BeginSplittingSession(requestedRange, r.End, MessagesParserDirection.Forward);
if (await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
fixedEnd = aligmentCapture.EndPosition;
if (inflateRangeBy != null)
{
for (int i = 0; i < inflateRangeBy.Value; ++i)
{
if (!await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
break;
}
fixedEnd = aligmentCapture.EndPosition;
}
}
}
else
{
fixedEnd = requestedRange.End;
}
aligmentSplitter.EndSplittingSession();
await aligmentSplitter.BeginSplittingSession(requestedRange, fixedBegin, MessagesParserDirection.Backward);
if (await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
fixedBegin = aligmentCapture.BeginPosition;
if (inflateRangeBy != null)
{
for (int i = 0; i < inflateRangeBy.Value; ++i)
{
if (!await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
break;
}
fixedBegin = aligmentCapture.BeginPosition;
}
}
}
aligmentSplitter.EndSplittingSession();
var ret = new FileRange.Range(fixedBegin, fixedEnd);
ret = FileRange.Range.Intersect(ret, requestedRange).Common;
var lastRangeIntersection = FileRange.Range.Intersect(ret, lastRange);
if (lastRangeIntersection.RelativePosition == 0)
{
ret = lastRangeIntersection.Leftover1Right;
}
return(ret);
}
