public static async Task <long?> FindPrevMessagePosition(IPositionedMessagesReader reader, long originalMessagePos) { long nextMessagePos = 0; await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(originalMessagePos, null, MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed, MessagesParserDirection.Forward)), async p => { var msgAtOriginalPos = await p.ReadNext(); if (msgAtOriginalPos != null) { nextMessagePos = msgAtOriginalPos.Position; } else { nextMessagePos = reader.EndPosition; } }); return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(nextMessagePos, null, MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed, MessagesParserDirection.Backward)), async p => { IMessage msg = await p.ReadNext(); if (msg != null) { return msg.Position; } return (long?)null; })); }
/// <summary> /// Finds the first and the last available messages in the reader. /// </summary> /// <param name="reader">Messages reader to read from</param> /// <param name="cachedFirstMessage">When the caller passes non-null value /// the function doesn't search for the first message in the reader and return the /// value precalculated by the client instead. That can be user for optimization: /// if the client is sure that the first message didn't change then it can /// pass the value calculated before. If <paramref name="firstMessage"/> is <value>null</value> /// the function will search for the first message in the reader.</param> /// <param name="firstMessage">When the function returns <paramref name="firstMessage"/> receives /// the message with the smallest available position.</param> /// <param name="lastMessage">When the function returns /// <paramref name="lastMessage"/> receives the message with the largest available position.</param> public static async Task <(IMessage firstMessage, IMessage lastMessage)> GetBoundaryMessages( IPositionedMessagesReader reader, IMessage cachedFirstMessage) { IMessage firstMessage, lastMessage; if (cachedFirstMessage == null) { firstMessage = await ReadNearestMessage(reader, reader.BeginPosition); } else { firstMessage = cachedFirstMessage; } lastMessage = firstMessage; await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.EndPosition, null, MessagesParserFlag.Default, MessagesParserDirection.Backward)), async parser => { IMessage tmp = await parser.ReadNext(); if (tmp != null) { lastMessage = tmp; } }); return(firstMessage, lastMessage); }
public static async Task <long?> FindNextMessagePosition(IPositionedMessagesReader reader, long originalMessagePos) { // Validate the input. if (originalMessagePos < reader.BeginPosition) { return(null); } if (originalMessagePos >= reader.EndPosition) { return(null); } return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(originalMessagePos, null, MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageTimeIsNotNeeded, MessagesParserDirection.Forward, null)), async parser => { if (parser.ReadNext() == null) { return (long?)null; } IMessage p = await parser.ReadNext(); if (p == null) { return null; } return p.Position; })); }
static public async Task <IMessage> ReadNearestMessage(IPositionedMessagesReader reader, long position, MessagesParserFlag flags) { return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(position, null, flags, MessagesParserDirection.Forward)), async parser => { IMessage ret = await parser.ReadNext(); return ret; })); }
async Task CreateUnderlyingParserAndInitJitterBuffer(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory) { CreateParserParams reversedParserParams = originalParams; reversedParserParams.Range = null; reversedParserParams.Direction = GetOppositeDirection(originalParams.Direction); reversedParserParams.Flags |= MessagesParserFlag.DisableMultithreading; int reversedMessagesQueued = 0; await DisposableAsync.Using(await underlyingParserFactory(reversedParserParams), async reversedParser => { var tmp = new List <PostprocessedMessage>(); for (int i = 0; i < jitterBufferSize; ++i) { var tmpMsg = await reversedParser.ReadNextAndPostprocess(); if (tmpMsg.Message == null) { break; } tmp.Add(tmpMsg); } tmp.Reverse(); foreach (var tmpMsg in tmp) { jitterBuffer.Enqueue(new Entry() { data = tmpMsg, index = currentIndex++ }); positionsBuffer.Push(new MessagesPositions(tmpMsg.Message)); ++reversedMessagesQueued; } }); enumerator = await ReadAddMessagesFromRangeCompleteJitterBuffer(underlyingParserFactory).GetEnumerator(); for (int i = 0; i < jitterBufferSize; ++i) { var tmp = await LoadNextMessage(); reversedMessagesQueued -= tmp.DequeuedMessages; if (tmp.LoadedMessage == null) { break; } } for (int i = 0; i < reversedMessagesQueued && jitterBuffer.Count > 0; ++i) { jitterBuffer.Dequeue(); positionsBuffer.Pop(); } }
IEnumerableAsync <PostprocessedMessage> ReadAddMessagesFromRangeCompleteJitterBuffer( Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory) { return(EnumerableAsync.Produce <PostprocessedMessage>(async yieldAsync => { CreateParserParams mainParserParams = originalParams; //mainParserParams.Range = null; await DisposableAsync.Using(await underlyingParserFactory(mainParserParams), async mainParser => { for (; ;) { var msg = await mainParser.ReadNextAndPostprocess(); if (msg.Message == null) { break; } if (!await yieldAsync.YieldAsync(msg)) { break; } } }); CreateParserParams jitterBufferCompletionParams = originalParams; jitterBufferCompletionParams.Flags |= MessagesParserFlag.DisableMultithreading; jitterBufferCompletionParams.Range = null; jitterBufferCompletionParams.StartPosition = originalParams.Direction == MessagesParserDirection.Forward ? originalParams.Range.Value.End : originalParams.Range.Value.Begin; await DisposableAsync.Using(await underlyingParserFactory(jitterBufferCompletionParams), async completionParser => { for (int i = 0; i < jitterBufferSize; ++i) { var msg = await completionParser.ReadNextAndPostprocess(); if (msg.Message == null) { break; } if (!await yieldAsync.YieldAsync(msg)) { break; } } }); })); }
async Task IAsyncLogProviderCommandHandler.ContinueAsynchronously(CommandContext ctx) { var parserFlags = (flags & EnumMessagesFlag.IsSequentialScanningHint) != 0 ? MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading : MessagesParserFlag.None; await DisposableAsync.Using(await ctx.Reader.CreateParser( new CreateParserParams(positionToContinueAsync, null, parserFlags, direction)), async parser => { for (; ;) { ctx.Cancellation.ThrowIfCancellationRequested(); ctx.Preemption.ThrowIfCancellationRequested(); var m = await parser.ReadNext(); if (m == null) { break; } if (!callback(m)) { break; } } }); }
async Task FillCacheRanges(CancellationToken cancellationToken) { using (tracer.NewFrame) using (var perfop = new Profiling.Operation(tracer, "FillRanges")) { bool updateStarted = false; // Iterate through the ranges for (; ;) { cancellationToken.ThrowIfCancellationRequested(); currentRange = buffer.GetNextRangeToFill(); if (currentRange == null) // Nothing to fill { break; } currentMessagesContainer = buffer; tracer.Info("currentRange={0}", currentRange); bool failed = false; try { if (!updateStarted) { tracer.Info("Starting to update the messages."); updateStarted = true; } long messagesRead = 0; ResetFlags(); await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams( currentRange.GetPositionToStartReadingFrom(), currentRange.DesirableRange, MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading, MessagesParserDirection.Forward, postprocessor: null, cancellation: cancellationToken)), async parser => { tracer.Info("parser created"); for (; ;) { cancellationToken.ThrowIfCancellationRequested(); ResetFlags(); if (!await ReadNextMessage(parser)) { cancellationToken.ThrowIfCancellationRequested(); break; } ProcessLastReadMessageAndFlush(); ++messagesRead; } }); tracer.Info("reading finished"); FlushBuffer(); } catch { failed = true; throw; } finally { if (!failed) { tracer.Info("Loading of the range finished successfully. Completing the range."); currentRange.Complete(); tracer.Info("Disposing the range."); } else { tracer.Info("Loading failed. Disposing the range without completion."); } currentRange.Dispose(); currentRange = null; currentMessagesContainer = null; perfop.Milestone("range completed"); } } UpdateLoadedTimeStats(reader); if (updateStarted) { perfop.Milestone("great success"); owner.SetMessagesCache(new AsyncLogProviderDataCache() { Messages = new MessagesContainers.ListBasedCollection( buffer.Forward(0, int.MaxValue).Select(m => m.Message)), MessagesRange = buffer.ActiveRange, }); } } }
public static MessageFilteringResult GetFilteringResultFromPostprocessorResult(object obj, IFilter dummyFilter) { var f = (IFilter)obj; if (f == null) { return new MessageFilteringResult { Action = FilterAction.Exclude } } ; if (f == dummyFilter) { return new MessageFilteringResult { Action = FilterAction.Include } } ; return(new MessageFilteringResult { Action = f.Action, Filter = f }); } }; IEnumerableAsync <SearchResultMessage> Enum() { return(EnumerableAsync.Produce <SearchResultMessage>(async yieldAsync => { Func <IMessagesPostprocessor> postprocessor = () => new MessagesPostprocessor(parserParams.SearchParams, trace, dummyFilter); long searchableRangesLength = 0; int searchableRangesCount = 0; long totalMessagesCount = 0; long totalHitsCount = 0; await EnumSearchableRanges().ForEach(async currentSearchableRange => { searchableRangesLength += currentSearchableRange.Length; ++searchableRangesCount; await DisposableAsync.Using(await CreateParserForSearchableRange(currentSearchableRange, postprocessor), async parser => { long messagesCount = 0; long hitsCount = 0; for (; ;) { var tmp = await parser.ReadNextAndPostprocess(); if (tmp.Message == null) { break; } ++messagesCount; var msg = tmp.Message; var filteringResult = MessagesPostprocessor.GetFilteringResultFromPostprocessorResult( tmp.PostprocessingResult, dummyFilter); if (filteringResult.Action != FilterAction.Exclude) { ++hitsCount; await yieldAsync.YieldAsync(new SearchResultMessage(msg, filteringResult)); } progressAndCancellation.HandleMessageReadingProgress(msg.Position); progressAndCancellation.continuationToken.NextPosition = msg.EndPosition; progressAndCancellation.CheckTextIterationCancellation(); } PrintPctStats(string.Format("hits pct in range {0}", currentSearchableRange), hitsCount, messagesCount); totalMessagesCount += messagesCount; totalHitsCount += hitsCount; }); return true; }); trace.Info("Stats: searchable ranges count: {0}", searchableRangesCount); trace.Info("Stats: ave searchable range len: {0}", searchableRangesCount != 0 ? searchableRangesLength / searchableRangesCount : 0); PrintPctStats("searchable ranges coverage pct", searchableRangesLength, requestedRange.Length); PrintPctStats("hits pct overall", totalHitsCount, totalMessagesCount); await yieldAsync.YieldAsync(new SearchResultMessage(null, new MessageFilteringResult())); })); } void PrintPctStats(string name, long num, long denum) { trace.Info("Stats: {0}: {1:F4}%", name, denum != 0 ? num * 100d / denum : 0d); } IEnumerableAsync <FileRange.Range> EnumSearchableRanges() { return(EnumerableAsync.Produce <FileRange.Range>(async yieldAsync => { var matcher = new PlainTextMatcher(parserParams, textStreamPositioningParams, plainTextSearchOptimizationAllowed, regexFactory); if (!matcher.PlainTextSearchOptimizationPossible) { await yieldAsync.YieldAsync(requestedRange); return; } long?skipRangesDownThisPosition = null; await EnumSearchableRangesCore(matcher).ForEach(async currentRange => { if (skipRangesDownThisPosition == null) { await yieldAsync.YieldAsync(currentRange); } else { long skipRangesDownThisPositionVal = skipRangesDownThisPosition.Value; if (currentRange.End < skipRangesDownThisPositionVal) // todo: < or <= ? { return true; } skipRangesDownThisPosition = null; if (currentRange.Begin < skipRangesDownThisPositionVal) // todo: < or <= ? { await yieldAsync.YieldAsync(new FileRange.Range(skipRangesDownThisPositionVal, currentRange.End)); } else { await yieldAsync.YieldAsync(currentRange); } } return true; }); })); } async Task <IPositionedMessagesParser> CreateParserForSearchableRange( FileRange.Range searchableRange, Func <IMessagesPostprocessor> messagesPostprocessor) { bool disableMultithreading = false; return(await owner.CreateParser(new CreateParserParams( searchableRange.Begin, searchableRange, MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading | (disableMultithreading ? MessagesParserFlag.DisableMultithreading : MessagesParserFlag.None), MessagesParserDirection.Forward, messagesPostprocessor))); } IEnumerableAsync <FileRange.Range> EnumSearchableRangesCore(PlainTextMatcher matcher) { return(EnumerableAsync.Produce <FileRange.Range>(async yieldAsync => { ITextAccess ta = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams); using (var tai = await ta.OpenIterator(requestedRange.Begin, TextAccessDirection.Forward)) { var lastRange = new FileRange.Range(); await IterateMatchRanges( EnumCheckpoints(tai, matcher, progressAndCancellation, trace), // todo: tune next parameter to find the value giving max performance. // On one sample log bigger block was better than many small ones. // Hence quite big threshold. textStreamPositioningParams.AlignmentBlockSize * 8, progressAndCancellation ).ForEach(async r => { var postprocessedRange = await PostprocessHintRange(r, lastRange); lastRange = postprocessedRange; await yieldAsync.YieldAsync(postprocessedRange); return true; }); } })); } async Task <FileRange.Range> PostprocessHintRange(FileRange.Range r, FileRange.Range lastRange) { long fixedBegin = r.Begin; long fixedEnd = r.End; int?inflateRangeBy = null; if (dejitteringParams != null && (parserParams.Flags & MessagesParserFlag.DisableDejitter) == 0) { inflateRangeBy = dejitteringParams.Value.JitterBufferSize; } await aligmentSplitter.BeginSplittingSession(requestedRange, r.End, MessagesParserDirection.Forward); if (await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture)) { fixedEnd = aligmentCapture.EndPosition; if (inflateRangeBy != null) { for (int i = 0; i < inflateRangeBy.Value; ++i) { if (!await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture)) { break; } fixedEnd = aligmentCapture.EndPosition; } } } else { fixedEnd = requestedRange.End; } aligmentSplitter.EndSplittingSession(); await aligmentSplitter.BeginSplittingSession(requestedRange, fixedBegin, MessagesParserDirection.Backward); if (await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture)) { fixedBegin = aligmentCapture.BeginPosition; if (inflateRangeBy != null) { for (int i = 0; i < inflateRangeBy.Value; ++i) { if (!await aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture)) { break; } fixedBegin = aligmentCapture.BeginPosition; } } } aligmentSplitter.EndSplittingSession(); var ret = new FileRange.Range(fixedBegin, fixedEnd); ret = FileRange.Range.Intersect(ret, requestedRange).Common; var lastRangeIntersection = FileRange.Range.Intersect(ret, lastRange); if (lastRangeIntersection.RelativePosition == 0) { ret = lastRangeIntersection.Leftover1Right; } return(ret); }