private void UpdateSearchCompletionPercentage(
Progress.IProgressEventsSink progress,
long lastHandledPosition,
FileRange.Range fullSearchPositionsRange,
bool skipMessagesCountCheck)
{
if (progress == null)
{
return;
}
if (!skipMessagesCountCheck && (messagesReadSinceCompletionPercentageUpdate % 256) != 0)
{
++messagesReadSinceCompletionPercentageUpdate;
}
else
{
double value;
if (fullSearchPositionsRange.Length > 0)
{
value = Math.Max(0d, (double)(lastHandledPosition - fullSearchPositionsRange.Begin) / (double)fullSearchPositionsRange.Length);
}
else
{
value = 0;
}
progress.SetValue(value);
messagesReadSinceCompletionPercentageUpdate = 0;
}
}
public ParserImpl(LogEntry[] logContent, CreateParserParams parserParams)
{
this.logContent = logContent;
if (parserParams.Range.HasValue)
{
effectiveRange = parserParams.Range.Value;
if (effectiveRange.Begin < 0)
{
effectiveRange = new Range(0, effectiveRange.End);
}
if (effectiveRange.End > logContent.Length)
{
effectiveRange = new Range(effectiveRange.Begin, logContent.Length);
}
}
else
{
effectiveRange = new Range(0, logContent.Length);
}
reverse = parserParams.Direction == MessagesParserDirection.Backward;
if (!reverse)
{
pos = Math.Max(parserParams.StartPosition, effectiveRange.Begin);
}
else
{
pos = Math.Min(parserParams.StartPosition - 1, effectiveRange.End);
}
}
public long PositionRangeToBytes(FileRange.Range range)
{
// Here calculation is not precise: TextStreamPosition cannot be converted to bytes
// directly and efficiently. But this function is used only for statistics so it's ok to
// use approximate calculations here.
var encoding = StreamEncoding;
return(TextStreamPositionToStreamPosition_Approx(range.End, encoding, textStreamPositioningParams) - TextStreamPositionToStreamPosition_Approx(range.Begin, encoding, textStreamPositioningParams));
}
public EnumMessagesHelper(
ILogSource ls,
CancellationToken cancellation,
Progress.IProgressEventsSink progress
)
{
this.ls = ls;
this.cancellation = cancellation;
this.progress = progress;
this.lsRange = ls.Provider.Stats.PositionsRange;
this.lastReadPositon = lsRange.Begin;
}
static IEnumerableAsync <FileRange.Range> IterateMatchRanges(
IEnumerableAsync <Checkpoint> checkpoints, long threshhold, ProgressAndCancellation progressAndCancellation)
{
return(EnumerableAsync.Produce <FileRange.Range>(async yieldAsync =>
{
FileRange.Range?lastMatch = null;
await checkpoints.ForEach(async checkpoint =>
{
if (lastMatch == null)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
}
}
else
{
FileRange.Range lastMatchVal = lastMatch.Value;
if (checkpoint.Position - lastMatchVal.End < threshhold)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(lastMatchVal.Begin, checkpoint.EndPosition);
}
}
else
{
await yieldAsync.YieldAsync(lastMatchVal);
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
lastMatch = null;
}
}
}
return true;
});
if (lastMatch != null)
{
await yieldAsync.YieldAsync(lastMatch.Value);
}
}));
}
static IEnumerable <FileRange.Range> IterateMatchRanges(
IEnumerable <Checkpoint> checkpoints, long threshhold, ProgressAndCancellation progressAndCancellation)
{
FileRange.Range?lastMatch = null;
foreach (var checkpoint in checkpoints)
{
if (lastMatch == null)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
}
}
else
{
FileRange.Range lastMatchVal = lastMatch.Value;
if (checkpoint.Position - lastMatchVal.End < threshhold)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(lastMatchVal.Begin, checkpoint.EndPosition);
}
}
else
{
yield return(lastMatchVal);
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
lastMatch = null;
}
}
}
}
if (lastMatch != null)
{
yield return(lastMatch.Value);
}
}
static bool DetectOutOfRangeCondition(FileRange.Range range, long startPosition, MessagesParserDirection direction)
{
bool posIsOutOfRange = !range.IsInRange(startPosition);
if (posIsOutOfRange &&
direction == MessagesParserDirection.Backward &&
startPosition == range.End)
{
// it's ok to start reading from end position when we move backward
posIsOutOfRange = false;
}
return(posIsOutOfRange);
}
async Task IAsyncLogProviderCommandHandler.ContinueAsynchronously(CommandContext ctx)
{
using (var innerCancellation = CancellationTokenSource.CreateLinkedTokenSource(ctx.Cancellation, ctx.Preemption))
{
var searchRange = new FileRange.Range(
searchParams.FromPosition.GetValueOrDefault(ctx.Reader.BeginPosition), ctx.Reader.EndPosition);
var parserParams = new CreateSearchingParserParams()
{
Range = searchRange,
SearchParams = searchParams,
Cancellation = innerCancellation.Token,
ContinuationToken = continuationToken,
ProgressHandler = pos => UpdateSearchCompletionPercentage(progress, pos, searchRange, false)
};
try
{
var parser = ctx.Reader.CreateSearchingParser(parserParams);
try
{
for (; ;)
{
var msg = await parser.GetNext();
if (msg.Message == null || !callback(msg))
{
break;
}
}
}
finally
{
await parser.Dispose();
}
}
catch (SearchCancelledException e) // todo: impl it for xml reader
{
if (ctx.Preemption.IsCancellationRequested)
{
continuationToken = e.ContinuationToken;
}
throw;
}
}
}
public SearchingParser(
IPositionedMessagesReader owner,
CreateSearchingParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
DejitteringParams?dejitteringParams,
Stream rawStream,
Encoding streamEncoding,
bool allowPlainTextSearchOptimization,
LoadedRegex headerRe,
ILogSourceThreads threads,
ITraceSourceFactory traceSourceFactory,
RegularExpressions.IRegexFactory regexFactory
)
{
this.owner = owner;
this.parserParams = p;
this.plainTextSearchOptimizationAllowed = allowPlainTextSearchOptimization && ((p.Flags & MessagesParserFlag.DisablePlainTextSearchOptimization) == 0);
this.threads = threads;
this.requestedRange = p.Range;
this.textStreamPositioningParams = textStreamPositioningParams;
this.dejitteringParams = dejitteringParams;
this.rawStream = rawStream;
this.streamEncoding = streamEncoding;
this.regexFactory = regexFactory;
this.trace = traceSourceFactory.CreateTraceSource("LogSource", "srchp." + GetHashCode().ToString("x"));
this.dummyFilter = new Filter(FilterAction.Include, "", true, new Search.Options(), null, regexFactory);
var continuationToken = p.ContinuationToken as ContinuationToken;
if (continuationToken != null)
{
this.requestedRange = new FileRange.Range(continuationToken.NextPosition, requestedRange.End);
}
this.aligmentTextAccess = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams);
this.aligmentSplitter = new MessagesSplitter(aligmentTextAccess, headerRe.Clone().Regex, headerRe.GetHeaderReSplitterFlags());
this.aligmentCapture = new TextMessageCapture();
this.progressAndCancellation = new ProgressAndCancellation()
{
progressHandler = p.ProgressHandler,
cancellationToken = p.Cancellation,
continuationToken = new ContinuationToken()
{
NextPosition = requestedRange.Begin
}
};
this.impl = Enum();
}
public void BeginSplittingSession(FileRange.Range range, long startPosition, MessagesParserDirection direction)
{
if (sessionIsOpen)
{
throw new InvalidOperationException("Cannot start more than one reading session for a single splitter");
}
try
{
TryBeginSplittingSession(range, startPosition, direction);
}
catch
{
ReadingSessionCleanup();
throw;
}
sessionIsOpen = true;
}
void TryBeginSplittingSession(FileRange.Range range, long startPosition, MessagesParserDirection direction)
{
bool posIsOutOfRange = DetectOutOfRangeCondition(range, startPosition, direction);
if (!posIsOutOfRange)
{
TextAccessDirection accessDirection = direction == MessagesParserDirection.Forward ?
TextAccessDirection.Forward : TextAccessDirection.Backward;
textIterator = textAccess.OpenIterator(startPosition, accessDirection);
try
{
headerPointer1 = textIterator.PositionToCharIndex(startPosition);
prevHeaderPointer1 = headerPointer1;
}
catch (ArgumentOutOfRangeException)
{
posIsOutOfRange = true;
}
}
headersCounter = 0;
if (posIsOutOfRange)
{
this.range = new FileRange.Range();
SetCachedCurrentBuffer("");
currentMessageHeaderMatch = null;
ReadingSessionCleanup();
}
else
{
this.range = range;
SetCurrentDirection(direction);
UpdateCachedCurrentBuffer();
FindNextMessageStart();
}
}
static long CalcMaxActiveRangeSize(Settings.IGlobalSettingsAccessor settings, FileRange.Range availableRange)
{
long MB = 1024 * 1024;
long sizeThreshold = settings.FileSizes.Threshold * MB;
long partialLoadingSize = settings.FileSizes.WindowSize * MB;
long currentSize = availableRange.End - availableRange.Begin;
if (currentSize < sizeThreshold)
{
return(currentSize);
}
else
{
return(partialLoadingSize);
}
}
bool UpdateAvailableTime(bool incrementalMode)
{
bool itIsFirstUpdate = firstUpdateFlag;
firstUpdateFlag = false;
UpdateBoundsStatus status = reader.UpdateAvailableBounds(incrementalMode);
if (status == UpdateBoundsStatus.NothingUpdated && incrementalMode)
{
return(false);
}
if (status == UpdateBoundsStatus.OldMessagesAreInvalid)
{
incrementalMode = false;
}
// Get new boundary values into temporary variables
IMessage newFirst, newLast;
PositionedMessagesUtils.GetBoundaryMessages(reader, null, out newFirst, out newLast);
if (firstMessage != null)
{
if (newFirst == null || MessageTimestamp.Compare(newFirst.Time, firstMessage.Time) != 0)
{
// The first message we've just read differs from the cached one.
// This means that the log was overwritten. Fall to non-incremental mode.
incrementalMode = false;
}
}
if (!incrementalMode)
{
if (!itIsFirstUpdate)
{
// Reset everything that has been loaded so far
InvalidateEverythingThatHasBeenLoaded();
}
firstMessage = null;
}
// Try to get the dates range for new bounday messages
DateRange newAvailTime = GetAvailableDateRangeHelper(newFirst, newLast);
firstMessage = newFirst;
// Getting here means that the boundaries changed.
// Fire the notfication.
var positionsRange = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
if (!incrementalMode)
{
readerContentsEtag = reader.GetContentsEtag();
}
int contentsEtag =
readerContentsEtag
^ positionsRange.Begin.GetHashCode()
^ positionsRange.End.GetHashCode();
StatsTransaction(stats =>
{
stats.AvailableTime = newAvailTime;
LogProviderStatsFlag f = LogProviderStatsFlag.AvailableTime;
if (incrementalMode)
{
f |= LogProviderStatsFlag.AvailableTimeUpdatedIncrementallyFlag;
}
stats.TotalBytes = reader.SizeInBytes;
f |= LogProviderStatsFlag.BytesCount;
stats.PositionsRange = positionsRange;
f |= LogProviderStatsFlag.PositionsRange;
stats.PositionsRangeUpdatesCount++;
if (stats.ContentsEtag == null || contentsEtag != stats.ContentsEtag.Value)
{
stats.ContentsEtag = contentsEtag;
f |= LogProviderStatsFlag.ContentsEtag;
}
return(f);
});
return(true);
}
public static MessageFilteringResult GetFilteringResultFromPostprocessorResult(object obj)
{
var f = (IFilter)obj;
if (f == null)
{
return new MessageFilteringResult {
Action = FilterAction.Exclude
}
}
;
if (f == dummyFilter)
{
return new MessageFilteringResult {
Action = FilterAction.Include
}
}
;
return(new MessageFilteringResult
{
Action = f.Action,
Filter = f
});
}
};
IEnumerable <SearchResultMessage> Enum()
{
using (var threadsBulkProcessing = threads.UnderlyingThreadsContainer.StartBulkProcessing())
{
Func <IMessagesPostprocessor> postprocessor =
() => new MessagesPostprocessor(parserParams.SearchParams, trace);
long searchableRangesLength = 0;
int searchableRangesCount = 0;
long totalMessagesCount = 0;
long totalHitsCount = 0;
foreach (var currentSearchableRange in EnumSearchableRanges())
{
searchableRangesLength += currentSearchableRange.Length;
++searchableRangesCount;
using (var parser = CreateParserForSearchableRange(currentSearchableRange, postprocessor))
{
long messagesCount = 0;
long hitsCount = 0;
for (;;)
{
var tmp = parser.ReadNextAndPostprocess();
if (tmp.Message == null)
{
break;
}
++messagesCount;
var msg = tmp.Message;
var filteringResult = MessagesPostprocessor.GetFilteringResultFromPostprocessorResult(
tmp.PostprocessingResult);
if (filteringResult.Action != FilterAction.Exclude)
{
++hitsCount;
yield return(new SearchResultMessage(msg, filteringResult));
}
progressAndCancellation.HandleMessageReadingProgress(msg.Position);
progressAndCancellation.continuationToken.NextPosition = msg.EndPosition;
progressAndCancellation.CheckTextIterationCancellation();
}
PrintPctStats(string.Format("hits pct in range {0}", currentSearchableRange),
hitsCount, messagesCount);
totalMessagesCount += messagesCount;
totalHitsCount += hitsCount;
}
}
trace.Info("Stats: searchable ranges count: {0}", searchableRangesCount);
trace.Info("Stats: ave searchable range len: {0}",
searchableRangesCount != 0 ? searchableRangesLength / searchableRangesCount : 0);
PrintPctStats("searchable ranges coverage pct", searchableRangesLength, requestedRange.Length);
PrintPctStats("hits pct overall", totalHitsCount, totalMessagesCount);
}
yield return(new SearchResultMessage(null, new MessageFilteringResult()));
}
void PrintPctStats(string name, long num, long denum)
{
trace.Info("Stats: {0}: {1:F4}%", name, denum != 0 ? num * 100d / denum : 0d);
}
IEnumerable <FileRange.Range> EnumSearchableRanges()
{
var matcher = new PlainTextMatcher(parserParams, textStreamPositioningParams, plainTextSearchOptimizationAllowed);
if (!matcher.PlainTextSearchOptimizationPossible)
{
yield return(requestedRange);
yield break;
}
long?skipRangesDownThisPosition = null;
foreach (var currentRange in EnumSearchableRangesCore(matcher))
{
if (skipRangesDownThisPosition == null)
{
yield return(currentRange);
}
else
{
long skipRangesDownThisPositionVal = skipRangesDownThisPosition.Value;
if (currentRange.End < skipRangesDownThisPositionVal) // todo: < or <= ?
{
continue;
}
skipRangesDownThisPosition = null;
if (currentRange.Begin < skipRangesDownThisPositionVal) // todo: < or <= ?
{
yield return(new FileRange.Range(skipRangesDownThisPositionVal, currentRange.End));
}
else
{
yield return(currentRange);
}
}
}
}
IPositionedMessagesParser CreateParserForSearchableRange(
FileRange.Range searchableRange,
Func <IMessagesPostprocessor> messagesPostprocessor)
{
bool disableMultithreading = false;
return(owner.CreateParser(new CreateParserParams(
searchableRange.Begin, searchableRange,
MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading
| (disableMultithreading ? MessagesParserFlag.DisableMultithreading : MessagesParserFlag.None),
MessagesParserDirection.Forward,
messagesPostprocessor)));
}
IEnumerable <FileRange.Range> EnumSearchableRangesCore(PlainTextMatcher matcher)
{
ITextAccess ta = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams);
using (var tai = ta.OpenIterator(requestedRange.Begin, TextAccessDirection.Forward))
{
var lastRange = new FileRange.Range();
foreach (var r in
IterateMatchRanges(
EnumCheckpoints(tai, matcher, progressAndCancellation, trace),
// todo: tune next parameter to find the value giving max performance.
// On one sample log bigger block was better than many small ones.
// Hence quite big threshold.
textStreamPositioningParams.AlignmentBlockSize * 8,
progressAndCancellation
)
.Select(r => PostprocessHintRange(r, lastRange))
)
{
lastRange = r;
yield return(r);
}
}
}
FileRange.Range PostprocessHintRange(FileRange.Range r, FileRange.Range lastRange)
{
long fixedBegin = r.Begin;
long fixedEnd = r.End;
int?inflateRangeBy = null;
if (dejitteringParams != null && (parserParams.Flags & MessagesParserFlag.DisableDejitter) == 0)
{
inflateRangeBy = dejitteringParams.Value.JitterBufferSize;
}
aligmentSplitter.BeginSplittingSession(requestedRange, r.End, MessagesParserDirection.Forward);
if (aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
fixedEnd = aligmentCapture.EndPosition;
if (inflateRangeBy != null)
{
for (int i = 0; i < inflateRangeBy.Value; ++i)
{
if (!aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
break;
}
fixedEnd = aligmentCapture.EndPosition;
}
}
}
else
{
fixedEnd = requestedRange.End;
}
aligmentSplitter.EndSplittingSession();
aligmentSplitter.BeginSplittingSession(requestedRange, fixedBegin, MessagesParserDirection.Backward);
if (aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
fixedBegin = aligmentCapture.BeginPosition;
if (inflateRangeBy != null)
{
for (int i = 0; i < inflateRangeBy.Value; ++i)
{
if (!aligmentSplitter.GetCurrentMessageAndMoveToNextOne(aligmentCapture))
{
break;
}
fixedBegin = aligmentCapture.BeginPosition;
}
}
}
aligmentSplitter.EndSplittingSession();
var ret = new FileRange.Range(fixedBegin, fixedEnd);
ret = FileRange.Range.Intersect(ret, requestedRange).Common;
var lastRangeIntersection = FileRange.Range.Intersect(ret, lastRange);
if (lastRangeIntersection.RelativePosition == 0)
{
ret = lastRangeIntersection.Leftover1Right;
}
return(ret);
}
public void BeginSplittingSession(FileRange.Range range, long startPosition, MessagesParserDirection direction)
{
if (direction == MessagesParserDirection.Forward)
{
if (startPosition > range.Begin)
{
long?fixedStartPosition = null;
underlyingSplitter.BeginSplittingSession(range, startPosition, MessagesParserDirection.Backward);
try
{
TextMessageCapture capt = new TextMessageCapture();
if (underlyingSplitter.GetCurrentMessageAndMoveToNextOne(capt))
{
fixedStartPosition = capt.EndPosition;
}
}
finally
{
underlyingSplitter.EndSplittingSession();
}
if (fixedStartPosition != null)
{
underlyingSplitter.BeginSplittingSession(range, fixedStartPosition.Value, direction);
try
{
TextMessageCapture capt = new TextMessageCapture();
while (underlyingSplitter.GetCurrentMessageAndMoveToNextOne(capt))
{
if (capt.BeginPosition >= startPosition)
{
break;
}
fixedStartPosition = capt.EndPosition;
}
}
finally
{
underlyingSplitter.EndSplittingSession();
}
startPosition = fixedStartPosition.Value;
}
}
}
else
{
if (startPosition < range.End)
{
long?fixedStartPosition = null;
underlyingSplitter.BeginSplittingSession(range, startPosition, MessagesParserDirection.Forward);
try
{
TextMessageCapture capt = new TextMessageCapture();
if (underlyingSplitter.GetCurrentMessageAndMoveToNextOne(capt))
{
fixedStartPosition = capt.BeginPosition;
}
}
finally
{
underlyingSplitter.EndSplittingSession();
}
if (fixedStartPosition != null)
{
underlyingSplitter.BeginSplittingSession(range, fixedStartPosition.Value, direction);
try
{
TextMessageCapture capt = new TextMessageCapture();
while (underlyingSplitter.GetCurrentMessageAndMoveToNextOne(capt))
{
if (capt.EndPosition <= startPosition)
{
break;
}
fixedStartPosition = capt.BeginPosition;
}
}
finally
{
underlyingSplitter.EndSplittingSession();
}
startPosition = fixedStartPosition.Value;
}
}
}
underlyingSplitter.BeginSplittingSession(range, startPosition, direction);
}