static IEnumerable <Checkpoint> EnumCheckpoints(
ITextAccessIterator tai, PlainTextMatcher matcher,
ProgressAndCancellation progressAndCancellation,
LJTraceSource trace)
{
var advanceTime = new Stopwatch();
long advancesCount = 0;
var matchingTime = new Stopwatch();
long matchCount = 0;
for (;;)
{
StringSlice buf = new StringSlice(tai.CurrentBuffer);
for (int startIdx = 0; ;)
{
matchingTime.Start();
var match = matcher.Match(buf, startIdx);
matchingTime.Stop();
++matchCount;
if (!match.HasValue)
{
break;
}
yield return(new Checkpoint()
{
Position = tai.CharIndexToPosition(match.Value.MatchBegin),
EndPosition = tai.CharIndexToPosition(match.Value.MatchEnd),
IsMatch = true
});
startIdx = match.Value.MatchEnd;
progressAndCancellation.CheckTextIterationCancellation();
}
advanceTime.Start();
bool stop = !tai.Advance(Math.Max(0, tai.CurrentBuffer.Length - matcher.MaxMatchLength));
advanceTime.Stop();
++advancesCount;
if (stop)
{
break;
}
yield return(new Checkpoint()
{
EndPosition = tai.CharIndexToPosition(0),
IsMatch = false
});
progressAndCancellation.CheckTextIterationCancellation();
}
trace.Info("Stats: text buffer matching time: {0} ({1} times)",
matchingTime.Elapsed, matchCount);
trace.Info("Stats: text buffer advance time: {0}/{1}={2}",
advanceTime.Elapsed, advancesCount,
TimeSpan.FromTicks(advanceTime.ElapsedTicks / Math.Max(1, advancesCount)));
}
static IEnumerableAsync <FileRange.Range> IterateMatchRanges(
IEnumerableAsync <Checkpoint> checkpoints, long threshhold, ProgressAndCancellation progressAndCancellation)
{
return(EnumerableAsync.Produce <FileRange.Range>(async yieldAsync =>
{
FileRange.Range?lastMatch = null;
await checkpoints.ForEach(async checkpoint =>
{
if (lastMatch == null)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
}
}
else
{
FileRange.Range lastMatchVal = lastMatch.Value;
if (checkpoint.Position - lastMatchVal.End < threshhold)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(lastMatchVal.Begin, checkpoint.EndPosition);
}
}
else
{
await yieldAsync.YieldAsync(lastMatchVal);
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
lastMatch = null;
}
}
}
return true;
});
if (lastMatch != null)
{
await yieldAsync.YieldAsync(lastMatch.Value);
}
}));
}
static IEnumerable <FileRange.Range> IterateMatchRanges(
IEnumerable <Checkpoint> checkpoints, long threshhold, ProgressAndCancellation progressAndCancellation)
{
FileRange.Range?lastMatch = null;
foreach (var checkpoint in checkpoints)
{
if (lastMatch == null)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
}
}
else
{
FileRange.Range lastMatchVal = lastMatch.Value;
if (checkpoint.Position - lastMatchVal.End < threshhold)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(lastMatchVal.Begin, checkpoint.EndPosition);
}
}
else
{
yield return(lastMatchVal);
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
lastMatch = null;
}
}
}
}
if (lastMatch != null)
{
yield return(lastMatch.Value);
}
}
public SearchingParser(
IPositionedMessagesReader owner,
CreateSearchingParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
DejitteringParams?dejitteringParams,
Stream rawStream,
Encoding streamEncoding,
bool allowPlainTextSearchOptimization,
LoadedRegex headerRe,
ILogSourceThreads threads,
ITraceSourceFactory traceSourceFactory,
RegularExpressions.IRegexFactory regexFactory
)
{
this.owner = owner;
this.parserParams = p;
this.plainTextSearchOptimizationAllowed = allowPlainTextSearchOptimization && ((p.Flags & MessagesParserFlag.DisablePlainTextSearchOptimization) == 0);
this.threads = threads;
this.requestedRange = p.Range;
this.textStreamPositioningParams = textStreamPositioningParams;
this.dejitteringParams = dejitteringParams;
this.rawStream = rawStream;
this.streamEncoding = streamEncoding;
this.regexFactory = regexFactory;
this.trace = traceSourceFactory.CreateTraceSource("LogSource", "srchp." + GetHashCode().ToString("x"));
this.dummyFilter = new Filter(FilterAction.Include, "", true, new Search.Options(), null, regexFactory);
var continuationToken = p.ContinuationToken as ContinuationToken;
if (continuationToken != null)
{
this.requestedRange = new FileRange.Range(continuationToken.NextPosition, requestedRange.End);
}
this.aligmentTextAccess = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams);
this.aligmentSplitter = new MessagesSplitter(aligmentTextAccess, headerRe.Clone().Regex, headerRe.GetHeaderReSplitterFlags());
this.aligmentCapture = new TextMessageCapture();
this.progressAndCancellation = new ProgressAndCancellation()
{
progressHandler = p.ProgressHandler,
cancellationToken = p.Cancellation,
continuationToken = new ContinuationToken()
{
NextPosition = requestedRange.Begin
}
};
this.impl = Enum();
}
