static bool HeuristicallyDetectWhetherMultithreadingMakesSense(CreateParserParams parserParams,
TextStreamPositioningParams textStreamPositioningParams)
{
#if SILVERLIGHT
return(false);
#else
if (System.Environment.ProcessorCount == 1)
{
return(false);
}
long approxBytesToRead;
if (parserParams.Direction == MessagesParserDirection.Forward)
{
approxBytesToRead = new TextStreamPosition(parserParams.Range.Value.End, textStreamPositioningParams).StreamPositionAlignedToBlockSize
- new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize;
}
else
{
approxBytesToRead = new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize
- new TextStreamPosition(parserParams.Range.Value.Begin, textStreamPositioningParams).StreamPositionAlignedToBlockSize;
}
if (approxBytesToRead < MultiThreadedStrategy <int> .GetBytesToParsePerThread(textStreamPositioningParams) * 2)
{
return(false);
}
return(true);
#endif
}
public StreamTextAccess(Stream stream, Encoding streamEncoding, TextStreamPositioningParams textStreamPositioningParams)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (streamEncoding == null)
{
throw new ArgumentNullException(nameof(streamEncoding));
}
this.textStreamPositioningParams = textStreamPositioningParams;
this.binaryBufferSize = textStreamPositioningParams.AlignmentBlockSize;
this.maximumSequentialAdvancesAllowed = 4;
this.textBufferCapacity = maximumSequentialAdvancesAllowed * binaryBufferSize;
this.stream = stream;
this.encoding = streamEncoding;
this.decoder = this.encoding.GetDecoder();
this.maxBytesPerChar = GetMaxBytesPerChar(this.encoding);
this.binaryBuffer = new byte[binaryBufferSize];
this.charBuffer = new char[binaryBufferSize];
this.textBuffer = new char[textBufferCapacity];
this.iterator = new TextAccessIterator(this);
}
public TextStreamPosition(long positionValue, TextStreamPositioningParams positioningParams)
{
if (positionValue < 0)
{
throw new ArgumentOutOfRangeException("positionValue", "position cannot be negative");
}
this.positioningParams = positioningParams;
data = positionValue;
}
private static TextStreamPosition FindBound(BoundFinder finder, Stream stm, Encoding encoding, string boundName,
TextStreamPositioningParams textStreamPositioningParams)
{
TextStreamPosition?pos = finder.Find(stm, encoding, textStreamPositioningParams);
if (!pos.HasValue)
{
throw new Exception(string.Format("Cannot detect the {0} of the log", boundName));
}
return(pos.Value);
}
public static async Task <StreamParser> Create(
IPositionedMessagesReader owner,
CreateParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
IGlobalSettingsAccessor globalSettings,
StrategiesCache strategiesCache)
{
var parser = new StreamParser(owner, p, textStreamPositioningParams, globalSettings, strategiesCache);
await parser.Strategy.ParserCreated(parser.InitialParams);
return(parser);
}
public override TextStreamPosition?Find(Stream stream, Encoding encoding, TextStreamPositioningParams positioningParams)
{
EnsureInitialized(encoding);
stream.Position = 0;
var tmp = trieNode.Find(stream, 1024 * 1024 * 1);
if (tmp == null)
{
return(null);
}
return(new TextStreamPosition(tmp.Value, positioningParams));
}
public TextStreamPosition(long unalignedStreamPosition, AlignMode mode, TextStreamPositioningParams positioningParams)
{
this.positioningParams = positioningParams;
long containingBlockBegin = unalignedStreamPosition & positioningParams.StreamPosMask;
if (mode == AlignMode.BeginningOfContainingBlock)
{
data = containingBlockBegin;
}
else
{
data = containingBlockBegin + positioningParams.AlignmentBlockSize;
}
}
public SearchingParser(
IPositionedMessagesReader owner,
CreateSearchingParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
DejitteringParams?dejitteringParams,
Stream rawStream,
Encoding streamEncoding,
bool allowPlainTextSearchOptimization,
LoadedRegex headerRe,
ILogSourceThreads threads,
ITraceSourceFactory traceSourceFactory,
RegularExpressions.IRegexFactory regexFactory
)
{
this.owner = owner;
this.parserParams = p;
this.plainTextSearchOptimizationAllowed = allowPlainTextSearchOptimization && ((p.Flags & MessagesParserFlag.DisablePlainTextSearchOptimization) == 0);
this.threads = threads;
this.requestedRange = p.Range;
this.textStreamPositioningParams = textStreamPositioningParams;
this.dejitteringParams = dejitteringParams;
this.rawStream = rawStream;
this.streamEncoding = streamEncoding;
this.regexFactory = regexFactory;
this.trace = traceSourceFactory.CreateTraceSource("LogSource", "srchp." + GetHashCode().ToString("x"));
this.dummyFilter = new Filter(FilterAction.Include, "", true, new Search.Options(), null, regexFactory);
var continuationToken = p.ContinuationToken as ContinuationToken;
if (continuationToken != null)
{
this.requestedRange = new FileRange.Range(continuationToken.NextPosition, requestedRange.End);
}
this.aligmentTextAccess = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams);
this.aligmentSplitter = new MessagesSplitter(aligmentTextAccess, headerRe.Clone().Regex, headerRe.GetHeaderReSplitterFlags());
this.aligmentCapture = new TextMessageCapture();
this.progressAndCancellation = new ProgressAndCancellation()
{
progressHandler = p.ProgressHandler,
cancellationToken = p.Cancellation,
continuationToken = new ContinuationToken()
{
NextPosition = requestedRange.Begin
}
};
this.impl = Enum();
}
public PlainTextMatcher(
CreateSearchingParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
bool plainTextSearchOptimizationAllowed,
RegularExpressions.IRegexFactory regexFactory)
{
var fixedOptions = new List <Search.Options>();
plainTextSearchOptimizationPossible = true;
if (!plainTextSearchOptimizationAllowed)
{
plainTextSearchOptimizationPossible = false;
}
else if (p.SearchParams.Filters.GetDefaultAction() == FilterAction.Exclude)
{
// todo: handle case of multiple positive filters
foreach (var filter in p.SearchParams.Filters.Items)
{
if (filter.Options.Template.Length == 0)
{
plainTextSearchOptimizationPossible = false;
}
else
{
var filterMaxMatchLength = filter.Options.Regexp
? textStreamPositioningParams.AlignmentBlockSize
: filter.Options.Template.Length;
maxMatchLength = Math.Max(maxMatchLength, filterMaxMatchLength);
var tmp = filter.Options;
tmp.ReverseSearch = false;
fixedOptions.Add(tmp);
};
}
;
}
else
{
plainTextSearchOptimizationPossible = false;
}
if (plainTextSearchOptimizationPossible)
{
opts = fixedOptions.Select(i => i.BeginSearch(regexFactory)).ToArray();
}
}
private StreamParser(
IPositionedMessagesReader owner,
CreateParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
IGlobalSettingsAccessor globalSettings,
StrategiesCache strategiesCache
)
{
p.EnsureRangeIsSet(owner);
this.InitialParams = p;
this.isSequentialReadingParser = (p.Flags & MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading) != 0;
this.multithreadingDisabled = (p.Flags & MessagesParserFlag.DisableMultithreading) != 0 ||
globalSettings.MultithreadedParsingDisabled;
CreateParsingStrategy(p, textStreamPositioningParams, strategiesCache, out this.Strategy);
}
void CreateParsingStrategy(
CreateParserParams parserParams,
TextStreamPositioningParams textStreamPositioningParams,
StrategiesCache strategiesCache,
out BaseStrategy strategy)
{
bool useMultithreadedStrategy;
if (multithreadingDisabled)
{
useMultithreadedStrategy = false;
}
else if (!isSequentialReadingParser)
{
useMultithreadedStrategy = false;
}
else
{
useMultithreadedStrategy = HeuristicallyDetectWhetherMultithreadingMakesSense(parserParams, textStreamPositioningParams);
}
//useMultithreadedStrategy = false;
Lazy <BaseStrategy> strategyToTryFirst;
Lazy <BaseStrategy> strategyToTrySecond;
if (useMultithreadedStrategy)
{
strategyToTryFirst = strategiesCache.MultiThreadedStrategy;
strategyToTrySecond = strategiesCache.SingleThreadedStrategy;
}
else
{
strategyToTryFirst = strategiesCache.SingleThreadedStrategy;
strategyToTrySecond = strategiesCache.MultiThreadedStrategy;
}
strategy = strategyToTryFirst.Value;
if (strategy == null)
{
strategy = strategyToTrySecond.Value;
}
}
internal MediaBasedPositionedMessagesReader(
ILogMedia media,
BoundFinder beginFinder,
BoundFinder endFinder,
MessagesReaderExtensions.XmlInitializationParams extensionsInitData,
TextStreamPositioningParams textStreamPositioningParams,
MessagesReaderFlags flags,
Settings.IGlobalSettingsAccessor settingsAccessor
)
{
this.beginFinder = beginFinder;
this.endFinder = endFinder;
this.media = media;
this.textStreamPositioningParams = textStreamPositioningParams;
this.singleThreadedStrategy = new Lazy <BaseStrategy>(CreateSingleThreadedStrategy);
this.multiThreadedStrategy = new Lazy <BaseStrategy>(CreateMultiThreadedStrategy);
this.extensions = new MessagesReaderExtensions(this, extensionsInitData);
this.flags = flags;
this.settingsAccessor = settingsAccessor;
}
private static long TextStreamPositionToStreamPosition_Approx(long pos, Encoding encoding, TextStreamPositioningParams positioningParams)
{
TextStreamPosition txtPos = new TextStreamPosition(pos, positioningParams);
int byteCount;
if (encoding == Encoding.UTF8)
{
byteCount = txtPos.CharPositionInsideBuffer; // usually utf8 use latin chars. 1 char -> 1 byte.
}
else if (encoding == Encoding.Unicode || encoding == Encoding.BigEndianUnicode)
{
byteCount = txtPos.CharPositionInsideBuffer * 2; // usually UTF16 does not user surrogates. 1 char -> 2 bytes.
}
else
{
byteCount = encoding.GetMaxByteCount(txtPos.CharPositionInsideBuffer); // default formula
}
return(txtPos.StreamPositionAlignedToBlockSize + byteCount);
}
public override TextStreamPosition?Find(Stream stream, Encoding encoding, bool findEnd, TextStreamPositioningParams positioningParams)
{
EnsureInitialized(encoding);
var bufSize = 1 * 1024 * 1024;
if (findEnd)
{
stream.Position = Math.Max(0, stream.Length - bufSize);
}
else
{
stream.Position = 0;
}
var tmp = trieNode.Find(stream, bufSize);
if (tmp == null)
{
return(null);
}
return(new TextStreamPosition(tmp.Value, positioningParams));
}
public TextStreamPosition(long streamPositionAlignedToBufferSize, int textPositionInsideBuffer, TextStreamPositioningParams positioningParams)
{
if ((streamPositionAlignedToBufferSize & positioningParams.TextPosMask) != 0)
{
throw new ArgumentException("Stream position must be aligned to buffer boundaries");
}
if (streamPositionAlignedToBufferSize < 0)
{
throw new ArgumentOutOfRangeException("streamPositionAlignedToBufferSize", "position cannot be negative");
}
if (textPositionInsideBuffer < 0)
{
throw new ArgumentOutOfRangeException("textPositionInsideBuffer", "text position cannot be negative");
}
this.positioningParams = positioningParams;
this.data = streamPositionAlignedToBufferSize + textPositionInsideBuffer;
}
/// <summary>
/// Creates valid TextStreamPosition object that points to the charachter that starts at or contains
/// the byte defined by <paramref name="streamPosition"/>
/// </summary>
/// <param name="streamPosition">Stream position. In other words 0-based byte index in stream's data.</param>
/// <param name="streamEncoding">Manadatory encoding information of the stream</param>
/// <param name="stream"></param>
/// <returns>Valid TextStreamPosition object</returns>
public static TextStreamPosition StreamPositionToTextStreamPosition(long streamPosition, Encoding streamEncoding, Stream stream, TextStreamPositioningParams textStreamPositioningParams)
{
if (streamEncoding == null)
{
throw new ArgumentNullException("streamEncoding");
}
TextStreamPosition tmp = new TextStreamPosition(streamPosition, textStreamPositioningParams);
#if !SILVERLIGHT
if (streamEncoding.IsSingleByte)
{
return(tmp);
}
#endif
if (streamEncoding == Encoding.Unicode || streamEncoding == Encoding.BigEndianUnicode)
{
return(new TextStreamPosition(tmp.StreamPositionAlignedToBlockSize, tmp.CharPositionInsideBuffer / 2, textStreamPositioningParams));
}
#if !SILVERLIGHT
if (streamEncoding == Encoding.UTF32)
{
return(new TextStreamPosition(tmp.StreamPositionAlignedToBlockSize, tmp.CharPositionInsideBuffer / 4, textStreamPositioningParams));
}
#endif
if (stream == null)
{
throw new ArgumentNullException("stream object is required to determine text stream position with given encoding", "stream");
}
var boundedStream = new BoundedStream();
boundedStream.SetStream(stream, false);
boundedStream.SetBounds(null, streamPosition);
StreamTextAccess tmpTextAccess = new StreamTextAccess(boundedStream, streamEncoding, textStreamPositioningParams);
tmpTextAccess.BeginReading(tmp.StreamPositionAlignedToBlockSize, TextAccessDirection.Forward);
tmp = tmpTextAccess.CharIndexToPosition(tmpTextAccess.BufferString.Length);
tmpTextAccess.EndReading();
return(tmp);
}
public abstract TextStreamPosition?Find(Stream stream, Encoding encoding, bool findEnd, TextStreamPositioningParams positioningParams);