예제 #1
0
        static bool HeuristicallyDetectWhetherMultithreadingMakesSense(CreateParserParams parserParams,
                                                                       TextStreamPositioningParams textStreamPositioningParams)
        {
#if SILVERLIGHT
            return(false);
#else
            if (System.Environment.ProcessorCount == 1)
            {
                return(false);
            }

            long approxBytesToRead;
            if (parserParams.Direction == MessagesParserDirection.Forward)
            {
                approxBytesToRead = new TextStreamPosition(parserParams.Range.Value.End, textStreamPositioningParams).StreamPositionAlignedToBlockSize
                                    - new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize;
            }
            else
            {
                approxBytesToRead = new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize
                                    - new TextStreamPosition(parserParams.Range.Value.Begin, textStreamPositioningParams).StreamPositionAlignedToBlockSize;
            }
            if (approxBytesToRead < MultiThreadedStrategy <int> .GetBytesToParsePerThread(textStreamPositioningParams) * 2)
            {
                return(false);
            }

            return(true);
#endif
        }
예제 #2
0
        public StreamTextAccess(Stream stream, Encoding streamEncoding, TextStreamPositioningParams textStreamPositioningParams)
        {
            if (stream == null)
            {
                throw new ArgumentNullException(nameof(stream));
            }
            if (streamEncoding == null)
            {
                throw new ArgumentNullException(nameof(streamEncoding));
            }

            this.textStreamPositioningParams      = textStreamPositioningParams;
            this.binaryBufferSize                 = textStreamPositioningParams.AlignmentBlockSize;
            this.maximumSequentialAdvancesAllowed = 4;
            this.textBufferCapacity               = maximumSequentialAdvancesAllowed * binaryBufferSize;

            this.stream          = stream;
            this.encoding        = streamEncoding;
            this.decoder         = this.encoding.GetDecoder();
            this.maxBytesPerChar = GetMaxBytesPerChar(this.encoding);
            this.binaryBuffer    = new byte[binaryBufferSize];
            this.charBuffer      = new char[binaryBufferSize];
            this.textBuffer      = new char[textBufferCapacity];
            this.iterator        = new TextAccessIterator(this);
        }
예제 #3
0
 public TextStreamPosition(long positionValue, TextStreamPositioningParams positioningParams)
 {
     if (positionValue < 0)
     {
         throw new ArgumentOutOfRangeException("positionValue", "position cannot be negative");
     }
     this.positioningParams = positioningParams;
     data = positionValue;
 }
        private static TextStreamPosition FindBound(BoundFinder finder, Stream stm, Encoding encoding, string boundName,
                                                    TextStreamPositioningParams textStreamPositioningParams)
        {
            TextStreamPosition?pos = finder.Find(stm, encoding, textStreamPositioningParams);

            if (!pos.HasValue)
            {
                throw new Exception(string.Format("Cannot detect the {0} of the log", boundName));
            }
            return(pos.Value);
        }
예제 #5
0
        public static async Task <StreamParser> Create(
            IPositionedMessagesReader owner,
            CreateParserParams p,
            TextStreamPositioningParams textStreamPositioningParams,
            IGlobalSettingsAccessor globalSettings,
            StrategiesCache strategiesCache)
        {
            var parser = new StreamParser(owner, p, textStreamPositioningParams, globalSettings, strategiesCache);
            await parser.Strategy.ParserCreated(parser.InitialParams);

            return(parser);
        }
예제 #6
0
        public override TextStreamPosition?Find(Stream stream, Encoding encoding, TextStreamPositioningParams positioningParams)
        {
            EnsureInitialized(encoding);
            stream.Position = 0;
            var tmp = trieNode.Find(stream, 1024 * 1024 * 1);

            if (tmp == null)
            {
                return(null);
            }
            return(new TextStreamPosition(tmp.Value, positioningParams));
        }
예제 #7
0
        public TextStreamPosition(long unalignedStreamPosition, AlignMode mode, TextStreamPositioningParams positioningParams)
        {
            this.positioningParams = positioningParams;
            long containingBlockBegin = unalignedStreamPosition & positioningParams.StreamPosMask;

            if (mode == AlignMode.BeginningOfContainingBlock)
            {
                data = containingBlockBegin;
            }
            else
            {
                data = containingBlockBegin + positioningParams.AlignmentBlockSize;
            }
        }
예제 #8
0
        public SearchingParser(
            IPositionedMessagesReader owner,
            CreateSearchingParserParams p,
            TextStreamPositioningParams textStreamPositioningParams,
            DejitteringParams?dejitteringParams,
            Stream rawStream,
            Encoding streamEncoding,
            bool allowPlainTextSearchOptimization,
            LoadedRegex headerRe,
            ILogSourceThreads threads,
            ITraceSourceFactory traceSourceFactory,
            RegularExpressions.IRegexFactory regexFactory
            )
        {
            this.owner        = owner;
            this.parserParams = p;
            this.plainTextSearchOptimizationAllowed = allowPlainTextSearchOptimization && ((p.Flags & MessagesParserFlag.DisablePlainTextSearchOptimization) == 0);
            this.threads        = threads;
            this.requestedRange = p.Range;
            this.textStreamPositioningParams = textStreamPositioningParams;
            this.dejitteringParams           = dejitteringParams;
            this.rawStream      = rawStream;
            this.streamEncoding = streamEncoding;
            this.regexFactory   = regexFactory;
            this.trace          = traceSourceFactory.CreateTraceSource("LogSource", "srchp." + GetHashCode().ToString("x"));
            this.dummyFilter    = new Filter(FilterAction.Include, "", true, new Search.Options(), null, regexFactory);
            var continuationToken = p.ContinuationToken as ContinuationToken;

            if (continuationToken != null)
            {
                this.requestedRange = new FileRange.Range(continuationToken.NextPosition, requestedRange.End);
            }
            this.aligmentTextAccess      = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams);
            this.aligmentSplitter        = new MessagesSplitter(aligmentTextAccess, headerRe.Clone().Regex, headerRe.GetHeaderReSplitterFlags());
            this.aligmentCapture         = new TextMessageCapture();
            this.progressAndCancellation = new ProgressAndCancellation()
            {
                progressHandler   = p.ProgressHandler,
                cancellationToken = p.Cancellation,
                continuationToken = new ContinuationToken()
                {
                    NextPosition = requestedRange.Begin
                }
            };
            this.impl = Enum();
        }
예제 #9
0
            public PlainTextMatcher(
                CreateSearchingParserParams p,
                TextStreamPositioningParams textStreamPositioningParams,
                bool plainTextSearchOptimizationAllowed,
                RegularExpressions.IRegexFactory regexFactory)
            {
                var fixedOptions = new List <Search.Options>();

                plainTextSearchOptimizationPossible = true;

                if (!plainTextSearchOptimizationAllowed)
                {
                    plainTextSearchOptimizationPossible = false;
                }
                else if (p.SearchParams.Filters.GetDefaultAction() == FilterAction.Exclude)
                {
                    // todo: handle case of multiple positive filters
                    foreach (var filter in p.SearchParams.Filters.Items)
                    {
                        if (filter.Options.Template.Length == 0)
                        {
                            plainTextSearchOptimizationPossible = false;
                        }
                        else
                        {
                            var filterMaxMatchLength = filter.Options.Regexp
                                                                ? textStreamPositioningParams.AlignmentBlockSize
                                                                : filter.Options.Template.Length;
                            maxMatchLength = Math.Max(maxMatchLength, filterMaxMatchLength);
                            var tmp = filter.Options;
                            tmp.ReverseSearch = false;
                            fixedOptions.Add(tmp);
                        };
                    }
                    ;
                }
                else
                {
                    plainTextSearchOptimizationPossible = false;
                }
                if (plainTextSearchOptimizationPossible)
                {
                    opts = fixedOptions.Select(i => i.BeginSearch(regexFactory)).ToArray();
                }
            }
예제 #10
0
        private StreamParser(
            IPositionedMessagesReader owner,
            CreateParserParams p,
            TextStreamPositioningParams textStreamPositioningParams,
            IGlobalSettingsAccessor globalSettings,
            StrategiesCache strategiesCache
            )
        {
            p.EnsureRangeIsSet(owner);

            this.InitialParams = p;

            this.isSequentialReadingParser = (p.Flags & MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading) != 0;
            this.multithreadingDisabled    = (p.Flags & MessagesParserFlag.DisableMultithreading) != 0 ||
                                             globalSettings.MultithreadedParsingDisabled;

            CreateParsingStrategy(p, textStreamPositioningParams, strategiesCache, out this.Strategy);
        }
예제 #11
0
        void CreateParsingStrategy(
            CreateParserParams parserParams,
            TextStreamPositioningParams textStreamPositioningParams,
            StrategiesCache strategiesCache,
            out BaseStrategy strategy)
        {
            bool useMultithreadedStrategy;

            if (multithreadingDisabled)
            {
                useMultithreadedStrategy = false;
            }
            else if (!isSequentialReadingParser)
            {
                useMultithreadedStrategy = false;
            }
            else
            {
                useMultithreadedStrategy = HeuristicallyDetectWhetherMultithreadingMakesSense(parserParams, textStreamPositioningParams);
            }

            //useMultithreadedStrategy = false;

            Lazy <BaseStrategy> strategyToTryFirst;
            Lazy <BaseStrategy> strategyToTrySecond;

            if (useMultithreadedStrategy)
            {
                strategyToTryFirst  = strategiesCache.MultiThreadedStrategy;
                strategyToTrySecond = strategiesCache.SingleThreadedStrategy;
            }
            else
            {
                strategyToTryFirst  = strategiesCache.SingleThreadedStrategy;
                strategyToTrySecond = strategiesCache.MultiThreadedStrategy;
            }

            strategy = strategyToTryFirst.Value;
            if (strategy == null)
            {
                strategy = strategyToTrySecond.Value;
            }
        }
 internal MediaBasedPositionedMessagesReader(
     ILogMedia media,
     BoundFinder beginFinder,
     BoundFinder endFinder,
     MessagesReaderExtensions.XmlInitializationParams extensionsInitData,
     TextStreamPositioningParams textStreamPositioningParams,
     MessagesReaderFlags flags,
     Settings.IGlobalSettingsAccessor settingsAccessor
     )
 {
     this.beginFinder = beginFinder;
     this.endFinder   = endFinder;
     this.media       = media;
     this.textStreamPositioningParams = textStreamPositioningParams;
     this.singleThreadedStrategy      = new Lazy <BaseStrategy>(CreateSingleThreadedStrategy);
     this.multiThreadedStrategy       = new Lazy <BaseStrategy>(CreateMultiThreadedStrategy);
     this.extensions       = new MessagesReaderExtensions(this, extensionsInitData);
     this.flags            = flags;
     this.settingsAccessor = settingsAccessor;
 }
        private static long TextStreamPositionToStreamPosition_Approx(long pos, Encoding encoding, TextStreamPositioningParams positioningParams)
        {
            TextStreamPosition txtPos = new TextStreamPosition(pos, positioningParams);
            int byteCount;

            if (encoding == Encoding.UTF8)
            {
                byteCount = txtPos.CharPositionInsideBuffer;                 // usually utf8 use latin chars. 1 char -> 1 byte.
            }
            else if (encoding == Encoding.Unicode || encoding == Encoding.BigEndianUnicode)
            {
                byteCount = txtPos.CharPositionInsideBuffer * 2;                 // usually UTF16 does not user surrogates. 1 char -> 2 bytes.
            }
            else
            {
                byteCount = encoding.GetMaxByteCount(txtPos.CharPositionInsideBuffer);                 // default formula
            }
            return(txtPos.StreamPositionAlignedToBlockSize + byteCount);
        }
예제 #14
0
        public override TextStreamPosition?Find(Stream stream, Encoding encoding, bool findEnd, TextStreamPositioningParams positioningParams)
        {
            EnsureInitialized(encoding);
            var bufSize = 1 * 1024 * 1024;

            if (findEnd)
            {
                stream.Position = Math.Max(0, stream.Length - bufSize);
            }
            else
            {
                stream.Position = 0;
            }
            var tmp = trieNode.Find(stream, bufSize);

            if (tmp == null)
            {
                return(null);
            }
            return(new TextStreamPosition(tmp.Value, positioningParams));
        }
예제 #15
0
        public TextStreamPosition(long streamPositionAlignedToBufferSize, int textPositionInsideBuffer, TextStreamPositioningParams positioningParams)
        {
            if ((streamPositionAlignedToBufferSize & positioningParams.TextPosMask) != 0)
            {
                throw new ArgumentException("Stream position must be aligned to buffer boundaries");
            }
            if (streamPositionAlignedToBufferSize < 0)
            {
                throw new ArgumentOutOfRangeException("streamPositionAlignedToBufferSize", "position cannot be negative");
            }
            if (textPositionInsideBuffer < 0)
            {
                throw new ArgumentOutOfRangeException("textPositionInsideBuffer", "text position cannot be negative");
            }

            this.positioningParams = positioningParams;
            this.data = streamPositionAlignedToBufferSize + textPositionInsideBuffer;
        }
예제 #16
0
        /// <summary>
        /// Creates valid TextStreamPosition object that points to the charachter that starts at or contains
        /// the byte defined by <paramref name="streamPosition"/>
        /// </summary>
        /// <param name="streamPosition">Stream position. In other words 0-based byte index in stream's data.</param>
        /// <param name="streamEncoding">Manadatory encoding information of the stream</param>
        /// <param name="stream"></param>
        /// <returns>Valid TextStreamPosition object</returns>
        public static TextStreamPosition StreamPositionToTextStreamPosition(long streamPosition, Encoding streamEncoding, Stream stream, TextStreamPositioningParams textStreamPositioningParams)
        {
            if (streamEncoding == null)
            {
                throw new ArgumentNullException("streamEncoding");
            }

            TextStreamPosition tmp = new TextStreamPosition(streamPosition, textStreamPositioningParams);

#if !SILVERLIGHT
            if (streamEncoding.IsSingleByte)
            {
                return(tmp);
            }
#endif
            if (streamEncoding == Encoding.Unicode || streamEncoding == Encoding.BigEndianUnicode)
            {
                return(new TextStreamPosition(tmp.StreamPositionAlignedToBlockSize, tmp.CharPositionInsideBuffer / 2, textStreamPositioningParams));
            }
#if !SILVERLIGHT
            if (streamEncoding == Encoding.UTF32)
            {
                return(new TextStreamPosition(tmp.StreamPositionAlignedToBlockSize, tmp.CharPositionInsideBuffer / 4, textStreamPositioningParams));
            }
#endif

            if (stream == null)
            {
                throw new ArgumentNullException("stream object is required to determine text stream position with given encoding", "stream");
            }

            var boundedStream = new BoundedStream();
            boundedStream.SetStream(stream, false);
            boundedStream.SetBounds(null, streamPosition);
            StreamTextAccess tmpTextAccess = new StreamTextAccess(boundedStream, streamEncoding, textStreamPositioningParams);
            tmpTextAccess.BeginReading(tmp.StreamPositionAlignedToBlockSize, TextAccessDirection.Forward);
            tmp = tmpTextAccess.CharIndexToPosition(tmpTextAccess.BufferString.Length);
            tmpTextAccess.EndReading();

            return(tmp);
        }
예제 #17
0
 public abstract TextStreamPosition?Find(Stream stream, Encoding encoding, bool findEnd, TextStreamPositioningParams positioningParams);