Пример #1
0
        /* Function: IsSkippable
         * Returns whether the token at the passed index should be skipped based on the <LineBoundsMode>.
         */
        internal bool IsSkippable(int testTokenIndex, int testRawTextIndex, LineBoundsMode boundsMode)
        {
            if (boundsMode == LineBoundsMode.Everything)
            {
                return(false);
            }

            FundamentalType fundamentalType = tokenizer.FundamentalTypeAt(testTokenIndex, testRawTextIndex);

            // Whitespace is skippable for both ExcludeWhitespace and CommentContent
            if (fundamentalType == FundamentalType.Whitespace ||
                fundamentalType == FundamentalType.LineBreak)
            {
                return(true);
            }

            if (boundsMode == LineBoundsMode.ExcludeWhitespace)
            {
                return(false);
            }

            // The only other choice is CommentContent

            CommentParsingType commentParsingType = tokenizer.CommentParsingTypeAt(testTokenIndex);

            return(commentParsingType == CommentParsingType.CommentSymbol ||
                   commentParsingType == CommentParsingType.CommentDecoration);
        }
Пример #2
0
 /// <summary>
 /// Initializes a new instance of the <see cref="FundamentalEventArgs"/> class
 /// </summary>
 /// <param name="fundamentalType">The type of fundamental data</param>
 /// <param name="configuration">Theconfiguration for the data</param>
 /// <param name="dateTimeUtc">The date time this event was fired in UTC</param>
 /// <param name="data">The data contained within this event</param>
 public FundamentalEventArgs(FundamentalType fundamentalType, SubscriptionDataConfig configuration, DateTime dateTimeUtc, IReadOnlyList <BaseData> data)
 {
     FundamentalType = fundamentalType;
     Configuration   = configuration;
     DateTimeUtc     = dateTimeUtc;
     Data            = data;
 }
Пример #3
0
        /* Function: CalculateLines
         * Calculates the <Lines> list for the current tokens.
         */
        protected void CalculateLines()
        {
            // Random guess, almost definitely too low, but that's better than being too high.  Will still be closer than the default.
            lines = new List <Line>(4 + (rawText.Length / 60));

            Line            line;
            int             tokenIndex   = 0;
            int             rawTextIndex = 0;
            FundamentalType previousType = FundamentalType.Null;

            while (tokenIndex < tokenLengths.Count)
            {
                line.TokenLength   = 0;
                line.RawTextLength = 0;

                do
                {
                    line.TokenLength++;
                    line.RawTextLength += tokenLengths[tokenIndex];

                    previousType = FundamentalTypeOf(rawText[rawTextIndex]);

                    rawTextIndex += tokenLengths[tokenIndex];
                    tokenIndex++;
                }while (tokenIndex < tokenLengths.Count && previousType != FundamentalType.LineBreak);

                lines.Add(line);
            }
        }
Пример #4
0
 /// <summary>
 /// Initializes a new instance of the <see cref="FundamentalEventArgs"/> class
 /// </summary>
 /// <param name="fundamentalType">The type of fundamental data</param>
 /// <param name="configuration">Theconfiguration for the data</param>
 /// <param name="dateTimeUtc">The date time this event was fired in UTC</param>
 /// <param name="data">The data contained within this event</param>
 public FundamentalEventArgs(FundamentalType fundamentalType, SubscriptionDataConfig configuration, DateTime dateTimeUtc, IReadOnlyList<BaseData> data)
 {
     FundamentalType = fundamentalType;
     Configuration = configuration;
     DateTimeUtc = dateTimeUtc;
     Data = data;
 }
Пример #5
0
        /// <summary>
        /// Event invocator for the <see cref="Fundamental"/> event
        /// </summary>
        protected virtual void OnFundamental(FundamentalType fundamentalType, DateTime dateTimeUtc, SubscriptionDataConfig configuration, IReadOnlyList <BaseData> data)
        {
            var handler = Fundamental;

            if (handler != null)
            {
                handler(this, new FundamentalEventArgs(fundamentalType, configuration, dateTimeUtc, data));
            }
        }
Пример #6
0
        /* Function: AddModifierBlock
         * Adds a full modifier block marked with <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier>
         * to the type builder.
         */
        public void AddModifierBlock(TokenIterator openingToken, TokenIterator closingToken)
        {
            if ((openingToken.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier &&
                 openingToken.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier) ||
                (closingToken.PrototypeParsingType != PrototypeParsingType.ClosingTypeModifier &&
                 closingToken.PrototypeParsingType != PrototypeParsingType.ClosingParamModifier))
            {
                throw new InvalidOperationException();
            }

            TokenIterator iterator = openingToken;
            TokenIterator end      = closingToken;

            end.Next();

            while (iterator < end)
            {
                FundamentalType thisTokenType = (iterator.Character == '_' ? FundamentalType.Text : iterator.FundamentalType);

                // Do we need to add a space?
                if ((thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Text &&
                     (iterator.Tokenizer != lastTokenIterator.Tokenizer || iterator.TokenIndex != lastTokenIterator.TokenIndex + 1))
                    ||
                    (thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Symbol &&
                     !dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock)))
                {
                    rawText.Append(' ');
                    prototypeParsingTypes.Add(PrototypeParsingType.Null);
                    syntaxHighlightingTypes.Add(SyntaxHighlightingType.Null);
                }

                while (iterator < end)
                {
                    iterator.AppendTokenTo(rawText);
                    prototypeParsingTypes.Add(iterator.PrototypeParsingType);
                    syntaxHighlightingTypes.Add(iterator.SyntaxHighlightingType);

                    iterator.Next();
                }

                lastTokenIterator       = closingToken;
                lastTokenType           = thisTokenType;
                lastSymbolWasBlock      = true;
                dontAddSpaceAfterSymbol = false;

                if (thisTokenType == FundamentalType.Text)
                {
                    pastFirstText = true;
                }
            }
        }
Пример #7
0
        /* Function: AddToken
         * Adds a single token to the type builder.  You must use <AddBlock()> for blocks.
         */
        public void AddToken(TokenIterator iterator)
        {
            if (iterator.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier ||
                iterator.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier)
            {
                throw new InvalidOperationException();
            }

            if (iterator.PrototypeParsingType == PrototypeParsingType.Type ||
                iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier ||
                iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier ||
                iterator.PrototypeParsingType == PrototypeParsingType.ParamModifier ||
                iterator.PrototypeParsingType == PrototypeParsingType.StartOfTuple ||
                iterator.PrototypeParsingType == PrototypeParsingType.EndOfTuple ||
                iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberSeparator ||
                iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberName)
            {
                FundamentalType thisTokenType = (iterator.Character == '_' ? FundamentalType.Text : iterator.FundamentalType);

                // ignore whitespace, we'll create it as necessary
                if (thisTokenType == FundamentalType.Text ||
                    thisTokenType == FundamentalType.Symbol)
                {
                    // Do we need to add a space?
                    if ((thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Text &&
                         (iterator.Tokenizer != lastTokenIterator.Tokenizer || iterator.TokenIndex != lastTokenIterator.TokenIndex + 1))
                        ||
                        (thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Symbol &&
                         !dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock))
                        ||
                        (!lastTokenIterator.IsNull && lastTokenIterator.PrototypeParsingType == PrototypeParsingType.TupleMemberSeparator))
                    {
                        rawText.Append(' ');
                        prototypeParsingTypes.Add(PrototypeParsingType.Null);
                        syntaxHighlightingTypes.Add(SyntaxHighlightingType.Null);
                    }

                    // Special handling for package separators and a few other things
                    if (iterator.FundamentalType == FundamentalType.Symbol)
                    {
                        if (iterator.Character == '.' || iterator.MatchesAcrossTokens("::") ||
                            (iterator.Character == ':' && dontAddSpaceAfterSymbol) ||  // second colon of ::
                            iterator.Character == '%' ||                               // used for MyVar%TYPE or MyTable%ROWTYPE in Oracle's PL/SQL
                            iterator.Character == '"' || iterator.Character == '\'' || // strings in Java annotations like @copyright("me")
                            iterator.Character == '@' ||                               // tags in Java annotations like @copyright
                            iterator.Character == '(')                                 // parens around tuples
                        {
                            dontAddSpaceAfterSymbol = true;
                        }
                        else
                        {
                            dontAddSpaceAfterSymbol = false;
                        }
                    }

                    iterator.AppendTokenTo(rawText);
                    prototypeParsingTypes.Add(iterator.PrototypeParsingType);
                    syntaxHighlightingTypes.Add(iterator.SyntaxHighlightingType);

                    lastTokenIterator  = iterator;
                    lastTokenType      = thisTokenType;
                    lastSymbolWasBlock = false;

                    if (thisTokenType == FundamentalType.Text)
                    {
                        pastFirstText = true;
                    }
                }
            }
        }
Пример #8
0
        /* Function: HasSimilarSpacing
         * Returns whether the spacing of the tokens between the two iterators matches what would have been built by this
         * class if the tokens were passed through it.
         */
        public static bool HasSimilarSpacing(TokenIterator start, TokenIterator end)
        {
            // ::Package::Name* array of const*[]
            // Single spaces only between words, and between words and prior symbols except for leading symbols and package separators

            TokenIterator iterator = start;

            bool            pastFirstText           = false;
            FundamentalType lastTokenType           = FundamentalType.Null;
            bool            dontAddSpaceAfterSymbol = false;
            bool            lastSymbolWasBlock      = false;

            while (iterator < end)
            {
                if (iterator.FundamentalType == FundamentalType.Text ||
                    iterator.Character == '_')
                {
                    if (lastTokenType == FundamentalType.Null ||
                        lastTokenType == FundamentalType.Text ||
                        (lastTokenType == FundamentalType.Symbol && (dontAddSpaceAfterSymbol || (!pastFirstText && !lastSymbolWasBlock))) ||
                        lastTokenType == FundamentalType.Whitespace)
                    {
                        pastFirstText = true;
                        lastTokenType = FundamentalType.Text;

                        if (!TryToSkipModifierBlock(ref iterator))
                        {
                            iterator.Next();
                        }
                    }
                    else
                    {
                        return(false);
                    }
                }
                else if (iterator.FundamentalType == FundamentalType.Symbol)
                {
                    if (iterator.Character == ',')
                    {
                        // Quit early on commas, since it could be x[,,] or (x, y), in which case it's not clear whether there should be
                        // a space without making this logic even more complicated.  Just fail out and build a new one.
                        return(false);
                    }

                    if (lastTokenType == FundamentalType.Null ||
                        lastTokenType == FundamentalType.Text ||
                        lastTokenType == FundamentalType.Symbol)
                    {
                        lastTokenType = FundamentalType.Symbol;

                        if (iterator.MatchesAcrossTokens("::"))
                        {
                            lastSymbolWasBlock      = false;
                            dontAddSpaceAfterSymbol = true;
                            iterator.NextByCharacters(2);
                        }
                        else if (iterator.Character == '.' || iterator.Character == '%' ||
                                 iterator.Character == '"' || iterator.Character == '\'' || iterator.Character == '@' ||
                                 iterator.Character == '(')
                        {
                            lastSymbolWasBlock      = false;
                            dontAddSpaceAfterSymbol = true;
                            iterator.Next();
                        }
                        else if (TryToSkipModifierBlock(ref iterator))
                        {
                            lastSymbolWasBlock      = true;
                            dontAddSpaceAfterSymbol = false;
                            // already moved iterator
                        }
                        else
                        {
                            lastSymbolWasBlock      = false;
                            dontAddSpaceAfterSymbol = false;
                            iterator.Next();
                        }
                    }
                    else
                    {
                        return(false);
                    }
                }
                else if (iterator.FundamentalType == FundamentalType.Whitespace &&
                         iterator.Character == ' ' && iterator.RawTextLength == 1)
                {
                    if ((lastTokenType == FundamentalType.Symbol && !dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock)) ||
                        lastTokenType == FundamentalType.Text)
                    {
                        lastTokenType = FundamentalType.Whitespace;
                        iterator.Next();
                    }
                    else
                    {
                        return(false);
                    }
                }
                else
                {
                    return(false);
                }
            }

            return(true);
        }
Пример #9
0
        /* Function: HasSimilarSpacing
         * Returns whether the spacing of the tokens between the two iterators matches what would have been built by this
         * class if the tokens were passed through it.
         */
        public static bool HasSimilarSpacing(TokenIterator start, TokenIterator end)
        {
            // ::Package::Name* array of const*[]
            // Single spaces only between words, and between words and prior symbols except for leading symbols and package separators

            TokenIterator iterator = start;

            bool            pastFirstText           = false;
            FundamentalType lastTokenType           = FundamentalType.Null;
            bool            dontAddSpaceAfterSymbol = false;
            bool            lastSymbolWasBlock      = false;

            while (iterator < end)
            {
                if (iterator.FundamentalType == FundamentalType.Text ||
                    iterator.Character == '_')
                {
                    if (lastTokenType == FundamentalType.Null ||
                        lastTokenType == FundamentalType.Text ||
                        (lastTokenType == FundamentalType.Symbol && (dontAddSpaceAfterSymbol || (!pastFirstText && !lastSymbolWasBlock))) ||
                        lastTokenType == FundamentalType.Whitespace)
                    {
                        pastFirstText = true;
                        lastTokenType = FundamentalType.Text;

                        if (!TryToSkipModifierBlock(ref iterator))
                        {
                            iterator.Next();
                        }
                    }
                    else
                    {
                        return(false);
                    }
                }
                else if (iterator.FundamentalType == FundamentalType.Symbol)
                {
                    if (lastTokenType == FundamentalType.Null ||
                        lastTokenType == FundamentalType.Text ||
                        lastTokenType == FundamentalType.Symbol)
                    {
                        lastTokenType = FundamentalType.Symbol;

                        if (iterator.MatchesAcrossTokens("::"))
                        {
                            lastSymbolWasBlock      = false;
                            dontAddSpaceAfterSymbol = true;
                            iterator.NextByCharacters(2);
                        }
                        else if (iterator.Character == '.' || iterator.Character == '%')
                        {
                            lastSymbolWasBlock      = false;
                            dontAddSpaceAfterSymbol = true;
                            iterator.Next();
                        }
                        else if (TryToSkipModifierBlock(ref iterator))
                        {
                            lastSymbolWasBlock      = true;
                            dontAddSpaceAfterSymbol = false;
                            // already moved iterator
                        }
                        else
                        {
                            lastSymbolWasBlock      = false;
                            dontAddSpaceAfterSymbol = false;
                            iterator.Next();
                        }
                    }
                    else
                    {
                        return(false);
                    }
                }
                else if (iterator.FundamentalType == FundamentalType.Whitespace &&
                         iterator.Character == ' ' && iterator.RawTextLength == 1)
                {
                    if ((lastTokenType == FundamentalType.Symbol && !dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock)) ||
                        lastTokenType == FundamentalType.Text)
                    {
                        lastTokenType = FundamentalType.Whitespace;
                        iterator.Next();
                    }
                    else
                    {
                        return(false);
                    }
                }
                else
                {
                    return(false);
                }
            }

            return(true);
        }
Пример #10
0
 /// <summary>
 /// Event invocator for the <see cref="Fundamental"/> event
 /// </summary>
 protected virtual void OnFundamental(FundamentalType fundamentalType, DateTime dateTimeUtc, SubscriptionDataConfig configuration, IReadOnlyList<BaseData> data)
 {
     var handler = Fundamental;
     if (handler != null) handler(this, new FundamentalEventArgs(fundamentalType, configuration, dateTimeUtc, data));
 }