예제 #1
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenXmlTagAttributeValue()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (ch == '"' && CurrentState == ScannerState.XmlTagAttributeValueDouble)
            {
                token.Type = TokenType.XmlDoubleQuote;
                GetCh();
                state.Pop(); // Pop XmlTagAttributeValue
            }
            else if (ch == '\'' && CurrentState == ScannerState.XmlTagAttributeValueSingle)
            {
                token.Type = TokenType.XmlSingleQuote;
                GetCh();
                state.Pop(); // Pop XmlTagAttributeValue
            }
            else if (ch == '\'')
            {
                token.Type = TokenType.XmlSingleQuote;
                GetCh();
                state.Pop(); // Pop XmlTagAttributeValue
            }
            else if (ch == '#' && NVDirectiveFollows() && LookAhead(1) != '"')
            {
                token.Type = TokenType.NVDirectiveHash;
                state.Push(ScannerState.NVPreDirective);
                GetCh();
            }
            else if (ch == '$' && NVReferenceFollows())
            {
                token.Type = TokenType.NVDollar;
                state.Push(ScannerState.NVReference);
                GetCh();
            }
            else if (ch == '<' && _options.EnableIntelliSenseTriggerTokens)
            {
                // The IntelliSense mode of the scanner should break out of an XML tag if a new tag is starting.
                // This usually means that the current tag is incomplete when the user is typing.
                state.Pop(); // Pop XmlTagAttributes
                state.Pop(); // Pop XmlTag

                token = GetNextToken();
            }
            else
            {
                token = ReadText(TokenType.XmlAttributeText, delegate
                {
                    return (ch == '"' && CurrentState == ScannerState.XmlTagAttributeValueDouble) ||
                           (ch == '\'' && CurrentState == ScannerState.XmlTagAttributeValueSingle) ||
                           (ch == '#' && NVDirectiveFollows() && LookAhead(1) != '"') ||
                           (ch == '$' && NVReferenceFollows());
                });
            }

            if (eof && !_options.IsLineScanner)
            {
                //TODO throw new ScannerError("End-of-file found but quoted string literal was not closed");
                AddError("End-of-file found but quoted string literal was not closed");
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #2
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenXmlScriptElementContent()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (ch == '<' && LookAhead(1) == '/' &&
                LookAhead(2) == 's' && LookAhead(3) == 'c' && LookAhead(4) == 'r' &&
                LookAhead(5) == 'i' && LookAhead(6) == 'p' && LookAhead(7) == 't' &&
                LookAhead(8) == '>')
            {
                token.Type = TokenType.XmlTagStart;
                GetCh();
                state.Push(ScannerState.XmlTag);
                token.SetEndPosition(lineNo, linePos);
                return token;
            }

            // Scan for text
            token = ReadText(TokenType.XmlText, delegate
            {
                return ch == '<' && LookAhead(1) == '/' && LookAhead(2) == 's' &&
                       LookAhead(3) == 'c' && LookAhead(4) == 'r' && LookAhead(5) == 'i' &&
                       LookAhead(6) == 'p' && LookAhead(7) == 't' && LookAhead(8) == '>';
            });

            if (eof && !_options.IsLineScanner)
            {
                AddError("Expected closing 'script' element");
            }

            return token;
        }
예제 #3
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenXmlTag()
        {
            Token token = new Token();

            // Return whitespace characters between XML attributes as IntelliSense token
            if (_options.EnableIntelliSenseTriggerTokens)
            {
                if (char.IsWhiteSpace(ch))
                {
                    token.Type = TokenType.XmlAttributeMemberSelect;
                    token.Image = ch.ToString();
                    token.Position = CurrentPos;
                    GetCh();
                    return token;
                }
            }
            else
            {
                ConsumeWhiteSpace();
            }

            token.SetStartPosition(lineNo, linePos);

            if (char.IsLetter(ch) || ch == '_' || ch == ':')
            {
                int startPos = pos;
                GetCh();
                while (char.IsLetterOrDigit(ch) || ch == '.' || ch == '-' || ch == '_' || ch == ':')
                {
                    GetCh();
                }

                token.Image = _source.Substring(startPos - 1, pos - startPos);

                if (state.Peek() == ScannerState.XmlTag)
                {
                    token.Type = TokenType.XmlTagName;

                    state.Push(ScannerState.XmlTagAttributes); // It is now in the attributes section
                }
                else
                {
                    token.Type = TokenType.XmlAttributeName;
                }
            }
            else if (ch == '<' && _options.EnableIntelliSenseTriggerTokens)
            {
                // The IntelliSense mode of the scanner should break out of an XML tag if a new tag is starting.
                // This usually means that the current tag is incomplete when the user is typing.
                if (state.Peek() == ScannerState.XmlTagAttributes)
                {
                    state.Pop(); // Pop XmlTagAttributes
                }
                state.Pop(); // Pop XmlTag

                token = GetNextToken();
            }
            else
            {
                switch (ch)
                {
                    case '>':
                        token.Type = TokenType.XmlTagEnd;
                        if (state.Peek() == ScannerState.XmlTagAttributes)
                            state.Pop(); // Pop XmlTagAttributes
                        state.Pop(); // Pop XmlTag
                        break;
                    case '/':
                        token.Type = TokenType.XmlForwardSlash;
                        break;
                    case '=':
                        token.Type = TokenType.XmlEquals;
                        break;
                    case '"':
                        token.Type = TokenType.XmlDoubleQuote;
                        state.Push(ScannerState.XmlTagAttributeValueDouble);
                        break;
                    case '\'':
                        token.Type = TokenType.XmlSingleQuote;
                        state.Push(ScannerState.XmlTagAttributeValueSingle);
                        break;
                    case '?':
                        token.Type = TokenType.XmlQuestionMark;
                        break;
                    case '!':
                        token.Type = TokenType.XmlExclaimationMark;
                        break;
                    case '#':
                        token.Type = TokenType.NVDirectiveHash;
                        state.Push(ScannerState.NVPreDirective);
                        break;
                    case '$':
                        token.Type = TokenType.NVDollar;
                        state.Push(ScannerState.NVReference);
                        break;
                    default:
                        token.Type = TokenType.Error;
                        break;
                }

                if (token.Type != TokenType.Error)
                    GetCh();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #4
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVDictionaryInner()
        {
            ConsumeSingleLineWhiteSpace();

            // If the user has typed an incomplete dictionary then pop out of it so we can attempt to continue
            if (ch == '\n' && Options.EnableIntelliSenseTriggerTokens)
            {
                state.Pop(); // Pop NVDictionaryInner
                state.Pop(); // Pop NVDictionary
                return GetNextToken();
            }

            Token token = new Token();
            token.SetStartPosition(lineNo, linePos);

            if (ch == '}')
            {
                token.Type = TokenType.NVDictionaryRCurly;
                GetCh();
                state.Pop(); // Pop NVDictionaryInner
            }
            else if (char.IsLetter(ch))
            {
                int startPos = pos;
                GetCh();
                while (char.IsLetter(ch))
                {
                    GetCh();
                }
                token.Type = TokenType.NVDictionaryKey;
                token.Image = _source.Substring(startPos - 1, pos - startPos);
            }
            else if (ch == '=')
            {
                token.Type = TokenType.NVDictionaryEquals;
                GetCh();
            }
            else if (ch == ',')
            {
                token.Type = TokenType.NVComma;
                GetCh();
            }
            else
            {
                token = ReadNVelocityToken();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #5
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenXmlComment()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (ch == '-' && LookAhead(1) == '-' && LookAhead(2) == '>')
            {
                token.Type = TokenType.XmlCommentEnd;
                state.Pop(); // XmlComment
                GetCh(); // Skip over '-'
                GetCh(); // Skip over '-'
                GetCh(); // Skip over '>'
            }
            else
            {
                int startPos = pos;
                bool endFound = false;
                while (!eof && !endFound)
                {
                    if (ch == '-' && LookAhead(1) == '-' && LookAhead(2) == '>')
                        endFound = true;
                    else
                        GetCh();
                }

                token.Type = TokenType.XmlComment;
                token.Image = _source.Substring(startPos - 1, pos - startPos);
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #6
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ReadNVelocityToken()
        {
            Token token = new Token();

            ConsumeSingleLineWhiteSpace();

            token.SetStartPosition(lineNo, linePos);

            if (char.IsLetter(ch) || ch == '_')
            {
                if (state.Peek() == ScannerState.NVReference)
                    token = ReadNVelocityReference();
                else
                    token = ReadNVelocityIdentifier();
            }
            else if (char.IsNumber(ch))
            {
                int startPos = pos;
                GetCh();
                while (char.IsDigit(ch))
                    GetCh();
                token.Type = TokenType.NVIntegerLiteral;
                token.Image = _source.Substring(startPos - 1, pos - startPos);
            }
            else
            {
                switch (ch)
                {
                    case '(':
                        token.Type = TokenType.NVLParen;
                        state.Push(ScannerState.NVParens);
                        break;
                    case '[':
                        token.Type = TokenType.NVLBrack;
                        state.Push(ScannerState.NVBracks);
                        break;
                    case '$':
                        token.Type = TokenType.NVDollar;
                        state.Push(ScannerState.NVReference);
                        break;
                    case '=':
                        if (LookAhead(1) == '=')
                        {
                            token.Type = TokenType.NVEqEq;
                            GetCh();
                        }
                        else
                            token.Type = TokenType.NVEq;
                        break;
                    case '>':
                        if (LookAhead(1) == '=')
                        {
                            token.Type = TokenType.NVGte;
                            GetCh();
                        }
                        else
                            token.Type = TokenType.NVGt;
                        break;
                    case '<':
                        if (LookAhead(1) == '=')
                        {
                            token.Type = TokenType.NVLte;
                            GetCh();
                        }
                        else
                            token.Type = TokenType.NVLt;
                        break;
                    case '+':
                        token.Type = TokenType.NVPlus;
                        break;
                    case '-':
                        token.Type = TokenType.NVMinus;
                        break;
                    case '*':
                        token.Type = TokenType.NVMul;
                        break;
                    case '/':
                        token.Type = TokenType.NVDiv;
                        break;
                    case '%':
                        token.Type = TokenType.NVMod;
                        break;
                    case '&':
                        if (LookAhead(1) == '&')
                        {
                            token.Type = TokenType.NVAnd;
                            GetCh();
                        }
                        else
                            token.Type = TokenType.Error;
                        break;
                    case '|':
                        if (LookAhead(1) == '|')
                        {
                            token.Type = TokenType.NVOr;
                            GetCh();
                        }
                        else
                            token.Type = TokenType.Error;
                        break;
                    case '!':
                        token.Type = TokenType.NVNot;
                        break;
                    case ',':
                        token.Type = TokenType.NVComma;
                        break;
                    case '.':
                        if (LookAhead(1) == '.')
                        {
                            token.Type = TokenType.NVDoubleDot;
                            GetCh();
                        }
                        else
                            token.Type = TokenType.NVDot;
                        break;
                    case '"':
                        token.Type = TokenType.NVDoubleQuote;
                        if (LookAhead(1) == '%')
                            state.Push(ScannerState.NVDictionary);
                        else
                            state.Push(ScannerState.NVStringLiteralDouble);
                        break;
                    case '\'':
                        token.Type = TokenType.NVSingleQuote;
                        state.Push(ScannerState.NVStringLiteralSingle);
                        break;
                    default:
                        token.Type = TokenType.Error;
                        break;
                }

                if (token.Type != TokenType.Error)
                    GetCh();
            }
            return token;
        }
예제 #7
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        /// <summary>
        /// Scans the input and returns XML text tokens.
        /// </summary>
        /// <returns>XML text tokens.</returns>
        private Token ScanTokenDefault()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            bool scanXmlText = true;

            if (ch == '<')
            {
                if (LookAhead(1) == '!')
                {
                    if (LookAhead(2) == '-' && LookAhead(3) == '-')
                    {
                        // It is an XML comment
                        int startCommentPos = pos;
                        GetCh(4);
                        if (_options.IsLineScanner)
                        {
                            token.Type = TokenType.XmlCommentStart;
                            state.Push(ScannerState.XmlComment);
                        }
                        else
                        {
                            token.Type = TokenType.XmlComment;
                            ReadXmlComment();
                            token.Image = _source.Substring(startCommentPos - 1, pos - startCommentPos);
                        }
                        token.SetEndPosition(lineNo, linePos);
                        return token;
                    }

                    if (LookAhead(2) == '[' && LookAhead(3) == 'C' && LookAhead(4) == 'D' &&
                        LookAhead(5) == 'A' && LookAhead(6) == 'T' && LookAhead(7) == 'A' &&
                        LookAhead(8) == '[')
                    {
                        // It is an XML CData section
                        GetCh(9);
                        token.Type = TokenType.XmlCDataStart;
                        state.Push(ScannerState.XmlCData);
                        token.SetEndPosition(lineNo, linePos);
                        return token;
                    }
                }

                // It is an XML element
                token.Type = TokenType.XmlTagStart;
                state.Push(ScannerState.XmlTag);
                GetCh();
                scanXmlText = false;
            }
            else if (ch == '#')
            {
                if (LookAhead(1) == '#')
                {
                    GetCh();
                    token.Type = TokenType.NVSingleLineComment;
                    int prevCommentPos = pos - 2;
                    GetCh();
                    while (ch != CR && ch != LF && !eof)
                        GetCh();
                    if (ch == CR)
                        GetCh();
                    if (ch == LF)
                        GetCh();
                    token.Image = _source.Substring(prevCommentPos, pos - prevCommentPos - 1);
                    scanXmlText = false;
                }
                else if (LookAhead(1) == '*')
                {
                    GetCh();
                    if (_options.IsLineScanner)
                    {
                        token.Type = TokenType.NVMultilineCommentStart;
                        state.Push(ScannerState.NVMultilineComment);
                        GetCh();
                    }
                    else
                    {
                        token = ReadNVelocityMultiLineComment();
                    }
                    scanXmlText = false;
                }
                else if (NVDirectiveFollows())
                {
                    GetCh();
                    token.Type = TokenType.NVDirectiveHash;
                    state.Push(ScannerState.NVPreDirective);
                    scanXmlText = false;
                }
            }
            else if (ch == '$' && NVReferenceFollows())
            {
                token.Type = TokenType.NVDollar;
                state.Push(ScannerState.NVReference);
                GetCh();
                scanXmlText = false;
            }

            // If it is not any other type of parseable syntax
            if (scanXmlText)
            {
                token = ReadText(TokenType.XmlText, delegate
                {
                    return ch == '<' ||
                           (ch == '#' && NVDirectiveFollows()) ||
                           (ch == '$' && NVReferenceFollows());
                });
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #8
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVReferenceSelectors()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (char.IsLetter(ch))
            {
                int startPos = pos;
                GetCh();
                while (char.IsLetterOrDigit(ch) || ch == '_' || ch == '-')
                {
                    GetCh();
                }
                token.Type = TokenType.NVIdentifier;
                token.Image = _source.Substring(startPos - 1, pos - startPos);
            }
            else if (ch == '.' && LookAhead(1) != '.') // Do not scan a double dot as a dot
            {
                token.Type = TokenType.NVDot;
                GetCh();
            }
            else if (ch == '(')
            {
                token.Type = TokenType.NVLParen;
                state.Push(ScannerState.NVParens);
                GetCh();
            }
            else if (ch == '}')
            {
                state.Pop(); // Pop NVReferenceSelectors
                state.Pop(); // Pop NVReference

                if (state.Peek() == ScannerState.NVReferenceFormal)
                {
                    token.Type = TokenType.NVReferenceRCurly;
                    state.Pop(); // Pop NVReferenceFormal

                    GetCh();
                }
                else
                {
                    token = GetNextToken();
                }
            }
            else
            {
                state.Pop(); // Pop NVReferenceSelectors
                state.Pop(); // Pop NVReference
                if (state.Peek() == ScannerState.NVReferenceFormal)
                    state.Pop(); // Pop NVReferenceFormal
                token = GetNextToken();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #9
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVStringLiteralDouble()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (ch == '"')
            {
                state.Pop(); // Pop NVStringLiteralDouble
                token.Type = TokenType.NVDoubleQuote;
                GetCh();
                token.SetEndPosition(lineNo, linePos);
                return token;
            }
            else if (ch == '#' && NVDirectiveFollows())
            {
                token.Type = TokenType.NVDirectiveHash;
                state.Push(ScannerState.NVPreDirective);

                GetCh();
                token.SetEndPosition(lineNo, linePos);
                return token;
            }
            else if (ch == '$' && NVReferenceFollows())
            {
                token.Type = TokenType.NVDollar;
                state.Push(ScannerState.NVReference);

                GetCh();
                token.SetEndPosition(lineNo, linePos);
                return token;
            }

            token = ReadText(TokenType.NVStringLiteral, delegate
            {
                return ch == '"' ||
                       (ch == '#' && NVDirectiveFollows()) ||
                       (ch == '$' && NVReferenceFollows());
            });

            if (eof && !_options.IsLineScanner)
            {
                AddError("Expected end of string literal");
                state.Pop();
            }

            return token;
        }
예제 #10
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVPreDirective()
        {
            // If a directive is being started but is followed by another directive this directive is not finished
            if (Options.EnableIntelliSenseTriggerTokens && (char.IsWhiteSpace(ch) || NextCharAfterSingleLineWhiteSpace() == '#'))
            {
                // Exit from this directive
                state.Pop(); // Pop NVPreDirective

                return GetNextToken();
            }

            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            bool hasBraces = false;
            if (ch == '{')
            {
                hasBraces = true;
                GetCh();
            }

            if (char.IsLetter(ch))
            {
                int startPos = pos;
                while (char.IsLetter(ch))
                {
                    GetCh();
                }
                token.Type = TokenType.NVDirectiveName;
                token.Image = _source.Substring(startPos - 1, pos - startPos);

                if (hasBraces)
                {
                    if (ch == '}')
                    {
                        GetCh();
                    }
                    else
                    {
                        AddError("Expected '}' for closing directive name");
                    }
                }

                state.Pop(); // Pop NVPreDirective
                state.Push(ScannerState.NVDirective);
            }
            else
            {
                token.Type = TokenType.Error;
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #11
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVReference()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (char.IsLetter(ch) || ch == '_')
            {
                token = ReadNVelocityReference();
            }
            else if (ch == '!')
            {
                token.Type = TokenType.NVReferenceSilent;
                GetCh();
            }
            else if (ch == '{')
            {
                token.Type = TokenType.NVReferenceLCurly;
                state.Pop(); // Pop NVReference
                state.Push(ScannerState.NVReferenceFormal);
                state.Push(ScannerState.NVReference);
                GetCh();
            }
            else if (ch == '"')
            {
                token.Type = TokenType.NVDoubleQuote;
                state.Pop(); // Pop NVReference
                if (state.Peek() == ScannerState.NVReferenceFormal)
                    state.Pop(); // Pop NVReferenceFormal
                state.Pop(); // Pop NVStringLiteralDouble
                GetCh();
            }
            else
            {
                AddError("Expected reference identifier");
                if (_options.EnableIntelliSenseTriggerTokens)
                {
                    state.Pop(); // Pop NVReference
                    if (state.Peek() == ScannerState.NVReferenceFormal)
                    {
                        state.Pop(); // Pop NVReferenceFormal
                    }
                    return null;
                }
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #12
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVParens()
        {
            Token token = new Token();

            ConsumeSingleLineWhiteSpace();

            // If the user has typed an incomplete directive then pop out of it so we can attempt to continue
            if (ch == '\n' && Options.EnableIntelliSenseTriggerTokens)
            {
                AddError("Incomplete reference.");

                state.Pop(); // Pop NVParens
                if (state.Peek() == ScannerState.NVReferenceSelectors)
                {
                    state.Pop(); // Pop NVReferenceSelectors
                    state.Pop(); // Pop NVReference
                }

                return GetNextToken();
            }

            token.SetStartPosition(lineNo, linePos);

            if (ch == ')')
            {
                token.Type = TokenType.NVRParen;
                GetCh();
                state.Pop(); // Pop NVParens

                // Pop out of the reference if the closing parenthesis isn't followed by a '.'.
                // For example: $var.Method()text
                if (state.Peek() == ScannerState.NVReferenceSelectors && ch != '.')
                {
                    state.Pop(); // Pop NVReferenceSelectors
                    state.Pop(); // Pop NVReference
                }
            }
            else
            {
                token = ReadNVelocityToken();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #13
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVDirectiveParams()
        {
            Token token = new Token();

            ConsumeSingleLineWhiteSpace();

            // If a new directive starts in the directive params and the scanner is running in IntelliSense mode,
            // break out of the directive params
            if (ch == '#' && Options.EnableIntelliSenseTriggerTokens)
            {
                AddError("Incomplete directive.");

                state.Pop(); // Pop NVDirectiveParams
                state.Pop(); // Pop NVDirective

                // Return the directive hash
                return GetNextToken();
            }

            if (ch == ')')
            {
                token.SetStartPosition(lineNo, linePos);
                token.Type = TokenType.NVDirectiveRParen;
                GetCh();
                state.Pop(); // Pop NVDirectiveParams
                state.Pop(); // Pop NVDirective
            }
            else if (ch == '\n')
            {
                // Break out of the directive
                state.Pop(); // Pop NVDirectiveParams
                state.Pop(); // Pop NVDirective
                return GetNextToken();
            }
            else
            {
                token = ReadNVelocityToken();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #14
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVDirective()
        {
            Token token = new Token();

            if (NextCharAfterSingleLineWhiteSpace() == '(')
            {
                ConsumeSingleLineWhiteSpace();
                token.SetStartPosition(lineNo, linePos);
                token.Type = TokenType.NVDirectiveLParen;
                GetCh();

                state.Push(ScannerState.NVDirectiveParams);
            }
            else
            {
                state.Pop(); // Pop NVDirective
                token = GetNextToken();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #15
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ReadNVelocityIdentifier()
        {
            Token token = new Token();
            token.SetStartPosition(lineNo, linePos);

            int startPos = pos;
            GetCh();
            while (char.IsLetterOrDigit(ch) || ch == '_' || ch == '-')
                GetCh();

            string ident = _source.Substring(startPos - 1, pos - startPos);
            if (nvKeywords.ContainsKey(ident))
            {
                token.Type = nvKeywords[ident];
            }
            else
            {
                token.Type = TokenType.NVIdentifier;
                token.Image = ident;
            }

            return token;
        }
예제 #16
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        /// <summary>
        /// Returns the next token from the current position.
        /// </summary>
        /// <returns>The next token from the current position.</returns>
        public Token GetToken()
        {
            if (_prereadTokens.Count > 0)
            {
                _currentToken = _prereadTokens.Dequeue();
            }
            else
            {
                _currentToken = GetNextToken();
            }

            if (_currentToken != null)
            {
                eof = false;
            }

            return _currentToken;
        }
예제 #17
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ReadNVelocityMultiLineComment()
        {
            Token token = new Token();
            int startPos = pos - 1; // '- 1' to include '#'

            bool endFound = false;
            while (!eof && !endFound)
            {
                if (ch == '*')
                {
                    GetCh();
                    if (ch == '#')
                    {
                        endFound = true;
                    }
                }
                else
                    GetCh();
            }
            if (!endFound && eof)
            {
                AddError("Expected end of NVelocity comment");
                return token;
            }

            token.Type = TokenType.NVSingleLineComment;
            token.Image = _source.Substring(startPos - 1, pos - startPos + 1);
            GetCh();

            return token;
        }
예제 #18
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVStringLiteralSingle()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (ch == '\'')
            {
                state.Pop(); // Pop NVStringLiteralSingle
                token.Type = TokenType.NVSingleQuote;
                GetCh();
                token.SetEndPosition(lineNo, linePos);
                return token;
            }

            token = ReadText(TokenType.NVStringLiteral, delegate
            {
                return ch == '\'';
            });

            if (eof && !_options.IsLineScanner)
            {
                AddError("Expected end of string literal");
            }

            return token;
        }
예제 #19
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ReadText(TokenType tokenType, StopScanningDelegate stopScanningDelegate)
        {
            Token token = new Token();
            token.SetStartPosition(lineNo, linePos);

            int startPos = pos;
            bool isText = true;
            bool inWord = false;
            bool inSpace = false;
            bool hasTokenToReturn = false;

            while (isText && !hasTokenToReturn && !eof)
            {
                // The predicate is used to decide whether the current character is text
                // or it belongs to something else (eg. directive or reference).
                if (stopScanningDelegate.Invoke())
                {
                    isText = false;
                }
                else
                {
                    if (_options.SplitTextTokens)
                    {
                        if (char.IsLetter(ch))
                        {
                            if (inSpace)
                            {
                                // End of space found
                                inSpace = false;
                                hasTokenToReturn = true;
                            }
                            else
                            {
                                if (!inWord)
                                {
                                    // Entering a new word
                                    inWord = true;
                                }
                                GetCh();
                            }
                        }
                        else if (char.IsWhiteSpace(ch))
                        {
                            if (inWord)
                            {
                                // End of word found
                                inWord = false;
                                hasTokenToReturn = true;
                            }
                            else
                            {
                                if (!inSpace)
                                {
                                    // Entering new space block
                                    inSpace = true;
                                }
                                GetCh();
                            }
                        }
                        else
                        {
                            if (inWord)
                            {
                                // End of word found
                                inWord = false;
                                hasTokenToReturn = true;
                            }
                            else if (inSpace)
                            {
                                // End of space block found
                                inSpace = false;
                                hasTokenToReturn = true;
                            }
                            else
                            {
                                hasTokenToReturn = true;
                                GetCh();
                            }
                        }
                    }
                    else
                    {
                        GetCh();
                    }
                }
            }

            token.SetEndPosition(lineNo, linePos);
            token.Type = tokenType;
            token.Image = _source.Substring(startPos - 1, pos - startPos);

            return token;
        }
예제 #20
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenXmlCData()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (ch == ']' && LookAhead(1) == ']' && LookAhead(2) == '>')
            {
                token.Type = TokenType.XmlCDataEnd;
                GetCh(3);
                state.Pop(); // Pop NVXmlCData
                token.SetEndPosition(lineNo, linePos);
                return token;
            }

            token = ReadText(TokenType.XmlCDataSection, delegate
            {
                return (ch == ']' && LookAhead(1) == ']' && LookAhead(2) == '>') || eof;
            });

            if (eof && !_options.IsLineScanner)
            {
                throw new ScannerError("End-of-file found but CData section was not closed");
            }

            return token;
        }
예제 #21
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVBrack()
        {
            Token token = new Token();

            ConsumeSingleLineWhiteSpace();

            token.SetStartPosition(lineNo, linePos);

            if (ch == ']')
            {
                token.Type = TokenType.NVRBrack;
                GetCh();
                state.Pop(); // Pop NVBrack
            }
            else
            {
                token = ReadNVelocityToken();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }
예제 #22
0
파일: Scanner.cs 프로젝트: jonorossi/cvsi
        private Token ScanTokenNVDictionary()
        {
            Token token = new Token();

            token.SetStartPosition(lineNo, linePos);

            if (ch == '%')
            {
                token.Type = TokenType.NVDictionaryPercent;
                GetCh();
            }
            else if (ch == '{')
            {
                token.Type = TokenType.NVDictionaryLCurly;
                GetCh();
                state.Push(ScannerState.NVDictionaryInner);
            }
            else if (ch == '"')
            {
                token.Type = TokenType.NVDoubleQuote;
                state.Pop(); // Pop NVDictionary
                GetCh();
            }
            else
            {
                // If the dictionary is not complete and in IntelliSense mode then get out of it
                if (_options.EnableIntelliSenseTriggerTokens)
                {
                    state.Pop(); // Pop NVDictionary
                }

                AddError("Expected opening dictionary declaration");

                return GetNextToken();
            }

            token.SetEndPosition(lineNo, linePos);

            return token;
        }