Пример #1
0
 public bool TheSame(IdentifierToken identifier)
 {
     return(identifier.Lexeme == Identifier.Lexeme);
 }
Пример #2
0
        override internal bool FindNextToken()
        {
            int startPosition = _reader.Position;

            // Dealing with whitespace?
            if (_reader.SinkMultipleWhiteSpace())
            {
                current = new WhitespaceToken();
                return(true);
            }
            // Check for one-line comment
            else if (_reader.Sink("//"))
            {
                // Looks like a one-line comment. Follow it to the End-of-line
                _reader.SinkToEndOfLine();

                current = new CommentToken();
                return(true);
            }
            // Check for multi-line comment
            else if (_reader.Sink("/*"))
            {
                _reader.SinkUntil("*/");

                // Was the ending */ found?
                if (_reader.EndOfLines)
                {
                    // No. There was a /* without a */. Return this a syntax error token.
                    current = new CSharpTokenizer.EndOfFileInsideCommentToken();
                    return(true);
                }

                current = new CommentToken();
                return(true);
            }
            // Handle chars
            else if (_reader.Sink("\'"))
            {
                while (_reader.CurrentCharacter != '\'')
                {
                    if (_reader.Sink("\\"))
                    {
                        /* reader.Skip the escape sequence.
                         *  This isn't exactly right. We should detect:
                         *
                         *  simple-escape-sequence: one of
                         \' \" \\ \0 \a \b \f \n \r \t \v
                         *
                         *  hexadecimal-escape-sequence:
                         *  \x   hex-digit   hex-digit[opt]   hex-digit[opt]  hex-digit[opt]
                         */
                    }

                    _reader.SinkCharacter();
                }

                if (_reader.SinkCharacter() != '\'')
                {
                    Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick.");
                }
                current = new CSharpTokenizer.CharLiteralToken();
                return(true);
            }
            // Check for verbatim string
            else if (_reader.Sink("@\""))
            {
                do
                {
                    // Inside a verbatim string "" is treated as a special character
                    while (_reader.Sink("\"\""))
                    {
                    }
                }while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');

                // Can't end a file inside a string
                if (_reader.EndOfLines)
                {
                    current = new EndOfFileInsideStringToken();
                    return(true);
                }

                // reader.Skip the ending quote.
                current           = new StringLiteralToken();
                current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1);
                return(true);
            }
            // Check for a quoted string.
            else if (_reader.Sink("\""))
            {
                while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral())
                {
                    // See if we have an escape sequence.
                    if (_reader.SinkCharacter() == '\\')
                    {
                        // This is probably an escape character.
                        if (_reader.SinkStringEscape())
                        {
                            // This isn't nearly right. We just do barely enough to make a string
                            // with an embedded escape sequence return _some_ string whose start and
                            // end match the real bounds of the string.
                        }
                        else
                        {
                            // This is a compiler error.
                            _reader.SinkCharacter();
                            current = new CSharpTokenizer.UnrecognizedStringEscapeToken();
                            return(true);
                        }
                    }
                }

                // Is it a newline?
                if (TokenChar.IsNewLine(_reader.CurrentCharacter))
                {
                    current = new CSharpTokenizer.NewlineInsideStringToken();
                    return(true);
                }

                // Create the token.
                if (_reader.SinkCharacter() != '\"')
                {
                    Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote.");
                }
                current = new StringLiteralToken();
                return(true);
            }
            // Identifier or keyword?
            else if
            (
                // From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier.
                _reader.CurrentCharacter == '@' ||
                _reader.MatchNextIdentifierStart()
            )
            {
                if (_reader.CurrentCharacter == '@')
                {
                    _reader.SinkCharacter();
                }

                // Now, the next character must be an identifier start.
                if (!_reader.SinkIdentifierStart())
                {
                    current = new ExpectedIdentifierToken();
                    return(true);
                }

                // Sink the rest of the identifier.
                while (_reader.SinkIdentifierPart())
                {
                }
                string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);

                switch (identifierOrKeyword)
                {
                default:

                    if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0)
                    {
                        current = new KeywordToken();
                        return(true);
                    }

                    // If the identifier starts with '@' then we need to strip it off.
                    // The '@' is for escaping so that we can have an identifier called
                    // the same thing as a reserved keyword (i.e. class, if, foreach, etc)
                    string identifier = _reader.GetCurrentMatchedString(startPosition);
                    if (identifier.StartsWith("@", StringComparison.Ordinal))
                    {
                        identifier = identifier.Substring(1);
                    }

                    // Create the token.
                    current           = new IdentifierToken();
                    current.InnerText = identifier;
                    return(true);

                case "false":
                case "true":
                    current = new BooleanLiteralToken();
                    return(true);

                case "null":
                    current = new CSharpTokenizer.NullLiteralToken();
                    return(true);
                }
            }
            // Open scope
            else if (_reader.Sink("{"))
            {
                current = new CSharpTokenizer.OpenScopeToken();
                return(true);
            }
            // Close scope
            else if (_reader.Sink("}"))
            {
                current = new CSharpTokenizer.CloseScopeToken();
                return(true);
            }
            // Hexidecimal integer literal
            else if (_reader.SinkIgnoreCase("0x"))
            {
                // Sink the hex digits.
                if (!_reader.SinkMultipleHexDigits())
                {
                    current = new ExpectedValidHexDigitToken();
                    return(true);
                }

                // Skip the L, U, l, u, ul, etc.
                _reader.SinkLongIntegerSuffix();

                current = new HexIntegerLiteralToken();
                return(true);
            }
            // Decimal integer literal
            else if (_reader.SinkMultipleDecimalDigits())
            {
                // reader.Skip the L, U, l, u, ul, etc.
                _reader.SinkLongIntegerSuffix();

                current = new DecimalIntegerLiteralToken();
                return(true);
            }
            // Check for single-digit operators and punctuators
            else if (_reader.SinkOperatorOrPunctuator())
            {
                current = new OperatorOrPunctuatorToken();
                return(true);
            }
            // Preprocessor line
            else if (_reader.CurrentCharacter == '#')
            {
                if (_reader.Sink("#if"))
                {
                    current = new OpenConditionalDirectiveToken();
                }
                else if (_reader.Sink("#endif"))
                {
                    current = new CloseConditionalDirectiveToken();
                }
                else
                {
                    current = new PreprocessorToken();
                }

                _reader.SinkToEndOfLine();

                return(true);
            }

            // We didn't recognize the token, so this is a syntax error.
            _reader.SinkCharacter();
            current = new UnrecognizedToken();
            return(true);
        }
Пример #3
0
 public FbxNode(IdentifierToken identifier)
 {
     Identifier = identifier;
 }
Пример #4
0
        /// <summary>
        /// Create a new block argument.
        /// </summary>
        /// <param name="parent">Block closure that defines the argument node.</param>
        /// <param name="colon">Colon that precedes the argument name.</param>
        /// <param name="token">Identifier token containing the name of the argument.</param>
        protected internal BlockArgumentNode(BlockNode parent, SpecialCharacterToken colon, IdentifierToken token)
            : base(token)
        {
#if DEBUG
            if (parent == null)
            {
                throw new ArgumentNullException("parent");
            }
            if (colon == null)
            {
                throw new ArgumentNullException("colon");
            }
#endif
            this.Colon  = colon;
            this.Parent = parent;
        }
Пример #5
0
 public AtomicNameNode(IdentifierToken identifier)
     : this(ParseNodeType.Name, identifier)
 {
 }
Пример #6
0
 /// <summary>
 /// member_access = expression operator member_name
 /// operator      = "."
 /// member_name   = identifier
 /// </summary>
 /// <param name="expression"></param>
 /// <param name="name"></param>
 public ArmMemberAccessExpressionAst(ArmExpressionAst expression, SyntaxToken operatorToken, IdentifierToken name)
 {
     this.Expression    = expression ?? throw new ArgumentNullException(nameof(expression));
     this.OperatorToken = operatorToken ?? throw new ArgumentNullException(nameof(operatorToken));
     this.Name          = name ?? throw new ArgumentNullException(nameof(name));
 }
Пример #7
0
        public static Assignment ParseAssignment(this IEnumerator <Token> tokenStream, IdentifierToken id)
        {
            var valueToken = GetNext(tokenStream);

            switch (valueToken)
            {
            case IntegerToken:
                break;

            case FloatToken:
                break;

            case BooleanToken:
                break;

            case StringToken:
                break;

            default:
                throw ParsingException.CreateError(valueToken, "value");
            }

            ExpectNext <SemicolonToken>(tokenStream);

            return(new Assignment(id, valueToken));
        }
 public PoolVariableDefinitionNode(IdentifierToken poolName)
     : base(poolName)
 {
 }
 public PoolConstantDefinitionNode(IdentifierToken poolName)
     : base(poolName)
 {
 }
Пример #10
0
 public Variable(IdentifierToken token) : base(token)
 {
 }
 public IdentifierExpression(IdentifierToken idToken)
 {
     IdToken = idToken;
 }
Пример #12
0
 public NamespaceNode(IdentifierToken identifier) => Identifier = identifier;
Пример #13
0
 /// <summary>
 /// function_reference = function_name
 /// function_name      = identifier
 /// </summary>
 /// <param name="name"></param>
 public ArmFunctionReferenceAst(IdentifierToken name)
 {
     this.Name = name ?? throw new ArgumentNullException(nameof(name));
 }
Пример #14
0
            public override Node Make(Parser parser)
            {
                IdentifierToken id = parser.AssertToken <IdentifierToken>();

                return(new Variable(id));
            }
Пример #15
0
 private Variable(IdentifierToken id)
 {
     this.id = id;
     region  = id.Region;
 }
Пример #16
0
 public PropertyAccessToken(int position, TokenImpl host, IdentifierToken property) : base(position)
 {
     this.Host     = host;
     this.Property = property;
 }
Пример #17
0
    public void TestTokenProperties()
    {
        IdentifierToken result = new IdentifierToken("identifier");

        Assert.AreEqual("identifier", result.Value);
    }
        /*
         * Method:  FindNextToken
         *
         * Find the next token. Return 'true' if one was found. False, otherwise.
         */
        override internal bool FindNextToken()
        {
            int startPosition = _reader.Position;

            // VB docs claim whitespace is Unicode category Zs. However,
            // this category does not contain tabs. Assuming a less restrictive
            // definition for whitespace...
            if (_reader.SinkWhiteSpace())
            {
                while (_reader.SinkWhiteSpace())
                {
                }

                // Now, we need to check for the line continuation character.
                if (_reader.SinkLineContinuationCharacter())    // Line continuation is '_'
                {
                    // Save the current position because we may need to come back here.
                    int savePosition = _reader.Position - 1;

                    // Skip all whitespace after the '_'
                    while (_reader.SinkWhiteSpace())
                    {
                    }

                    // Now, skip all the newlines.
                    // Need at least one newline for this to count as line continuation.
                    int count = 0;
                    while (_reader.SinkNewLine())
                    {
                        ++count;
                    }

                    if (count > 0)
                    {
                        current = new VisualBasicTokenizer.LineContinuationToken();
                        return(true);
                    }

                    // Otherwise, fall back to plain old whitespace.
                    _reader.Position = savePosition;
                }

                current = new WhitespaceToken();
                return(true);
            }
            // Line terminators are separate from whitespace and are significant.
            else if (_reader.SinkNewLine())
            {
                // We want one token per line terminator.
                current = new VisualBasicTokenizer.LineTerminatorToken();
                return(true);
            }
            // Check for a comment--either those that start with ' or rem.
            else if (_reader.SinkLineCommentStart())
            {
                // Skip to the first EOL.
                _reader.SinkToEndOfLine();

                current = new CommentToken();
                return(true);
            }
            // Identifier or keyword?
            else if
            (
                // VB allows escaping of identifiers by surrounding them with []
                // In other words,
                //      Date is a keyword but,
                //      [Date] is an identifier.
                _reader.CurrentCharacter == '[' ||
                _reader.MatchNextIdentifierStart()
            )
            {
                bool escapedIdentifier = false;
                if (_reader.CurrentCharacter == '[')
                {
                    escapedIdentifier = true;
                    _reader.SinkCharacter();

                    // Now, the next character must be an identifier start.
                    if (!_reader.SinkIdentifierStart())
                    {
                        current = new ExpectedIdentifierToken();
                        return(true);
                    }
                }

                // Sink the rest of the identifier.
                while (_reader.SinkIdentifierPart())
                {
                }

                // If this was an escaped identifier the we need to get the terminating ']'.
                if (escapedIdentifier)
                {
                    if (!_reader.Sink("]"))
                    {
                        current = new ExpectedIdentifierToken();
                        return(true);
                    }
                }
                else
                {
                    // Escaped identifiers are not allowed to have trailing type character.
                    _reader.SinkTypeCharacter(); // Type character is optional.
                }

                // An identifier that is only a '_' is illegal because it is
                // ambiguous with line continuation
                string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);
                if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]")
                {
                    current = new ExpectedIdentifierToken();
                    return(true);
                }

                // Make an upper-case version in order to check whether this may be a keyword.
                string upper = identifierOrKeyword.ToUpper(CultureInfo.InvariantCulture);

                switch (upper)
                {
                default:

                    if (Array.IndexOf(s_keywordList, upper) >= 0)
                    {
                        current = new KeywordToken();
                        return(true);
                    }

                    // Create the token.
                    current = new IdentifierToken();

                    // Trim off the [] if this is an escaped identifier.
                    if (escapedIdentifier)
                    {
                        current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2);
                    }
                    return(true);

                case "FALSE":
                case "TRUE":
                    current = new BooleanLiteralToken();
                    return(true);
                }
            }
            // Is it a hex integer?
            else if (_reader.SinkHexIntegerPrefix())
            {
                if (!_reader.SinkMultipleHexDigits())
                {
                    current = new ExpectedValidHexDigitToken();
                    return(true);
                }

                // Sink a suffix if there is one.
                _reader.SinkIntegerSuffix();

                current = new HexIntegerLiteralToken();
                return(true);
            }
            // Is it an octal integer?
            else if (_reader.SinkOctalIntegerPrefix())
            {
                if (!_reader.SinkMultipleOctalDigits())
                {
                    current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken();
                    return(true);
                }

                // Sink a suffix if there is one.
                _reader.SinkIntegerSuffix();

                current = new VisualBasicTokenizer.OctalIntegerLiteralToken();
                return(true);
            }
            // Is it a decimal integer?
            else if (_reader.SinkMultipleDecimalDigits())
            {
                // Sink a suffix if there is one.
                _reader.SinkDecimalIntegerSuffix();

                current = new DecimalIntegerLiteralToken();
                return(true);
            }
            // Preprocessor line
            else if (_reader.CurrentCharacter == '#')
            {
                if (_reader.SinkIgnoreCase("#if"))
                {
                    current = new OpenConditionalDirectiveToken();
                }
                else if (_reader.SinkIgnoreCase("#end if"))
                {
                    current = new CloseConditionalDirectiveToken();
                }
                else
                {
                    current = new PreprocessorToken();
                }

                _reader.SinkToEndOfLine();

                return(true);
            }
            // Is it a separator?
            else if (_reader.SinkSeparatorCharacter())
            {
                current = new VisualBasicTokenizer.SeparatorToken();
                return(true);
            }
            // Is it an operator?
            else if (_reader.SinkOperator())
            {
                current = new OperatorToken();
                return(true);
            }
            // A string?
            else if (_reader.Sink("\""))
            {
                do
                {
                    // Inside a verbatim string "" is treated as a special character
                    while (_reader.Sink("\"\""))
                    {
                    }
                }while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');

                // Can't end a file inside a string
                if (_reader.EndOfLines)
                {
                    current = new EndOfFileInsideStringToken();
                    return(true);
                }

                current = new StringLiteralToken();
                return(true);
            }


            // We didn't recognize the token, so this is a syntax error.
            _reader.SinkCharacter();
            current = new UnrecognizedToken();
            return(true);
        }
Пример #19
0
 public FunctionCall(IdentifierToken name, IList <Ast> args) : base(name)
 {
     Name      = name;
     Arguments = args;
 }
Пример #20
0
        // Constant Binding - cannot be changed

        /// <summary>
        /// Create a new argument node.
        /// </summary>
        /// <param name="token">Identifier token containing the name of the argument.</param>
        protected ArgumentNode(IdentifierToken token)
        {
            this.Token = token;
        }
Пример #21
0
        /// <inheritdoc />
        public Node Visit(BlockParseNode d)
        {
            var  parameters    = new List <Node>();
            Node forcedPattern = null;

            foreach (ParseNode p in d.Parameters)
            {
                var id    = p as IdentifierParseNode;
                var tppn  = p as TypedParameterParseNode;
                var vappn = p as VarArgsParameterParseNode;
                if (id != null)
                {
                    parameters.Add(new ParameterNode(id.Token, id));
                }
                else if (tppn != null)
                {
                    parameters.Add(
                        new ParameterNode(tppn.Token,
                                          tppn.Name as IdentifierParseNode,
                                          tppn.Type.Visit(this)));
                }
                else if (vappn != null)
                {
                    // Inside could be either an identifier or a
                    // TypedParameterParseNode - check for both.
                    var inIPN  = vappn.Name as IdentifierParseNode;
                    var inTPPN = vappn.Name as TypedParameterParseNode;
                    if (inIPN != null)
                    {
                        parameters.Add(new ParameterNode(inIPN.Token,
                                                         inIPN,
                                                         true // Variadic
                                                         ));
                    }
                    else if (inTPPN != null)
                    {
                        parameters.Add(new ParameterNode(inTPPN.Token,
                                                         inTPPN.Name as IdentifierParseNode,
                                                         true, // Variadic
                                                         inTPPN.Type.Visit(this)
                                                         ));
                    }
                }
                else if (p is NumberParseNode || p is StringLiteralParseNode ||
                         p is OperatorParseNode)
                {
                    parameters.Add(p.Visit(this));
                }
                else if (p is ParenthesisedParseNode)
                {
                    var tok = p.Token;
                    var it  = new IdentifierToken(tok.module, tok.line,
                                                  tok.column, "_");
                    id = new IdentifierParseNode(it);
                    parameters.Add(new ParameterNode(tok, id, p.Visit(this)));
                }
                else
                {
                    throw new Exception("unimplemented - unusual parameters");
                }
            }
            var ret = new BlockNode(d.Token, d,
                                    parameters,
                                    map(d.Body),
                                    forcedPattern);

            return(ret);
        }
Пример #22
0
        /**
         * This is the main lexing algorithm. It consumes source file as string and puts out token list.
         * Token consists of token type and range that token spans.
         */
        public static IToken[] Parse(string s)
        {
            LinkedList <IToken> ll = new LinkedList <IToken>();
            // use five kinds of token to produce token stream.
            var index = 0;
            var next  = 0;

            while (index < s.Length)
            {
                // try to parse as number
                if (NumberToken.ParseNumber(s, index, out var tokenNumber, out next))
                {
                    ll.AddLast(tokenNumber);
                    index = next;
                    continue;
                }

                // then try to parse as string
                if (StringToken.ParseString(s, index, out var tokenString, out next))
                {
                    ll.AddLast(tokenString);
                    index = next;
                    continue;
                }

                // then try to parse as identifier
                if (IdentifierToken.ParseIdentifier(s, index, out var tokenIdentifier, out next))
                {
                    ll.AddLast(tokenIdentifier);
                    index = next;
                    continue;
                }

                // then try to parse as comment
                if (CommentToken.ParseComment(s, index, out var tokensComment, out next))
                {
                    foreach (var t in tokensComment)
                    {
                        ll.AddLast(t);
                    }
                    index = next;
                    continue;
                }

                // then try to parse as symbol token
                if (SymbolToken.ParseSymbol(s, index, out var tokenSymbol, out next))
                {
                    ll.AddLast(tokenSymbol);
                    index = next;
                    continue;
                }

                if (Char.IsWhiteSpace(s[index]))
                {
                    // skip spaces
                    index++;
                    continue;
                }

                // otherwise token is unknown
                throw new Exception("unknown token " + s[index] + " at position " + index);
            }

            // return collected tokens
            return(ll.ToArray());
        }
Пример #23
0
 public EnumElementAst(QualifierListAst qualifierList, IdentifierToken enumElementName, IEnumElementValueAst enumElementValue)
 {
     this.QualifierList    = qualifierList ?? new QualifierListAst.Builder().Build();
     this.EnumElementName  = enumElementName;
     this.EnumElementValue = enumElementValue;
 }
Пример #24
0
 public GenericNameNode(IdentifierToken name, ParseNodeList typeArguments)
     : base(ParseNodeType.GenericName, name)
 {
     TypeArguments = typeArguments;
 }
Пример #25
0
 protected AtomicNameNode(ParseNodeType nodeType, IdentifierToken identifier)
     : base(nodeType, identifier)
 {
     Identifier = identifier;
 }
Пример #26
0
        public IEnumerable <IToken> Tokenize(SafeStreamReader source)
        {
            uint lineNumber = 1;
            uint lexemeStartPositionInLine = 1;
            uint absolutePosition          = 1;
            var  maybeCurrentChar          = Option <int> .None;

            var currentLexemeBuffer = new StringBuilder();

            var maybeToken = Option <IToken> .None;;

            while ((maybeCurrentChar = source.Read()).IsSome)
            {
                var currentChar = maybeCurrentChar.Value();

                maybeToken = Option <IToken> .None;

                switch (currentChar)
                {
                case var c when string.IsNullOrWhiteSpace(char.ConvertFromUtf32(c)):
                    // if a whitespace was encountered - strip it
                    // and yield whatever in the buffer to the output

                    maybeToken = FlushBuffer(
                        currentLexemeBuffer,
                        ref absolutePosition,
                        lineNumber,
                        ref lexemeStartPositionInLine
                        );

                    if (maybeToken.IsSome)
                    {
                        yield return(maybeToken.ValueUnsafe());
                    }

                    switch (c)
                    {
                    case '\r':
                        yield return(source.Read()
                                     .Some <IToken>(cn =>
                                                    cn == '\n' ?
                                                    (IToken) new NewLineSymbolToken(
                                                        absolutePosition,
                                                        lineNumber,
                                                        lexemeStartPositionInLine
                                                        ) :
                                                    (IToken) new UnrecognizedToken(
                                                        $"\r{cn}",
                                                        absolutePosition,
                                                        lineNumber,
                                                        lexemeStartPositionInLine
                                                        )
                                                    )
                                     .None(new UnrecognizedToken(
                                               $"\r",
                                               absolutePosition,
                                               lineNumber,
                                               lexemeStartPositionInLine
                                               ))
                                     );

                        absolutePosition         += 2;
                        lineNumber               += 1;
                        lexemeStartPositionInLine = 1;

                        break;

                    case '\n':
                        yield return(new NewLineSymbolToken(
                                         absolutePosition,
                                         lineNumber,
                                         lexemeStartPositionInLine
                                         ));

                        absolutePosition         += 1;
                        lineNumber               += 1;
                        lexemeStartPositionInLine = 1;

                        break;

                    default:
                        absolutePosition          += 1;
                        lexemeStartPositionInLine += 1;
                        break;
                    }

                    break;

                case '.':
                    var currentLexeme = currentLexemeBuffer.ToString();

                    var maybeBeforeToken =
                        IntegerLiteralToken.FromString(
                            currentLexeme,
                            absolutePosition,
                            lineNumber,
                            lexemeStartPositionInLine
                            ) ||
                        IdentifierToken.FromString(
                            currentLexeme,
                            absolutePosition,
                            lineNumber,
                            lexemeStartPositionInLine
                            ) ||
                        UnrecognizedToken.FromString(
                            currentLexeme,
                            absolutePosition,
                            lineNumber,
                            lexemeStartPositionInLine
                            )
                    ;

                    var tokes =
                        source.Peek()
                        .Some <ImmutableList <IToken> >(c =>
                    {
                        var result        = ImmutableList <IToken> .Empty;
                        IToken tokenToAdd = null;

                        switch (c)
                        {
                        case var _ when IsDigit(char.ConvertFromUtf32(c)):
                            currentLexemeBuffer.Append('.');
                            return(ImmutableList <IToken> .Empty);

                        case '.':
                            absolutePosition += maybeBeforeToken
                                                .Map(t => (uint)t.Lexeme.Length)
                                                .IfNone(0);
                            lexemeStartPositionInLine += maybeBeforeToken
                                                         .Some(t => (uint)t.Lexeme.Length)
                                                         .None(0u);

                            tokenToAdd = new RangeSymbolToken(
                                absolutePosition,
                                lineNumber,
                                lexemeStartPositionInLine
                                );

                            result = maybeBeforeToken
                                     .ToImmutableList()
                                     .Add(tokenToAdd);
                            source.Read();
                            currentLexemeBuffer.Clear();
                            lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
                            absolutePosition          += (uint)(tokenToAdd?.Lexeme.Length ?? 0);

                            return(result);

                        default:
                            absolutePosition += maybeBeforeToken
                                                .Map(t => (uint)t.Lexeme.Length)
                                                .IfNone(0);
                            lexemeStartPositionInLine += maybeBeforeToken
                                                         .Some(t => (uint)t.Lexeme.Length)
                                                         .None(0u);

                            tokenToAdd = new DotSymbolToken(
                                absolutePosition,
                                lineNumber,
                                lexemeStartPositionInLine
                                );

                            result = maybeBeforeToken
                                     .ToImmutableList()
                                     .Add(tokenToAdd);
                            currentLexemeBuffer.Clear();
                            lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
                            absolutePosition          += (uint)(tokenToAdd?.Lexeme.Length ?? 0);

                            return(result);
                        }
                    })
                        .None(() =>
                    {
                        absolutePosition += maybeBeforeToken
                                            .Map(t => (uint)t.Lexeme.Length)
                                            .IfNone(0);
                        lexemeStartPositionInLine += maybeBeforeToken
                                                     .Some(t => (uint)t.Lexeme.Length)
                                                     .None(0u);

                        var tokenToAdd = new DotSymbolToken(
                            absolutePosition,
                            lineNumber,
                            lexemeStartPositionInLine
                            );

                        var result = maybeBeforeToken
                                     .ToImmutableList()
                                     .Add(tokenToAdd);
                        currentLexemeBuffer.Clear();
                        lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
                        absolutePosition          += (uint)(tokenToAdd?.Lexeme.Length ?? 0);

                        return(result);
                    })
                    ;

                    foreach (var token in tokes)
                    {
                        yield return(token);
                    }

                    break;

                case '/':
                    maybeToken = FlushBuffer(
                        currentLexemeBuffer,
                        ref absolutePosition,
                        lineNumber,
                        ref lexemeStartPositionInLine
                        );
                    if (maybeToken.IsSome)
                    {
                        yield return(maybeToken.ValueUnsafe());
                    }

                    yield return(source.Peek()
                                 .Some <IToken>(c =>
                    {
                        switch (c)
                        {
                        case '/':
                            var commentContent = source.ReadLine();

                            var commentToken = new CommentToken(
                                $"/{commentContent}",
                                absolutePosition,
                                lineNumber,
                                lexemeStartPositionInLine
                                );

                            absolutePosition += (uint)commentContent.Length;
                            lineNumber += 1;
                            lexemeStartPositionInLine = 0;

                            return commentToken;

                        case '=':
                            var notEqualsToken = new NotEqualsOperatorToken(
                                absolutePosition,
                                lineNumber,
                                lexemeStartPositionInLine
                                );

                            source.Read();
                            absolutePosition += 1;
                            lexemeStartPositionInLine = 1;

                            return notEqualsToken;

                        default:
                            return new DivideOperatorToken(
                                (uint)source.BaseStream.Position,
                                lineNumber,
                                lexemeStartPositionInLine
                                );
                        }
                    })
                                 .None(() => new DivideOperatorToken(
                                           (uint)source.BaseStream.Position,
                                           lineNumber,
                                           lexemeStartPositionInLine
                                           )));

                    absolutePosition          += 1;
                    lexemeStartPositionInLine += 1;

                    break;

                case ':':
                    maybeToken = FlushBuffer(
                        currentLexemeBuffer,
                        ref absolutePosition,
                        lineNumber,
                        ref lexemeStartPositionInLine
                        );
                    if (maybeToken.IsSome)
                    {
                        yield return(maybeToken.ValueUnsafe());
                    }

                    yield return(source.Peek()
                                 .Filter(c => c == '=')
                                 .Some <IToken>(c =>
                    {
                        var result = new AssignmentOperatorToken(
                            absolutePosition,
                            lineNumber,
                            lexemeStartPositionInLine
                            );

                        source.Read();
                        absolutePosition += 1;
                        lexemeStartPositionInLine += 1;

                        return result;
                    })
                                 .None(new ColonSymbolToken(
                                           absolutePosition,
                                           lineNumber,
                                           lexemeStartPositionInLine
                                           )));

                    absolutePosition          += 1;
                    lexemeStartPositionInLine += 1;

                    break;

                case '>':
                    maybeToken = FlushBuffer(
                        currentLexemeBuffer,
                        ref absolutePosition,
                        lineNumber,
                        ref lexemeStartPositionInLine
                        );
                    if (maybeToken.IsSome)
                    {
                        yield return(maybeToken.ValueUnsafe());
                    }

                    yield return(source.Peek()
                                 .Filter(c => c == '=')
                                 .Some <IToken>(_ =>
                    {
                        var result = new GeOperatorToken(
                            absolutePosition,
                            lineNumber,
                            lexemeStartPositionInLine
                            );

                        source.Read();
                        absolutePosition += 1;
                        lexemeStartPositionInLine += 1;

                        return result;
                    })
                                 .None(new GtOperatorToken(
                                           (uint)absolutePosition,
                                           lineNumber,
                                           lexemeStartPositionInLine
                                           )));

                    absolutePosition          += 1;
                    lexemeStartPositionInLine += 1;

                    break;

                case '<':
                    maybeToken = FlushBuffer(
                        currentLexemeBuffer,
                        ref absolutePosition,
                        lineNumber,
                        ref lexemeStartPositionInLine
                        );
                    if (maybeToken.IsSome)
                    {
                        yield return(maybeToken.ValueUnsafe());
                    }

                    yield return(source.Peek()
                                 .Filter(c => c == '=')
                                 .Some <IToken>(_ =>
                    {
                        var result = new LeOperatorToken(
                            absolutePosition,
                            lineNumber,
                            lexemeStartPositionInLine
                            );

                        source.Read();
                        absolutePosition += 1;
                        lexemeStartPositionInLine += 1;

                        return result;
                    })
                                 .None(new LtOperatorToken(
                                           absolutePosition,
                                           lineNumber,
                                           lexemeStartPositionInLine
                                           )));

                    absolutePosition          += 1;
                    lexemeStartPositionInLine += 1;

                    break;

                case '*':
                case '%':
                case '+':
                case '-':
                case '=':
                case ',':
                case '[':
                case ']':
                case '(':
                case ')':
                case ';':
                    maybeToken = FlushBuffer(
                        currentLexemeBuffer,
                        ref absolutePosition,
                        lineNumber,
                        ref lexemeStartPositionInLine
                        );
                    if (maybeToken.IsSome)
                    {
                        yield return(maybeToken.ValueUnsafe());
                    }

                    yield return(SymbolLexemes
                                 .TryGetValue(((char)currentChar).ToString())
                                 .Some(cons => cons(
                                           absolutePosition,
                                           lineNumber,
                                           lexemeStartPositionInLine
                                           ))
                                 .None(() => new UnrecognizedToken(
                                           currentChar.ToString(),
                                           absolutePosition,
                                           lineNumber,
                                           lexemeStartPositionInLine
                                           )
                                       ));

                    absolutePosition          += 1;
                    lexemeStartPositionInLine += 1;

                    break;

                default:
                    currentLexemeBuffer.Append(char.ConvertFromUtf32(currentChar));
                    break;
                }
            }

            maybeToken = FlushBuffer(
                currentLexemeBuffer,
                ref absolutePosition,
                lineNumber,
                ref lexemeStartPositionInLine
                );
            if (maybeToken.IsSome)
            {
                yield return(maybeToken.ValueUnsafe());
            }
        }
Пример #27
0
        private IExpression ParseUnaryExpression(ref IToken token)
        {
            IExpression unaryExpression = ParseLiteralExpression(ref token);

            if (unaryExpression != null)
            {
                token = tokenParser.GetNextToken();
            }
            else if (token is MinusToken)
            {
                token = tokenParser.GetNextToken();
                return(new UnaryMinusExpression(ParseUnaryExpression(ref token)));
            }
            else if (token is IdentifierToken)
            {
                IdentifierToken idToken = token as IdentifierToken;
                token = tokenParser.GetNextToken();
                if (token is OpenParenthesisToken)
                {
                    try
                    {
                        ParameterExpression[] args = ParseArgumentList(ref token);
                        unaryExpression = new FunctionExpression(idToken, args);
                    }
                    catch (SyntaxErrorException e)
                    {
                        throw new SyntaxErrorException(e.Message, e, e.Expression, new FunctionExpression(idToken, e.Args));
                    }
                }
                else if (token is LambdaInvokeToken)
                {
                    try
                    {
                        token = tokenParser.GetNextToken();
                        IExpression rightExpression = ParseExpression(ref token);
                        unaryExpression = new LambdaExpression(idToken, rightExpression);
                    }
                    catch (SyntaxErrorException e)
                    {
                        throw new SyntaxErrorException(e.Message, e, e.Expression, new LambdaExpression(idToken, e.RootExpression));
                    }
                }
                else
                {
                    unaryExpression = new IdentifierExpression(idToken);
                }
            }
            else if (token is OpenParenthesisToken)
            {
                token = tokenParser.GetNextToken();
                IExpression complexExpression = ParseLogicalOrExpression(ref token);
                if (!(token is CloseParenthesisToken))
                {
                    throw new SyntaxErrorException("')' experado.");
                }
                token           = tokenParser.GetNextToken();
                unaryExpression = complexExpression;
            }
            else
            {
                throw new SyntaxErrorException("expressão esperada.");
            }

            //decide se é uma propery access expression
            while (token is DotToken)
            {
                token = tokenParser.GetNextToken();
                if (!(token is IdentifierToken))
                {
                    token = tokenParser.GetNextToken();
                    throw new SyntaxErrorException("identificador esperado", unaryExpression, unaryExpression);
                }

                IdentifierToken id = token as IdentifierToken;
                unaryExpression = new PropertyAccessExpression(unaryExpression, id.IdentifierName);

                token = tokenParser.GetNextToken();
            }

            return(unaryExpression);
        }
 public AssociationDeclarationAst(QualifierListAst qualifierList, IdentifierToken associationName, IdentifierToken superAssociation, ReadOnlyCollection <IClassFeatureAst> classFeatures)
 {
     this.QualifierList    = qualifierList ?? new QualifierListAst.Builder().Build();
     this.AssociationName  = associationName ?? throw new ArgumentNullException(nameof(associationName));
     this.SuperAssociation = superAssociation;
     this.ClassFeatures    = classFeatures ?? new ReadOnlyCollection <IClassFeatureAst>(new List <IClassFeatureAst>());
 }
Пример #29
0
 internal static TableQualifier Create(IdentifierToken identifier, List <MemberAccessToken> memberAccessors)
 {
     return(new TableQualifier(identifier.Content, memberAccessors.Select(accessor => accessor.MemberName)));
 }
Пример #30
0
        private static TriggerTemplate ParseTriggerTemplate(IEnumerator <Token> tokenStream, IdentifierToken id)
        {
            var oldNum =
                tokenStream
                .ExpectNext <IntegerToken>()
                .ValueAsUshort(token => ParsingException.CreateError(token, "UShort value"));

            tokenStream.ExpectNext <OpenBraceToken>();

            var block = tokenStream.ParseBlock(id);

            return(ReadTriggerTemplate(oldNum, block));
        }
Пример #31
0
        /*
        * Method:  FindNextToken
        *
        * Find the next token. Return 'true' if one was found. False, otherwise.
        */
        internal override bool FindNextToken()
        {
            int startPosition = _reader.Position;

            // VB docs claim whitespace is Unicode category Zs. However,
            // this category does not contain tabs. Assuming a less restrictive
            // definition for whitespace...
            if (_reader.SinkWhiteSpace())
            {
                while (_reader.SinkWhiteSpace())
                {
                }

                // Now, we need to check for the line continuation character.
                if (_reader.SinkLineContinuationCharacter())    // Line continuation is '_'
                {
                    // Save the current position because we may need to come back here.
                    int savePosition = _reader.Position - 1;

                    // Skip all whitespace after the '_'
                    while (_reader.SinkWhiteSpace())
                    {
                    }

                    // Now, skip all the newlines.
                    // Need at least one newline for this to count as line continuation.
                    int count = 0;
                    while (_reader.SinkNewLine())
                    {
                        ++count;
                    }

                    if (count > 0)
                    {
                        current = new VisualBasicTokenizer.LineContinuationToken();
                        return true;
                    }

                    // Otherwise, fall back to plain old whitespace.
                    _reader.Position = savePosition;
                }

                current = new WhitespaceToken();
                return true;
            }
            // Line terminators are separate from whitespace and are significant.
            else if (_reader.SinkNewLine())
            {
                // We want one token per line terminator.
                current = new VisualBasicTokenizer.LineTerminatorToken();
                return true;
            }
            // Check for a comment--either those that start with ' or rem.
            else if (_reader.SinkLineCommentStart())
            {
                // Skip to the first EOL.
                _reader.SinkToEndOfLine();

                current = new CommentToken();
                return true;
            }
            // Identifier or keyword?
            else if
            (
                // VB allows escaping of identifiers by surrounding them with []
                // In other words,
                //      Date is a keyword but,
                //      [Date] is an identifier.
                _reader.CurrentCharacter == '[' ||
                _reader.MatchNextIdentifierStart()
            )
            {
                bool escapedIdentifier = false;
                if (_reader.CurrentCharacter == '[')
                {
                    escapedIdentifier = true;
                    _reader.SinkCharacter();

                    // Now, the next character must be an identifier start.
                    if (!_reader.SinkIdentifierStart())
                    {
                        current = new ExpectedIdentifierToken();
                        return true;
                    }
                }

                // Sink the rest of the identifier.
                while (_reader.SinkIdentifierPart())
                {
                }

                // If this was an escaped identifier the we need to get the terminating ']'.
                if (escapedIdentifier)
                {
                    if (!_reader.Sink("]"))
                    {
                        current = new ExpectedIdentifierToken();
                        return true;
                    }
                }
                else
                {
                    // Escaped identifiers are not allowed to have trailing type character.
                    _reader.SinkTypeCharacter(); // Type character is optional.
                }

                // An identifier that is only a '_' is illegal because it is
                // ambiguous with line continuation
                string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);
                if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]")
                {
                    current = new ExpectedIdentifierToken();
                    return true;
                }

                // Make an upper-case version in order to check whether this may be a keyword.
                string upper = identifierOrKeyword.ToUpper(CultureInfo.InvariantCulture);

                switch (upper)
                {
                    default:

                        if (Array.IndexOf(s_keywordList, upper) >= 0)
                        {
                            current = new KeywordToken();
                            return true;
                        }

                        // Create the token.
                        current = new IdentifierToken();

                        // Trim off the [] if this is an escaped identifier.
                        if (escapedIdentifier)
                        {
                            current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2);
                        }
                        return true;
                    case "FALSE":
                    case "TRUE":
                        current = new BooleanLiteralToken();
                        return true;
                }
            }
            // Is it a hex integer?
            else if (_reader.SinkHexIntegerPrefix())
            {
                if (!_reader.SinkMultipleHexDigits())
                {
                    current = new ExpectedValidHexDigitToken();
                    return true;
                }

                // Sink a suffix if there is one.
                _reader.SinkIntegerSuffix();

                current = new HexIntegerLiteralToken();
                return true;
            }
            // Is it an octal integer?
            else if (_reader.SinkOctalIntegerPrefix())
            {
                if (!_reader.SinkMultipleOctalDigits())
                {
                    current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken();
                    return true;
                }

                // Sink a suffix if there is one.
                _reader.SinkIntegerSuffix();

                current = new VisualBasicTokenizer.OctalIntegerLiteralToken();
                return true;
            }
            // Is it a decimal integer?
            else if (_reader.SinkMultipleDecimalDigits())
            {
                // Sink a suffix if there is one.
                _reader.SinkDecimalIntegerSuffix();

                current = new DecimalIntegerLiteralToken();
                return true;
            }
            // Preprocessor line
            else if (_reader.CurrentCharacter == '#')
            {
                if (_reader.SinkIgnoreCase("#if"))
                {
                    current = new OpenConditionalDirectiveToken();
                }
                else if (_reader.SinkIgnoreCase("#end if"))
                {
                    current = new CloseConditionalDirectiveToken();
                }
                else
                {
                    current = new PreprocessorToken();
                }

                _reader.SinkToEndOfLine();

                return true;
            }
            // Is it a separator?
            else if (_reader.SinkSeparatorCharacter())
            {
                current = new VisualBasicTokenizer.SeparatorToken();
                return true;
            }
            // Is it an operator?
            else if (_reader.SinkOperator())
            {
                current = new OperatorToken();
                return true;
            }
            // A string?
            else if (_reader.Sink("\""))
            {
                do
                {
                    // Inside a verbatim string "" is treated as a special character
                    while (_reader.Sink("\"\""))
                    {
                    }
                }
                while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');

                // Can't end a file inside a string
                if (_reader.EndOfLines)
                {
                    current = new EndOfFileInsideStringToken();
                    return true;
                }

                current = new StringLiteralToken();
                return true;
            }

            // We didn't recognize the token, so this is a syntax error.
            _reader.SinkCharacter();
            current = new UnrecognizedToken();
            return true;
        }
Пример #32
0
        /*
        * Method:  FindNextToken
        * 
        * Find the next token. Return 'true' if one was found. False, otherwise.
        */
        override internal bool FindNextToken()
        {
            int startPosition = _reader.Position;

            // Dealing with whitespace?
            if (_reader.SinkMultipleWhiteSpace())
            {
                current = new WhitespaceToken();
                return true;
            }
            // Check for one-line comment
            else if (_reader.Sink("//"))
            {
                // Looks like a one-line comment. Follow it to the End-of-line
                _reader.SinkToEndOfLine();

                current = new CommentToken();
                return true;
            }
            // Check for multi-line comment
            else if (_reader.Sink("/*"))
            {
                _reader.SinkUntil("*/");

                // Was the ending */ found?
                if (_reader.EndOfLines)
                {
                    // No. There was a /* without a */. Return this a syntax error token.
                    current = new CSharpTokenizer.EndOfFileInsideCommentToken();
                    return true;
                }

                current = new CommentToken();
                return true;
            }
            // Handle chars
            else if (_reader.Sink("\'"))
            {
                while (_reader.CurrentCharacter != '\'')
                {
                    if (_reader.Sink("\\"))
                    {
                        /* reader.Skip the escape sequence. 
                            This isn't exactly right. We should detect:
                            
                            simple-escape-sequence: one of 
                            \' \" \\ \0 \a \b \f \n \r \t \v 
                            
                            hexadecimal-escape-sequence: 
                            \x   hex-digit   hex-digit[opt]   hex-digit[opt]  hex-digit[opt]                                
                        */
                    }

                    _reader.SinkCharacter();
                }

                if (_reader.SinkCharacter() != '\'')
                {
                    Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick.");
                }
                current = new CSharpTokenizer.CharLiteralToken();
                return true;
            }
            // Check for verbatim string
            else if (_reader.Sink("@\""))
            {
                do
                {
                    // Inside a verbatim string "" is treated as a special character
                    while (_reader.Sink("\"\""))
                    {
                    }
                }
                while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');

                // Can't end a file inside a string 
                if (_reader.EndOfLines)
                {
                    current = new EndOfFileInsideStringToken();
                    return true;
                }

                // reader.Skip the ending quote.
                current = new StringLiteralToken();
                current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1);
                return true;
            }
            // Check for a quoted string.
            else if (_reader.Sink("\""))
            {
                while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral())
                {
                    // See if we have an escape sequence.
                    if (_reader.SinkCharacter() == '\\')
                    {
                        // This is probably an escape character.
                        if (_reader.SinkStringEscape())
                        {
                            // This isn't nearly right. We just do barely enough to make a string
                            // with an embedded escape sequence return _some_ string whose start and 
                            // end match the real bounds of the string.
                        }
                        else
                        {
                            // This is a compiler error. 
                            _reader.SinkCharacter();
                            current = new CSharpTokenizer.UnrecognizedStringEscapeToken();
                            return true;
                        }
                    }
                }

                // Is it a newline?
                if (TokenChar.IsNewLine(_reader.CurrentCharacter))
                {
                    current = new CSharpTokenizer.NewlineInsideStringToken();
                    return true;
                }

                // Create the token.
                if (_reader.SinkCharacter() != '\"')
                {
                    Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote.");
                }
                current = new StringLiteralToken();
                return true;
            }
            // Identifier or keyword?
            else if
            (
                // From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier.
                _reader.CurrentCharacter == '@' ||
                _reader.MatchNextIdentifierStart()
            )
            {
                if (_reader.CurrentCharacter == '@')
                {
                    _reader.SinkCharacter();
                }

                // Now, the next character must be an identifier start.
                if (!_reader.SinkIdentifierStart())
                {
                    current = new ExpectedIdentifierToken();
                    return true;
                }

                // Sink the rest of the identifier.                     
                while (_reader.SinkIdentifierPart())
                {
                }
                string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);

                switch (identifierOrKeyword)
                {
                    default:

                        if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0)
                        {
                            current = new KeywordToken();
                            return true;
                        }

                        // If the identifier starts with '@' then we need to strip it off.
                        // The '@' is for escaping so that we can have an identifier called
                        // the same thing as a reserved keyword (i.e. class, if, foreach, etc)
                        string identifier = _reader.GetCurrentMatchedString(startPosition);
                        if (identifier.StartsWith("@", StringComparison.Ordinal))
                        {
                            identifier = identifier.Substring(1);
                        }

                        // Create the token.
                        current = new IdentifierToken();
                        current.InnerText = identifier;
                        return true;
                    case "false":
                    case "true":
                        current = new BooleanLiteralToken();
                        return true;
                    case "null":
                        current = new CSharpTokenizer.NullLiteralToken();
                        return true;
                }
            }
            // Open scope
            else if (_reader.Sink("{"))
            {
                current = new CSharpTokenizer.OpenScopeToken();
                return true;
            }
            // Close scope
            else if (_reader.Sink("}"))
            {
                current = new CSharpTokenizer.CloseScopeToken();
                return true;
            }
            // Hexidecimal integer literal
            else if (_reader.SinkIgnoreCase("0x"))
            {
                // Sink the hex digits.
                if (!_reader.SinkMultipleHexDigits())
                {
                    current = new ExpectedValidHexDigitToken();
                    return true;
                }

                // Skip the L, U, l, u, ul, etc.                    
                _reader.SinkLongIntegerSuffix();

                current = new HexIntegerLiteralToken();
                return true;
            }
            // Decimal integer literal
            else if (_reader.SinkMultipleDecimalDigits())
            {
                // reader.Skip the L, U, l, u, ul, etc.                    
                _reader.SinkLongIntegerSuffix();

                current = new DecimalIntegerLiteralToken();
                return true;
            }
            // Check for single-digit operators and punctuators
            else if (_reader.SinkOperatorOrPunctuator())
            {
                current = new OperatorOrPunctuatorToken();
                return true;
            }
            // Preprocessor line
            else if (_reader.CurrentCharacter == '#')
            {
                if (_reader.Sink("#if"))
                {
                    current = new OpenConditionalDirectiveToken();
                }
                else if (_reader.Sink("#endif"))
                {
                    current = new CloseConditionalDirectiveToken();
                }
                else
                {
                    current = new PreprocessorToken();
                }

                _reader.SinkToEndOfLine();

                return true;
            }

            // We didn't recognize the token, so this is a syntax error. 
            _reader.SinkCharacter();
            current = new UnrecognizedToken();
            return true;
        }