Exemplo n.º 1
0
 public BraceToken(Scope scope, Span span, BracesToken braces, bool open)
     : base(scope, span)
 {
     _braces        = braces;
     _open          = open;
     ClassifierType = Classifier.ProbeClassifierType.Operator;
 }
Exemplo n.º 2
0
        public static WhileStatement Parse(Scope scope, KeywordToken whileToken)
        {
            var code = scope.Code;
            var ret  = new WhileStatement(scope, whileToken);

            scope               = scope.Clone();
            scope.BreakOwner    = ret;
            scope.ContinueOwner = ret;

            // Expression
            var expressionScope = scope.Clone();

            expressionScope.Hint |= ScopeHint.SuppressDataType | ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressVarDecl;

            ret._expressionToken = ExpressionToken.TryParse(scope, null);
            if (ret._expressionToken != null)
            {
                ret.AddToken(ret._expressionToken);

                if (code.PeekExact('{'))
                {
                    var bodyScope = scope.Clone();
                    bodyScope.Hint |= ScopeHint.SuppressFunctionDefinition;

                    ret.AddToken(ret._bodyToken = BracesToken.Parse(bodyScope, null));
                }
            }

            return(ret);
        }
Exemplo n.º 3
0
 private BracesToken FindBodyToken()
 {
     if (!_bodyTokenSearched)
     {
         _bodyToken         = Scope.File.FindDownward(_bodyStart, t => t is BracesToken && t.Span.Start == _bodyStart).LastOrDefault() as BracesToken;
         _bodyTokenSearched = true;
     }
     return(_bodyToken);
 }
Exemplo n.º 4
0
        /// <summary>
        /// Parses a set of braces from the file.
        /// </summary>
        /// <param name="scope">The current scope.</param>
        /// <param name="ownerDef">(optional) A definition of an object (e.g. function) that owns these braces.</param>
        /// <param name="funcOutliningStart">(optional) To specify a custom outlining region start position. Function use this to get the outlining to appear at the end of the previous line.</param>
        /// <returns>The braces token.</returns>
        public static BracesToken Parse(Scope scope, Definition ownerDef, int funcOutliningStart = -1)
        {
            if (ownerDef != null)
            {
                scope = scope.CloneIndent();
                scope.ReturnDataType = ownerDef.DataType;
            }

            var code = scope.Code;

            if (!code.ReadExact('{'))
            {
                throw new InvalidOperationException("BracesToken.Parse expected next char to be '{'.");
            }
            var openBraceSpan = code.Span;

            var indentScope = scope.CloneIndentNonRoot();

            indentScope.Hint |= ScopeHint.SuppressFunctionDefinition;

            var ret = new BracesToken(scope);

            ret._funcOutliningStart = funcOutliningStart;
            ret._openToken          = new BraceToken(scope, openBraceSpan, ret, true);
            ret.AddToken(ret._openToken);

            while (!code.EndOfFile)
            {
                if (code.ReadExact('}'))
                {
                    ret._closeToken = new BraceToken(scope, code.Span, ret, false);
                    ret.AddToken(ret._closeToken);
                    return(ret);
                }

                var stmt = StatementToken.TryParse(scope);
                if (stmt != null)
                {
                    ret.AddToken(stmt);
                }
            }

            return(ret);
        }
Exemplo n.º 5
0
        public static IfStatement Parse(Scope scope, KeywordToken ifToken)
        {
            var code = scope.Code;

            var ret = new IfStatement(scope);

            ret.AddToken(ifToken);

            var scopeIndent = scope.Clone();

            scopeIndent.Hint |= ScopeHint.NotOnRoot | ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressVarDecl;

            ret._condition = ExpressionToken.TryParse(scopeIndent, null);
            if (ret._condition != null)
            {
                ret.AddToken(ret._condition);
            }

            if (code.PeekExact('{'))
            {
                ret.AddToken(ret._trueBody = BracesToken.Parse(scopeIndent, null));

                if (code.ReadExactWholeWord("else"))
                {
                    ret.AddToken(ret._elseToken = new KeywordToken(scopeIndent, code.Span, "else"));

                    if (code.PeekExact('{'))
                    {
                        ret.AddToken(ret._falseBody = BracesToken.Parse(scopeIndent, null));
                    }
                    else if (code.ReadExactWholeWord("if"))
                    {
                        var ifToken2 = new KeywordToken(scopeIndent, code.Span, "if");
                        ret.AddToken(ret._falseBody = IfStatement.Parse(scopeIndent, ifToken2));
                    }
                }
            }

            return(ret);
        }
Exemplo n.º 6
0
        public static StatementToken TryParse(Scope scope, StatementParseCallback callback = null)
        {
            var code = scope.Code;

            if (!code.SkipWhiteSpace())
            {
                return(null);
            }

            var ret = new StatementToken(scope);

            if (!code.Peek())
            {
                return(null);
            }

            switch (code.Type)
            {
            case CodeType.Word:
                switch (code.Text)
                {
                case "alter":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = CreateStatement.ParseAlter(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "break":
                    if (scope.BreakOwner != null)
                    {
                        var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                        var breakToken   = BreakStatement.Parse(scope, keywordToken);
                        ret.AddToken(breakToken);
                        if (callback != null)
                        {
                            callback(breakToken);
                        }
                        scope.BreakOwner.OnBreakAttached(breakToken);
                        return(ret);
                    }
                    break;

                case "create":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = CreateStatement.ParseCreate(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "col":
                case "row":
                case "colff":
                case "page":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = RowColStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "continue":
                    if (scope.ContinueOwner != null)
                    {
                        var keywordToken  = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                        var continueToken = ContinueStatement.Parse(scope, keywordToken);
                        ret.AddToken(continueToken);
                        if (callback != null)
                        {
                            callback(continueToken);
                        }
                        scope.ContinueOwner.OnContinueAttached(continueToken);
                        return(ret);
                    }
                    break;

                case "extern":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = ExternStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "extract":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = ExtractStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "footer":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = FooterStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "for":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = ForStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "format":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = FormatStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "header":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = HeaderStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "if":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = IfStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "public":
                case "private":
                case "protected":
                    if (scope.Model.FileContext.IsClass())
                    {
                        var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                        ret.AddToken(keywordToken);
                        if (callback != null)
                        {
                            callback(keywordToken);
                        }
                        return(ret);
                    }
                    else
                    {
                        break;
                    }

                case "return":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = ReturnStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "select":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = SelectStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "switch":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = SwitchStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "while":
                {
                    var keywordToken = new KeywordToken(scope, code.MovePeekedSpan(), code.Text);
                    var token        = WhileStatement.Parse(scope, keywordToken);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }
                }
                break;

            case CodeType.Preprocessor:
                switch (code.Text)
                {
                case "#insert":
                {
                    var token = InsertToken.Parse(scope, new InsertStartToken(scope, code.MovePeekedSpan()));
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "#endinsert":
                {
                    var token = new InsertEndToken(scope, code.MovePeekedSpan());
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "#replace":
                {
                    var token = ReplaceToken.Parse(scope, new ReplaceStartToken(scope, code.MovePeekedSpan()));
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "#with":
                {
                    var token = new ReplaceWithToken(scope, code.MovePeekedSpan());
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "#endreplace":
                {
                    var token = new ReplaceEndToken(scope, code.MovePeekedSpan());
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "#include":
                {
                    var token = IncludeToken.Parse(scope, new PreprocessorToken(scope, code.MovePeekedSpan(), code.Text));
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                default:
                {
                    var token = new PreprocessorToken(scope, code.MovePeekedSpan(), code.Text);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }
                }

            case CodeType.Operator:
                switch (code.Text)
                {
                case "{":
                {
                    // Start of a 'scope'. This is not allowed in PROBE/WBDK but allow it here anyway.
                    var token = BracesToken.Parse(scope, null);
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }

                case "}":
                {
                    // Unmatched '}'. This is a syntax error, but since it's a statement breaking token, add it here and end the statement.
                    var token = new OperatorToken(scope, code.MovePeekedSpan(), "}");
                    ret.AddToken(token);
                    if (callback != null)
                    {
                        callback(token);
                    }
                    return(ret);
                }
                }
                break;
            }

            var exp = ExpressionToken.TryParse(scope, null);

            if (exp != null)
            {
                ret.AddToken(exp);
                if (callback != null)
                {
                    callback(exp);
                }

                code.SkipWhiteSpace();
            }

            if (code.ReadExact(';'))
            {
                // Empty statement. This is not allowed in PROBE/WBDK, but allow it here anyway.
                var token = new StatementEndToken(scope, code.Span);
                ret.AddToken(token);
                if (callback != null)
                {
                    callback(token);
                }
                return(ret);
            }

            return(ret);
        }
Exemplo n.º 7
0
        private static Token ProcessWord(ExpressionToken exp, Scope scope, string word, Span wordSpan)
        {
            // Global keyword that take effect anywhere.
            switch (word)
            {
            case "static":
                return(new KeywordToken(scope, wordSpan, word));
            }

            var code = scope.Code;

            if (code.PeekExact('.'))
            {
                var dotSpan = code.MovePeekedSpan();
                var word2   = code.PeekWordR();
                if (!string.IsNullOrEmpty(word2))
                {
                    var word2Span           = code.MovePeekedSpan();
                    var argsPresent         = (scope.Hint & ScopeHint.SuppressFunctionCall) == 0 && code.PeekExact('(');
                    var argsOpenBracketSpan = argsPresent ? code.MovePeekedSpan() : Span.Empty;

                    foreach (var def in scope.DefinitionProvider.GetAny(wordSpan.Start, word))
                    {
                        if (def.AllowsChild)
                        {
                            // When arguments are present, take only the definitions that accept arguments
                            var childDefs = def.GetChildDefinitions(word2).Where(x => argsPresent ? x.ArgumentsRequired : true).ToArray();
                            if (childDefs.Any())
                            {
                                ArgsToken  argsToken = null;
                                Definition childDef  = null;

                                if (argsPresent)
                                {
                                    var openBracketToken = new OperatorToken(scope, argsOpenBracketSpan, "(");
                                    argsToken = ArgsToken.ParseAndChooseArguments(scope, openBracketToken, childDefs, out childDef);
                                }
                                else
                                {
                                    childDef = childDefs[0];
                                }

                                var word1Token = new IdentifierToken(scope, wordSpan, word, def);
                                var dotToken   = new DotToken(scope, dotSpan);
                                var word2Token = new IdentifierToken(scope, word2Span, word2, childDef);
                                var compToken  = new CompositeToken(scope, childDef.DataType);
                                compToken.AddToken(word1Token);
                                compToken.AddToken(dotToken);
                                compToken.AddToken(word2Token);
                                if (argsToken != null)
                                {
                                    compToken.AddToken(argsToken);
                                }
                                return(compToken);
                            }
                        }
                    }
                }
                else
                {
                    code.Position = dotSpan.Start;
                    return(new UnknownToken(scope, wordSpan, word));
                }
            }

            if ((scope.Hint & ScopeHint.SuppressFunctionCall) == 0 && code.PeekExact('('))
            {
                var argsOpenBracketSpan = code.MovePeekedSpan();

                foreach (var def in scope.DefinitionProvider.GetAny(wordSpan.Start, word))
                {
                    if (def.ArgumentsRequired)
                    {
                        var wordToken        = new IdentifierToken(scope, wordSpan, word, def);
                        var openBracketToken = new OperatorToken(scope, argsOpenBracketSpan, "(");
                        var argsToken        = ArgsToken.Parse(scope, openBracketToken, def.Signature);

                        var compToken = new CompositeToken(scope, def.DataType);
                        compToken.AddToken(wordToken);
                        compToken.AddToken(argsToken);

                        if (def.AllowsFunctionBody && (scope.Hint & ScopeHint.SuppressFunctionDefinition) == 0)
                        {
                            ParseFunctionAttributes(exp, scope, compToken);
                            if (code.PeekExact('{'))
                            {
                                compToken.AddToken(BracesToken.Parse(scope, def, argsToken.Span.End + 1));
                            }
                        }

                        return(compToken);
                    }
                }
            }

            foreach (var def in scope.DefinitionProvider.GetAny(wordSpan.Start, word))
            {
                if (def.ArgumentsRequired || def.RequiresChild)
                {
                    continue;
                }

                return(new IdentifierToken(scope, wordSpan, word, def));
            }

            if (StatementToken.IsStatementBreakingWord(scope, word))
            {
                // There could be a statement without a terminating ';' before this.
                // This can happen if it's a macro that already includes the ';'.
                return(null);
            }

            if (Constants.HighlightKeywords.Contains(word))
            {
                return(new KeywordToken(scope, wordSpan, word));
            }

            return(new UnknownToken(scope, wordSpan, word));
        }
Exemplo n.º 8
0
        public static ExpressionToken TryParse(Scope scope, IEnumerable <string> endTokens,
                                               WordTokenCallback wordCallback = null, DataType expectedDataType = null)
        {
            var code = scope.Code;

            code.SkipWhiteSpace();
            var startPos = code.Position;

            // Statement breaking tokens
            if (code.PeekExact(';') || code.PeekExact('{') || code.PeekExact('}'))
            {
                return(null);
            }

            // Caller-specific end tokens
            if (endTokens != null)
            {
                code.Peek();
                if (endTokens.Contains(code.Text))
                {
                    return(null);
                }
            }

            var exp = new ExpressionToken(scope);

            var dataTypeArgs = new DataType.ParseArgs
            {
                Code             = code,
                DataTypeCallback = (name) =>
                {
                    return(scope.DefinitionProvider.GetAny <DataTypeDefinition>(code.Position, name).FirstOrDefault());
                },
                VariableCallback = (name) =>
                {
                    return(scope.DefinitionProvider.GetLocal <VariableDefinition>(code.Position, name).FirstOrDefault());
                },
                Scope = scope,
                TokenCreateCallback = (token) =>
                {
                    exp.AddToken(token);
                },
                VisibleModel = true
            };

            var abortParsing  = false;
            var parseDataType = expectedDataType;

            while (!code.EndOfFile && !abortParsing)
            {
                // Statement breaking tokens
                if (code.PeekExact(';') || code.PeekExact('{') || code.PeekExact('}'))
                {
                    break;
                }
                if (endTokens != null && code.Peek() && endTokens.Contains(code.Text))
                {
                    break;
                }

                if (!code.Read())
                {
                    break;
                }

                switch (code.Type)
                {
                case CodeType.Word:
                {
                    var word     = code.Text;
                    var wordSpan = code.Span;

                    Token token;
                    if (wordCallback != null && (token = wordCallback(word, wordSpan)) != null)
                    {
                        exp.AddToken(token);
                    }
                    else if (parseDataType != null && parseDataType.HasCompletionOptions && parseDataType.IsValidEnumOption(word))
                    {
                        exp.AddToken(new EnumOptionToken(scope, wordSpan, word, parseDataType));
                    }
                    else
                    {
                        var oldPos = code.Position;
                        code.Position = wordSpan.Start;                                         // DataType.TryParse() needs to be before the first word
                        var dt = DataType.TryParse(dataTypeArgs);
                        if (dt == null)
                        {
                            code.Position = oldPos;
                            var wordToken = ProcessWord(exp, scope, word, wordSpan);
                            if (wordToken != null)
                            {
                                exp.AddToken(wordToken);
                            }
                            else
                            {
                                code.Position = wordSpan.Start;
                                abortParsing  = true;
                            }
                        }
                    }
                }
                break;

                case CodeType.Number:
                    exp.AddToken(new NumberToken(scope, code.Span, code.Text));
                    break;

                case CodeType.StringLiteral:
                    if (parseDataType != null && parseDataType.HasCompletionOptions && parseDataType.IsValidEnumOption(code.Text))
                    {
                        exp.AddToken(new EnumOptionToken(scope, code.Span, code.Text, parseDataType));
                    }
                    else
                    {
                        exp.AddToken(new StringLiteralToken(scope, code.Span, code.Text));
                    }
                    break;

                case CodeType.Operator:
                    switch (code.Text)
                    {
                    case "(":
                    {
                        code.Position = code.Span.Start;
                        var bracketsToken = BracketsToken.Parse(scope, parseDataType);
                        if (bracketsToken.IsCast)
                        {
                            exp.AddToken(Statements.CastStatement.Parse(scope, bracketsToken, endTokens));
                        }
                        else
                        {
                            exp.AddToken(bracketsToken);
                        }
                    }
                    break;

                    case "{":
                        code.Position = code.Span.Start;
                        exp.AddToken(BracesToken.Parse(scope, null));
                        break;

                    case "[":
                        code.Position = code.Span.Start;
                        exp.AddToken(ArrayBracesToken.Parse(scope));
                        break;

                    case ",":
                        exp.AddToken(new DelimiterToken(scope, code.Span));
                        break;

                    case ".":
                        exp.AddToken(new DotToken(scope, code.Span));
                        break;

                    case "&":
                        exp.AddToken(new ReferenceToken(scope, code.Span));
                        break;

                    case "==":
                    case "!=":
                    case "<":
                    case "<=":
                    case ">":
                    case ">=":
                        if ((scope.Hint & ScopeHint.SuppressLogic) == 0)
                        {
                            if (exp.ChildrenCount > 0)
                            {
                                var dataType = exp.LastChild.ValueDataType;
                                if (dataType != null)
                                {
                                    parseDataType = dataType;
                                }
                            }
                            exp.AddToken(ComparisonOperator.Parse(scope, exp.LastChild, new OperatorToken(scope, code.Span, code.Text), endTokens));
                        }
                        else
                        {
                            exp.AddToken(new OperatorToken(scope, code.Span, code.Text));
                        }
                        break;

                    case "=":
                        if ((scope.Hint & ScopeHint.SuppressLogic) == 0)
                        {
                            if (exp.ChildrenCount > 0)
                            {
                                var dataType = exp.LastChild.ValueDataType;
                                if (dataType != null)
                                {
                                    parseDataType = dataType;
                                }
                            }
                            exp.AddToken(AssignmentOperator.Parse(scope, exp.LastChild, new OperatorToken(scope, code.Span, code.Text), endTokens));
                        }
                        else
                        {
                            exp.AddToken(new OperatorToken(scope, code.Span, code.Text));
                        }
                        break;

                    case "+=":
                    case "-=":
                    case "*=":
                    case "/=":
                    case "%=":
                        if ((scope.Hint & ScopeHint.SuppressLogic) == 0)
                        {
                            exp.AddToken(AssignmentOperator.Parse(scope, exp.LastChild, new OperatorToken(scope, code.Span, code.Text), endTokens));
                        }
                        else
                        {
                            exp.AddToken(new OperatorToken(scope, code.Span, code.Text));
                        }
                        break;

                    case "?":
                        parseDataType = expectedDataType;
                        exp.AddToken(ConditionalOperator.Parse(scope, exp.LastChild, new OperatorToken(scope, code.Span, code.Text),
                                                               endTokens, parseDataType));
                        break;

                    default:
                        exp.AddToken(new OperatorToken(scope, code.Span, code.Text));
                        break;
                    }
                    break;

                case CodeType.Preprocessor:
                    exp.AddToken(new PreprocessorToken(scope, code.Span, code.Text));
                    break;

                default:
                    exp.AddToken(new UnknownToken(scope, code.Span, code.Text));
                    break;
                }
            }

            if (exp.ChildrenCount == 0)
            {
                return(null);
            }
            return(exp);
        }
Exemplo n.º 9
0
        public static SelectStatement Parse(Scope parentScope, KeywordToken selectToken)
        {
            var ret   = new SelectStatement(parentScope, selectToken);
            var scope = parentScope.Clone();

            scope.BreakOwner    = ret;
            scope.ContinueOwner = ret;

            var code = scope.Code;

            if (code.ReadStringLiteral())
            {
                ret.AddToken(new StringLiteralToken(scope, code.Span, code.Text));
            }
            if (code.ReadExact('*'))
            {
                ret.AddToken(new OperatorToken(scope, code.Span, "*"));
            }

            if (!code.ReadExactWholeWord("from"))
            {
                return(ret);
            }
            ret.AddToken(new KeywordToken(scope, code.Span, "from"));

            ExtractTableDefinition extractDef = null;

            DkDict.Table table = null;

            if (code.ReadWord())
            {
                if ((table = DkDict.Dict.GetTable(code.Text)) != null)
                {
                    ret.AddToken(new TableToken(scope, code.Span, code.Text, table.Definition));
                }
                else if ((extractDef = scope.DefinitionProvider.GetAny <ExtractTableDefinition>(code.TokenStartPostion, code.Text).FirstOrDefault()) != null)
                {
                    ret.AddToken(new IdentifierToken(scope, code.Span, code.Text, extractDef));
                }
                else
                {
                    ret.AddToken(new UnknownToken(scope, code.Span, code.Text));
                }
            }

            if (code.ReadExactWholeWord("of"))
            {
                ret.AddToken(new KeywordToken(scope, code.Span, "of"));

                if (code.ReadWord())
                {
                    if ((table = DkDict.Dict.GetTable(code.Text)) != null)
                    {
                        ret.AddToken(new TableToken(scope, code.Span, code.Text, table.Definition));
                    }
                    else
                    {
                        ret.AddToken(new UnknownToken(scope, code.Span, code.Text));
                    }
                }
            }
            else if (code.ReadExact(','))
            {
                ret.AddToken(new DelimiterToken(scope, code.Span));

                var expectingComma = false;

                while (!code.EndOfFile)
                {
                    if (code.PeekExact('{'))
                    {
                        break;
                    }
                    if (expectingComma)
                    {
                        if (code.ReadExact(','))
                        {
                            ret.AddToken(new DelimiterToken(scope, code.Span));
                            expectingComma = false;
                        }
                        else
                        {
                            break;
                        }
                    }
                    else if (code.ReadWord())
                    {
                        if ((table = DkDict.Dict.GetTable(code.Text)) != null)
                        {
                            ret.AddToken(new TableToken(scope, code.Span, code.Text, table.Definition));
                        }
                        else
                        {
                            ret.AddToken(new UnknownToken(scope, code.Span, code.Text));
                        }
                        expectingComma = true;
                    }
                    else
                    {
                        break;
                    }
                }
            }

            // WHERE and ORDER BY
            var gotWhere   = false;
            var gotOrderBy = false;

            while (!code.EndOfFile)
            {
                if (code.PeekExact('{'))
                {
                    break;
                }
                if (!gotWhere && code.ReadExactWholeWord("where"))
                {
                    ret.AddToken(new KeywordToken(scope, code.Span, code.Text));
                    gotWhere = true;

                    var exp = ExpressionToken.TryParse(scope, _whereEndTokens);
                    if (exp != null)
                    {
                        ret.AddToken(exp);
                    }
                    else
                    {
                        break;
                    }
                }
                else if (!gotOrderBy && code.ReadExactWholeWord("order"))
                {
                    ret.AddToken(new KeywordToken(scope, code.Span, code.Text));
                    gotOrderBy = true;

                    if (!code.ReadExactWholeWord("by"))
                    {
                        break;
                    }
                    ret.AddToken(new KeywordToken(scope, code.Span, code.Text));

                    while (!code.EndOfFile)
                    {
                        if (code.PeekExact('{'))
                        {
                            break;
                        }

                        if (code.ReadExact(','))
                        {
                            ret.AddToken(new DelimiterToken(scope, code.Span));
                            continue;
                        }

                        if (code.ReadExactWholeWord("asc") || code.ReadExactWholeWord("desc"))
                        {
                            ret.AddToken(new KeywordToken(scope, code.Span, code.Text));
                            continue;
                        }

                        if (TryParseColumn(scope, ret, true, extractDef))
                        {
                            continue;
                        }

                        break;
                    }
                }
                else
                {
                    break;
                }
            }

            // Body
            if (code.ReadExact('{'))
            {
                var braces = new BracesToken(scope);
                braces.AddOpen(code.Span);
                ret.AddToken(braces);

                while (!code.EndOfFile)
                {
                    if (code.ReadExact('}'))
                    {
                        braces.AddClose(code.Span);
                        break;
                    }

                    if (code.ReadExactWholeWord("for"))
                    {
                        braces.AddToken(new KeywordToken(scope, code.Span, "for"));

                        if (!code.ReadExactWholeWord("each"))
                        {
                            continue;
                        }
                        braces.AddToken(new KeywordToken(scope, code.Span, "each"));

                        if (!code.ReadExact(':'))
                        {
                            continue;
                        }
                        braces.AddToken(new OperatorToken(scope, code.Span, ":"));
                    }
                    else if (code.ReadExactWholeWord("before") || code.ReadExactWholeWord("after"))
                    {
                        braces.AddToken(new KeywordToken(scope, code.Span, code.Text));

                        if (!code.ReadExactWholeWord("group"))
                        {
                            continue;
                        }
                        braces.AddToken(new KeywordToken(scope, code.Span, "group"));

                        if (code.ReadExactWholeWord("all"))
                        {
                            braces.AddToken(new KeywordToken(scope, code.Span, "all"));

                            if (!code.ReadExact(':'))
                            {
                                continue;
                            }
                            braces.AddToken(new OperatorToken(scope, code.Span, ":"));
                        }
                        else if (TryParseColumn(scope, braces, false, extractDef))
                        {
                            if (!code.ReadExact(':'))
                            {
                                continue;
                            }
                            braces.AddToken(new OperatorToken(scope, code.Span, ":"));
                        }
                    }
                    else if (code.ReadExactWholeWord("default"))
                    {
                        braces.AddToken(new KeywordToken(scope, code.Span, "default"));

                        if (!code.ReadExact(':'))
                        {
                            continue;
                        }
                        braces.AddToken(new OperatorToken(scope, code.Span, ":"));
                    }
                    else
                    {
                        var stmt = StatementToken.TryParse(scope);
                        if (stmt != null)
                        {
                            braces.AddToken(stmt);
                        }
                    }
                }
            }

            return(ret);
        }