Exemple #1
0
        private void ParseTokens()
        {
            _tokens = new List <Token>();

            foreach (var dslScript in _dslScriptsProvider.DslScripts)
            {
                int scriptPosition = 0;

                while (true)
                {
                    TokenizerInternals.SkipWhitespaces(dslScript.Script, ref scriptPosition);
                    if (scriptPosition >= dslScript.Script.Length)
                    {
                        break;
                    }

                    int   startPosition = scriptPosition;
                    Token t             = TokenizerInternals.GetNextToken_ValueType(dslScript, ref scriptPosition);
                    t.DslScript           = dslScript;
                    t.PositionInDslScript = startPosition;

                    if (t.Type != TokenType.Comment)
                    {
                        _tokens.Add(t);
                    }
                }

                _tokens.Add(new Token {
                    DslScript = dslScript, PositionInDslScript = dslScript.Script.Length, Type = TokenType.EndOfFile, Value = ""
                });
            }
        }
        public TokenizerResult GetTokens()
        {
            var tokens = new List <Token>();
            DslSyntaxException syntaxError = null;

            try
            {
                var tokenizerInternals = new TokenizerInternals(_syntax.Value);

                foreach (var dslScript in _dslScriptsProvider.DslScripts)
                {
                    int scriptPosition = 0;

                    while (true)
                    {
                        TokenizerInternals.SkipWhitespaces(dslScript.Script, ref scriptPosition);
                        if (scriptPosition >= dslScript.Script.Length)
                        {
                            break;
                        }

                        int   startPosition = scriptPosition;
                        Token t             = tokenizerInternals.GetNextToken_ValueType(dslScript, ref scriptPosition, _filesUtility.ReadAllText);
                        t.DslScript              = dslScript;
                        t.PositionInDslScript    = startPosition;
                        t.PositionEndInDslScript = scriptPosition;

                        if (t.Type != TokenType.Comment)
                        {
                            tokens.Add(t);
                        }
                    }

                    tokens.Add(new Token {
                        DslScript = dslScript, PositionInDslScript = dslScript.Script.Length, PositionEndInDslScript = dslScript.Script.Length, Type = TokenType.EndOfFile, Value = ""
                    });
                }
            }
            catch (DslSyntaxException e)
            {
                syntaxError = e;
            }

            return(new TokenizerResult
            {
                Tokens = tokens,
                SyntaxError = syntaxError
            });
        }