private void ParseTokens()
{
_tokens = new List <Token>();
foreach (var dslScript in _dslScriptsProvider.DslScripts)
{
int scriptPosition = 0;
while (true)
{
TokenizerInternals.SkipWhitespaces(dslScript.Script, ref scriptPosition);
if (scriptPosition >= dslScript.Script.Length)
{
break;
}
int startPosition = scriptPosition;
Token t = TokenizerInternals.GetNextToken_ValueType(dslScript, ref scriptPosition);
t.DslScript = dslScript;
t.PositionInDslScript = startPosition;
if (t.Type != TokenType.Comment)
{
_tokens.Add(t);
}
}
_tokens.Add(new Token {
DslScript = dslScript, PositionInDslScript = dslScript.Script.Length, Type = TokenType.EndOfFile, Value = ""
});
}
}
public TokenizerResult GetTokens()
{
var tokens = new List <Token>();
DslSyntaxException syntaxError = null;
try
{
var tokenizerInternals = new TokenizerInternals(_syntax.Value);
foreach (var dslScript in _dslScriptsProvider.DslScripts)
{
int scriptPosition = 0;
while (true)
{
TokenizerInternals.SkipWhitespaces(dslScript.Script, ref scriptPosition);
if (scriptPosition >= dslScript.Script.Length)
{
break;
}
int startPosition = scriptPosition;
Token t = tokenizerInternals.GetNextToken_ValueType(dslScript, ref scriptPosition, _filesUtility.ReadAllText);
t.DslScript = dslScript;
t.PositionInDslScript = startPosition;
t.PositionEndInDslScript = scriptPosition;
if (t.Type != TokenType.Comment)
{
tokens.Add(t);
}
}
tokens.Add(new Token {
DslScript = dslScript, PositionInDslScript = dslScript.Script.Length, PositionEndInDslScript = dslScript.Script.Length, Type = TokenType.EndOfFile, Value = ""
});
}
}
catch (DslSyntaxException e)
{
syntaxError = e;
}
return(new TokenizerResult
{
Tokens = tokens,
SyntaxError = syntaxError
});
}