// Tokenizer just skips comments, so we are unable to detect whether typing is done inside a comment
// Therefore we need to reparse and detect all comment tokens
private List <Token> ParseCommentTokens()
{
var script = textDocument.DslScripts.Single();
var commentTokens = new List <Token>();
var tokenizerInternals = new TokenizerInternals(rhetosProjectContext.DslSyntax);
try
{
var scriptPosition = 0;
while (true)
{
TokenizerInternals.SkipWhitespaces(script.Script, ref scriptPosition);
if (scriptPosition >= textDocument.Text.Length)
{
break;
}
var startPosition = scriptPosition;
var token = tokenizerInternals.GetNextToken_ValueType(script, ref scriptPosition, _ => "");
token.DslScript = script;
token.PositionInDslScript = startPosition;
if (token.Type == TokenType.Comment)
{
commentTokens.Add(token);
}
}
}
catch
{
// we will ignore all errors as any relevant ones are captured by CreateTokenizerWithCapturedErrors()
}
return(commentTokens);
}
public void SkipWhitespaces_AllWhitespaces()
{
string dsl = "a \t\r\n. \t\r\n";
int position = 1;
TokenizerInternals.SkipWhitespaces(dsl, ref position);
Assert.AreEqual('.', dsl[position]);
}
public void SkipWhitespaces_EndOfText()
{
string dsl = "a ";
int position = 2;
TokenizerInternals.SkipWhitespaces(dsl, ref position);
Assert.AreEqual(dsl.Length, position);
}
public void SkipWhitespaces_NoWhitespace()
{
string dsl = "a b";
int position = 0;
TokenizerInternals.SkipWhitespaces(dsl, ref position);
Assert.AreEqual(0, position);
}