public static TokenCollection Tokenize(Core.Session session, Core.TextInput input)
{
var output = new TokenCollection();
// Iterate through all characters in input.
var index = 0;
while (index < input.Length())
{
// Skip whitespace.
if (IsWhitespace(input[index]))
{
index++;
continue;
}
// Match next characters to a token, and add it to the output.
var match =
TryMatchModelToken(input, index) ??
TryMatchVaryingToken(input, index) ??
new TokenMatch(new string(input[index], 1), TokenKind.Error);
var span = new Span(input, index, index + match.representation.Length);
// Signal errors.
if (match.kind == TokenKind.Error)
{
session.AddMessage(MessageKind.Error, MessageCode.UnexpectedChar, "unexpected character", span);
}
// Skip line comments.
else if (match.kind == TokenKind.DoubleSlash)
{
while (index < input.Length() && input[index] != '\n')
index++;
continue;
}
// Skip multiline comments.
else if (match.kind == TokenKind.SlashAsterisk)
{
var nesting = 1;
index += 2;
while (index < input.Length() - 1)
{
if (input[index] == '*' && input[index + 1] == '/')
{
nesting--;
index += 2;
if (nesting == 0)
break;
}
else if (input[index] == '/' && input[index + 1] == '*')
{
nesting++;
index += 2;
}
else
index++;
}
if (nesting > 0)
session.AddMessage(MessageKind.Error, MessageCode.UnexpectedChar, "unterminated comment", span);
continue;
}
output.tokens.Add(new Token(match.kind, span));
index += match.representation.Length;
}
output.tokenAfterEnd =
new Token(TokenKind.Error, new Diagnostics.Span(input, index, index));
return output;
}
public static TokenCollection Tokenize(Core.Session session, Core.TextInput input)
{
var output = new TokenCollection();
// Iterate through all characters in input.
var index = 0;
while (index < input.Length())
{
// Skip whitespace.
if (IsWhitespace(input[index]))
{
index++;
continue;
}
// Match next characters to a token, and add it to the output.
var match =
TryMatchModelToken(input, index) ??
TryMatchVaryingToken(input, index) ??
new TokenMatch(new string(input[index], 1), TokenKind.Error);
var span = new Span(input, index, index + match.representation.Length);
// Signal errors.
if (match.kind == TokenKind.Error)
{
session.AddMessage(MessageKind.Error, MessageCode.UnexpectedChar, "unexpected character", span);
}
// Skip line comments.
else if (match.kind == TokenKind.DoubleSlash)
{
while (index < input.Length() && input[index] != '\n')
{
index++;
}
continue;
}
// Skip multiline comments.
else if (match.kind == TokenKind.SlashAsterisk)
{
var nesting = 1;
index += 2;
while (index < input.Length() - 1)
{
if (input[index] == '*' && input[index + 1] == '/')
{
nesting--;
index += 2;
if (nesting == 0)
{
break;
}
}
else if (input[index] == '/' && input[index + 1] == '*')
{
nesting++;
index += 2;
}
else
{
index++;
}
}
if (nesting > 0)
{
session.AddMessage(MessageKind.Error, MessageCode.UnexpectedChar, "unterminated comment", span);
}
continue;
}
output.tokens.Add(new Token(match.kind, span));
index += match.representation.Length;
}
output.tokenAfterEnd =
new Token(TokenKind.Error, new Diagnostics.Span(input, index, index));
return(output);
}
