public static List <Token> Tokenize(string buffer)
{
TokenizerOutput ctx = new TokenizerOutput();
TokenizerState state = new TokenizerState(buffer);
while (!state.IsEndOfStream())
{
while (!state.IsEndOfStream() && state.GetChar() != '\n' && char.IsWhiteSpace(state.GetChar()))
{
state.NextChar();
}
if (state.IsEndOfStream())
{
break;
}
char c = state.GetChar();
switch (c)
{
case '\n':
state.NextLine();
state.NextChar();
break;
case '(':
ctx.AddSymbol(TokenType.BraceBegin, c, state.CreateInfo());
state.NextChar();
break;
case ')':
ctx.AddSymbol(TokenType.BraceEnd, c, state.CreateInfo());
state.NextChar();
break;
case ',':
ctx.AddSymbol(TokenType.ArgumentSeparator, c, state.CreateInfo());
state.NextChar();
break;
case '*':
ctx.AddSymbol(TokenType.Pointer, c, state.CreateInfo());
state.NextChar();
break;
default:
if (char.IsLetter(c) || c == '_')
{
int start = state.BufferPos;
while (!state.IsEndOfStream() && (char.IsLetterOrDigit(state.GetChar()) || (state.GetChar() == '_')))
{
state.NextChar();
}
int len = state.BufferPos - start;
string ident = buffer.Substring(start, len);
ctx.AddIdent(ident, state.CreateInfo());
}
else
{
ctx.AddChar(c, state.CreateInfo());
state.NextChar();
}
break;
}
}
return(ctx.Tokens);
}
