private DfaState InitToken(char ch)
{
if (_tokenText != null && _tokenText.Length > 0)
{
_token.Text = _tokenText.ToString();
_tokens.Add(_token);
_tokenText = new StringBuilder();
_token = new SimpleToken();
}
DfaState newState = DfaState.Initial;
if (IsAlpha(ch))
{
if (ch == 'i')
{
newState = DfaState.Id_int1;
}
else
{
newState = DfaState.Id;
}
_token.Type = TokenType.Identifier;
_tokenText.Append(ch);
}
else if (IsDigit(ch))
{
newState = DfaState.IntLiteral;
_token.Type = TokenType.IntLiteral;
_tokenText.Append(ch);
}
else if (ch == '>')
{
newState = DfaState.GT;
_token.Type = TokenType.GT;
_tokenText.Append(ch);
}
else if (ch == '+')
{
newState = DfaState.Plus;
_token.Type = TokenType.Plus;
_tokenText.Append(ch);
}
else if (ch == '-')
{
newState = DfaState.Minus;
_token.Type = TokenType.Minus;
_tokenText.Append(ch);
}
else if (ch == '*')
{
newState = DfaState.Star;
_token.Type = TokenType.Star;
_tokenText.Append(ch);
}
else if (ch == '/')
{
newState = DfaState.Slash;
_token.Type = TokenType.Slash;
_tokenText.Append(ch);
}
else if (ch == ';')
{
newState = DfaState.SemiColon;
_token.Type = TokenType.SemiColon;
_tokenText.Append(ch);
}
else if (ch == '(')
{
newState = DfaState.LeftParen;
_token.Type = TokenType.LeftParen;
_tokenText.Append(ch);
}
else if (ch == ')')
{
newState = DfaState.RightParen;
_token.Type = TokenType.RightParen;
_tokenText.Append(ch);
}
else if (ch == '=')
{
newState = DfaState.Assignment;
_token.Type = TokenType.Assignment;
_tokenText.Append(ch);
}
else
{
newState = DfaState.Initial;
}
return(newState);
}
public SimpleTokenReader Tokenize(string code)
{
_tokens = new List <Token>();
StringReader reader = new StringReader(code);
_tokenText = new StringBuilder();
_token = new SimpleToken();
int ich = 0;
char ch = '0';
DfaState state = DfaState.Initial;
try
{
while ((ich = reader.Read()) != -1)
{
ch = (char)ich;
switch (state)
{
case DfaState.Initial:
state = InitToken(ch);
break;
case DfaState.Id:
if (IsAlpha(ch) || IsDigit(ch))
{
_tokenText.Append(ch);
}
else
{
state = InitToken(ch);
}
break;
case DfaState.GT:
if (ch == '=')
{
_token.Type = TokenType.GE;
state = DfaState.GE;
_tokenText.Append(ch);
}
else
{
state = InitToken(ch);
}
break;
case DfaState.GE:
case DfaState.Assignment:
case DfaState.Plus:
case DfaState.Minus:
case DfaState.Star:
case DfaState.Slash:
case DfaState.SemiColon:
case DfaState.LeftParen:
case DfaState.RightParen:
state = InitToken(ch);
break;
case DfaState.IntLiteral:
if (IsDigit(ch))
{
_tokenText.Append(ch);
}
else
{
state = InitToken(ch);
}
break;
case DfaState.Id_int1:
if (ch == 'n')
{
state = DfaState.Id_int2;
_tokenText.Append(ch);
}
else if (IsDigit(ch) || IsAlpha(ch))
{
state = DfaState.Id;
_tokenText.Append(ch);
}
else
{
state = InitToken(ch);
}
break;
case DfaState.Id_int2:
if (ch == 't')
{
state = DfaState.Id_int3;
_tokenText.Append(ch);
}
else if (IsDigit(ch) || IsAlpha(ch))
{
state = DfaState.Id;
_tokenText.Append(ch);
}
else
{
state = InitToken(ch);
}
break;
case DfaState.Id_int3:
if (IsBlank(ch))
{
_token.Type = TokenType.Int;
state = InitToken(ch);
}
else
{
state = DfaState.Id;
_tokenText.Append(ch);
}
break;
default:
break;
}
}
if (_tokenText.Length > 0)
{
InitToken(ch);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(new SimpleTokenReader(_tokens));
}