public Tokens GenerateTokens()
{
int N = Expression.Length;
Tokens tokens = new Tokens(N);
StringBuilder tokenValue = new StringBuilder(N);
State currentState = State.BEGIN;
int position = 0;
while (position <= N && !LexicalError)
{
char symbol = position < N ? Expression[position] : EndOfExpression;
switch (currentState)
{
case State.BEGIN:
if (Char.IsLetter(symbol))
{
tokenValue.Append(symbol);
currentState = State.VAR;
}
else if (Char.IsDigit(symbol))
{
tokenValue.Append(symbol);
currentState = State.NUM1;
}
else if ("+-*/^()".Contains(symbol))
{
tokenValue.Append(symbol);
currentState = State.OK;
}
else if (symbol != EndOfExpression)
{
if (position < N)
{
var error = String.Format("Invalid symbol {0}", symbol);
TokenError = new Token(TerminalSymbolType.UnRecognized, position, error);
tokens.Add(TokenError);
LexicalError = true;
}
}
position++;
break;
case State.VAR:
if (Char.IsLetter(symbol))
{
tokenValue.Append(symbol);
position++;
currentState = State.FUNC;
}
else
{
tokens.Add(new Token(TokenType.Variable, position, tokenValue.ToString()));
tokenValue.Clear();
currentState = State.BEGIN;
}
break;
case State.FUNC:
if (Char.IsLetter(symbol))
{
tokenValue.Append(symbol);
position++;
}
else
{
var str = tokenValue.ToString();
if (IsFunction(str))
{
tokens.Add(new Token(TokenType.Function, position, str));
tokenValue.Clear();
currentState = State.BEGIN;
}
else
{
var error = String.Format("Unrecognized {0}", str);
TokenError = new Token(TokenType.UnRecognized, position, error);
tokens.Add(TokenError);
LexicalError = true;
}
}
break;
case State.OK:
tokens.Add(makeOneCharToken(tokenValue.ToString(), position));
tokenValue.Clear();
currentState = State.BEGIN;
break;
case State.NUM1:
if (Char.IsDigit(symbol))
{
tokenValue.Append(symbol);
position++;
}
else if (symbol == '.')
{
tokenValue.Append(symbol);
position++;
currentState = State.TRANS;
}
else
{
tokens.Add(new Token(TokenType.Value, position, tokenValue.ToString()));
tokenValue.Clear();
currentState = State.BEGIN;
}
break;
case State.TRANS:
if (Char.IsDigit(symbol))
{
tokenValue.Append(symbol);
position++;
currentState = State.NUM2;
}
else
{
var error = String.Format("Invalid number format {0}", tokenValue.Append(symbol));
TokenError = new Token(TokenType.UnRecognized, position, error);
tokens.Add(TokenError);
LexicalError = true;
}
break;
case State.NUM2:
if (Char.IsDigit(symbol))
{
tokenValue.Append(symbol);
position++;
}
else
{
tokens.Add(new Token(TokenType.Value, position, tokenValue.ToString()));
tokenValue.Clear();
currentState = State.BEGIN;
}
break;
}
}
return tokens;
}
public void ValidateSintax()
{
var stack = new Stack<Symbol>();
stack.Push(new Symbol(NonTerminalSymbol.E));
int N = Tokens.Count;
int position = 0;
while (stack.Count > 0)
{
Symbol current = stack.Pop();
if (!current.IsTerminal)
{
NonTerminalSymbol symbolValue = current.NonTerminalSymbol;
Console.Out.Write(symbolValue);
if (symbolValue == NonTerminalSymbol.E)
{
Console.Out.WriteLine(" -> TX");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(NonTerminalSymbol.T));
}
else if (symbolValue == NonTerminalSymbol.T)
{
if (Tokens[position] == TerminalSymbolType.Function)
{
Console.Out.WriteLine(" -> F(E)X");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.ClosingParenthesis));
stack.Push(new Symbol(NonTerminalSymbol.E));
stack.Push(new Symbol(TerminalSymbolType.OpeningParenthesis));
stack.Push(new Symbol(TerminalSymbolType.Function));
}
else if (Tokens[position] == TerminalSymbolType.OpeningParenthesis)
{
Console.Out.WriteLine(" -> (E)X");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.ClosingParenthesis));
stack.Push(new Symbol(NonTerminalSymbol.E));
stack.Push(new Symbol(TerminalSymbolType.OpeningParenthesis));
}
else if (Tokens[position] == TerminalSymbolType.Value)
{
Console.Out.WriteLine(" -> valueX");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.Value));
}
else if (Tokens[position] == TerminalSymbolType.Variable)
{
Console.Out.WriteLine(" -> varX");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.Variable));
}
else if ("+-".Contains(Tokens[position].Value[0]))
{
Console.Out.WriteLine(" -> {0}U", Tokens[position].Value);
stack.Push(new Symbol(NonTerminalSymbol.U));
stack.Push(new Symbol(TerminalSymbolType.Operator));
}
else
{
var errorMsg = String.Format("Unexpected token '{0}'",
Tokens[position].Value);
TokenError = new TerminalSymbol(Tokens[position].Type,
Tokens[position].Column, errorMsg);
SyntaxError = true;
break;
}
}
else if (symbolValue == NonTerminalSymbol.X)
{
if (position >= N)
{
Console.Out.WriteLine(" -> eps");
}
else if (Tokens[position] == TerminalSymbolType.Operator)
{
Console.Out.WriteLine(" -> {0}E", Tokens[position].Value);
stack.Push(new Symbol(NonTerminalSymbol.U));
stack.Push(new Symbol(TerminalSymbolType.Operator));
}
else if (Tokens[position] != TerminalSymbolType.ClosingParenthesis)
{
var errorMsg = String.Format("Unexpected token '{0}'",
Tokens[position].Value);
TokenError = new TerminalSymbol(Tokens[position].Type,
Tokens[position].Column, errorMsg);
SyntaxError = true;
break;
}
else
{ // Reading a closing parenthesis
Console.Out.WriteLine(" -> eps");
}
}
else if (symbolValue == NonTerminalSymbol.U)
{
if (Tokens[position] == TerminalSymbolType.Function)
{
Console.Out.WriteLine(" -> F(E)X");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.ClosingParenthesis));
stack.Push(new Symbol(NonTerminalSymbol.E));
stack.Push(new Symbol(TerminalSymbolType.OpeningParenthesis));
stack.Push(new Symbol(TerminalSymbolType.Function));
}
else if (Tokens[position] == TerminalSymbolType.OpeningParenthesis)
{
Console.Out.WriteLine(" -> (E)X");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.ClosingParenthesis));
stack.Push(new Symbol(NonTerminalSymbol.E));
stack.Push(new Symbol(TerminalSymbolType.OpeningParenthesis));
}
else if (Tokens[position] == TerminalSymbolType.Value)
{
Console.Out.WriteLine(" -> valueX");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.Value));
}
else if (Tokens[position] == TerminalSymbolType.Variable)
{
Console.Out.WriteLine(" -> varX");
stack.Push(new Symbol(NonTerminalSymbol.X));
stack.Push(new Symbol(TerminalSymbolType.Variable));
}
else
{
var errorMsg = String.Format("Unexpected token '{0}'",
Tokens[position].Value);
TokenError = new TerminalSymbol(Tokens[position].Type,
Tokens[position].Column, errorMsg);
SyntaxError = true;
break;
}
}
}
else
{
TerminalSymbolType symbolValue = current.TerminalSymbol;
Console.Out.Write(" >> Expecting {0}", symbolValue);
if (position < N)
{
if (Tokens[position] == symbolValue)
{
Console.Out.WriteLine(" - matched with {0}", Tokens[position]);
position++;
}
else
{
var errorMsg = String.Format("Expecting {0}, got {1}",
symbolValue, Tokens[position].Value);
TokenError = new TerminalSymbol(Tokens[position].Type,
Tokens[position].Column, errorMsg);
SyntaxError = true;
break;
}
}
else
{
var errorMsg = String.Format("Expecting {0}, got enf of expression",
symbolValue);
TokenError = new TerminalSymbol(TerminalSymbolType.UnRecognized,
N, errorMsg);
SyntaxError = true;
break;
}
}
}
if (position < N && !SyntaxError)
{
TokenError = new TerminalSymbol(TerminalSymbolType.UnRecognized, position,
"Sintax error");
SyntaxError = true;
}
}
