public void PushToken(Token token)
{
if (token != null)
this.tokenStack.Push(token);
}
private Token NextString()
{
string text = string.Empty;
char ch;
try
{
ch = this.NextChar();
while (ch != '"')
{
if (ch == '\\')
ch = this.NextChar();
text += ch;
ch = this.NextChar();
}
}
catch (EndOfInputException)
{
throw new LexerException("Unclosed string");
}
Token token = new Token() { TokenType = TokenType.String, Value = text };
return token;
}
private Token NextSymbol(char firstChar)
{
string name;
name = firstChar.ToString();
char ch;
try
{
ch = this.NextChar();
while (!char.IsWhiteSpace(ch) && !char.IsControl(ch) && SeparatorCharacters.IndexOf(ch) < 0)
{
name += ch;
ch = this.NextChar();
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token() { TokenType = TokenType.Symbol, Value = name };
return token;
}
private Token NextMacro(char firstChar)
{
string name = firstChar.ToString();
char ch;
try
{
ch = this.NextChar();
if (!char.IsWhiteSpace(ch))
{
string newname = name + ch.ToString();
if (macroBicharacters.Contains(newname))
name = newname;
else
this.PushChar(ch);
}
else
{
this.PushChar(ch);
}
}
catch (EndOfInputException)
{
}
Token token = new Token() { TokenType = TokenType.Macro, Value = name };
return token;
}
private Token NextKeyword()
{
string name = string.Empty;
char ch;
try
{
ch = this.NextChar();
while (!char.IsWhiteSpace(ch) && !char.IsControl(ch) && SeparatorCharacters.IndexOf(ch) < 0 && ch != '.')
{
name += ch;
ch = this.NextChar();
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token() { TokenType = TokenType.Keyword, Value = name };
return token;
}
private Token NextInteger(char firstChar)
{
string integer;
integer = new string(firstChar, 1);
char ch;
try
{
ch = this.NextChar();
while (char.IsDigit(ch))
{
integer += ch;
ch = this.NextChar();
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token() { TokenType = TokenType.Integer, Value = integer };
return token;
}
private Token NextCharacter()
{
Token token = new Token() { TokenType = TokenType.Character, Value = this.NextChar().ToString() };
try
{
char ch = this.NextChar();
if (!char.IsWhiteSpace(ch) && SeparatorCharacters.IndexOf(ch) == -1)
throw new LexerException("Invalid character");
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
return token;
}
public void PushToken()
{
Lexer lexer = new Lexer(string.Empty);
Token token = new Token { Value = "foo", TokenType = TokenType.Symbol };
lexer.PushToken(token);
Token retrieved = lexer.NextToken();
Assert.IsNotNull(retrieved);
Assert.AreEqual("foo", retrieved.Value);
Assert.AreEqual(TokenType.Symbol, retrieved.TokenType);
}