public Lexer(PushBackCharacterStream inputText)
{
_source = inputText;
}
public void ShouldReturnTwoSymbolsSeparatedByWhitespace()
{
var stream = new PushBackCharacterStream(new StringReader("symbol1 symbol2"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Symbol, token.Type);
Assert.AreEqual("symbol1", token.Text);
token = lexer.Next();
Assert.AreEqual(TokenType.Whitespace, token.Type);
token = lexer.Next();
Assert.AreEqual(TokenType.Symbol, token.Type);
Assert.AreEqual("symbol2", token.Text);
}
public void ShouldStopParsingSymbolWhenDoubleQuoteFound()
{
var stream = new PushBackCharacterStream(new StringReader("asdf\"str\""));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Symbol, token.Type);
Assert.AreEqual("asdf", token.Text);
token = lexer.Next();
Assert.AreEqual(TokenType.String, token.Type);
Assert.AreEqual("\"str\"", token.Text);
}
public void ShouldReturnSymbolImmediatelyFollowedByComment()
{
var stream = new PushBackCharacterStream(new StringReader("test;comment"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Symbol, token.Type);
Assert.AreEqual("test", token.Text);
token = lexer.Next();
Assert.AreEqual(TokenType.Comment, token.Type);
Assert.AreEqual(";comment", token.Text);
}
public void ShouldReturnSymbolWhenItHasADot()
{
var stream = new PushBackCharacterStream(new StringReader("namespace.test"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Symbol, token.Type);
Assert.AreEqual("namespace.test", token.Text);
}
public void ShouldReturnNumberTokenTypeWhenInputIsNumber()
{
var stream = new PushBackCharacterStream(new StringReader("123"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Number, token.Type);
Assert.AreEqual("123", token.Text);
Assert.IsFalse(stream.HasMore);
}
public void ShouldReturnRealNumber()
{
var stream = new PushBackCharacterStream(new StringReader("123.321"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Number, token.Type);
Assert.AreEqual("123.321", token.Text);
}
public void ShouldReturnNil()
{
var stream = new PushBackCharacterStream(new StringReader("nil"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Nil, token.Type);
Assert.AreEqual("nil", token.Text);
Assert.IsFalse(stream.HasMore);
}
public void ShouldReturnNumberFollowByWhitespaceAndAString()
{
var stream = new PushBackCharacterStream(new StringReader("123 \"asdf\""));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Number, token.Type);
Assert.AreEqual("123", token.Text);
Assert.AreEqual(0, token.StartIndex);
token = lexer.Next();
Assert.AreEqual(TokenType.Whitespace, token.Type);
Assert.AreEqual(" ", token.Text);
Assert.AreEqual(3, token.StartIndex);
token = lexer.Next();
Assert.AreEqual(TokenType.String, token.Type);
Assert.AreEqual("\"asdf\"", token.Text);
Assert.AreEqual(4, token.StartIndex);
}
public void ShouldReturnKeywordWithNoName()
{
var stream = new PushBackCharacterStream(new StringReader(":"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Keyword, token.Type);
Assert.AreEqual(":", token.Text);
}
public void ShouldReturnKeywordFollowByListStart()
{
var stream = new PushBackCharacterStream(new StringReader(":asdf("));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Keyword, token.Type);
Assert.AreEqual(":asdf", token.Text);
Assert.AreEqual(TokenType.ListStart, lexer.Next().Type);
}
public void ShouldReturnCommentWithTrailingWhitespace()
{
var stream = new PushBackCharacterStream(new StringReader("; test text \r\n"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Comment, token.Type);
Assert.AreEqual("; test text ", token.Text);
token = lexer.Next();
Assert.AreEqual(TokenType.Whitespace, token.Type);
Assert.AreEqual("\r\n", token.Text);
}
public void ShouldReturnCommentToEndOfLineOnly()
{
var stream = new PushBackCharacterStream(new StringReader("; test\r\n123"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Comment, token.Type);
Assert.AreEqual("; test", token.Text);
}
public void ShouldReturnBooleanWhenTrueIsInput()
{
var stream = new PushBackCharacterStream(new StringReader("true"));
Lexer lexer = new Lexer(stream);
Token token = lexer.Next();
Assert.AreEqual(TokenType.Boolean, token.Type);
Assert.AreEqual("true", token.Text);
Assert.IsFalse(stream.HasMore);
}
