public void ExampleProgramIsTokenizedCorrectly()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\example_program.txt", reporter, 8);
TokenHelper(lexer, new PrintToken(), 0, 0);
TokenHelper(lexer, new TextToken("Give a number"), 0, 6);
TokenHelper(lexer, new SemicolonToken(), 0, 21);
TokenHelper(lexer, new VarToken(), 1, 0);
TokenHelper(lexer, new IdentifierToken("n"), 1, 4);
TokenHelper(lexer, new ColonToken(), 1, 6);
TokenHelper(lexer, new IntToken(), 1, 8);
TokenHelper(lexer, new SemicolonToken(), 1, 11);
TokenHelper(lexer, new ReadToken(), 2, 0);
TokenHelper(lexer, new IdentifierToken("n"), 2, 5);
TokenHelper(lexer, new SemicolonToken(), 2, 6);
TokenHelper(lexer, new VarToken(), 3, 0);
TokenHelper(lexer, new IdentifierToken("v"), 3, 4);
TokenHelper(lexer, new ColonToken(), 3, 6);
TokenHelper(lexer, new IntToken(), 3, 8);
TokenHelper(lexer, new AssignmentToken(), 3, 12);
TokenHelper(lexer, new NumberToken(1), 3, 15);
TokenHelper(lexer, new SemicolonToken(), 3, 16);
TokenHelper(lexer, new VarToken(), 4, 0);
TokenHelper(lexer, new IdentifierToken("i"), 4, 4);
TokenHelper(lexer, new ColonToken(), 4, 6);
TokenHelper(lexer, new IntToken(), 4, 8);
TokenHelper(lexer, new SemicolonToken(), 4, 11);
TokenHelper(lexer, new ForToken(), 5, 0);
TokenHelper(lexer, new IdentifierToken("i"), 5, 4);
TokenHelper(lexer, new InToken(), 5, 6);
TokenHelper(lexer, new NumberToken(1), 5, 9);
TokenHelper(lexer, new RangeToken(), 5, 10);
TokenHelper(lexer, new IdentifierToken("n"), 5, 12);
TokenHelper(lexer, new DoToken(), 5, 14);
TokenHelper(lexer, new IdentifierToken("v"), 6, 8);
TokenHelper(lexer, new AssignmentToken(), 6, 10);
TokenHelper(lexer, new IdentifierToken("v"), 6, 13);
TokenHelper(lexer, new MultiplyToken(), 6, 15);
TokenHelper(lexer, new IdentifierToken("i"), 6, 17);
TokenHelper(lexer, new SemicolonToken(), 6, 18);
TokenHelper(lexer, new EndToken(), 7, 0);
TokenHelper(lexer, new ForToken(), 7, 4);
TokenHelper(lexer, new SemicolonToken(), 7, 7);
TokenHelper(lexer, new PrintToken(), 8, 0);
TokenHelper(lexer, new TextToken("The result is: "), 8, 6);
TokenHelper(lexer, new SemicolonToken(), 8, 23);
TokenHelper(lexer, new PrintToken(), 9, 0);
TokenHelper(lexer, new IdentifierToken("v"), 9, 6);
TokenHelper(lexer, new SemicolonToken(), 9, 7);
Assert.AreEqual(0, reporter.Errors.Count);
}
public void ExampleProgram2IsTokenizedCorrectly()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\example_program2.txt", reporter, 8);
TokenHelper(lexer, new VarToken(), 0, 0);
TokenHelper(lexer, new IdentifierToken("X"), 0, 4);
TokenHelper(lexer, new ColonToken(), 0, 6);
TokenHelper(lexer, new IntToken(), 0, 8);
TokenHelper(lexer, new AssignmentToken(), 0, 12);
TokenHelper(lexer, new NumberToken(4), 0, 15);
TokenHelper(lexer, new PlusToken(), 0, 17);
TokenHelper(lexer, new LParenToken(), 0, 19);
TokenHelper(lexer, new NumberToken(6), 0, 20);
TokenHelper(lexer, new MultiplyToken(), 0, 22);
TokenHelper(lexer, new NumberToken(2), 0, 24);
TokenHelper(lexer, new RParenToken(), 0, 25);
TokenHelper(lexer, new SemicolonToken(), 0, 26);
TokenHelper(lexer, new PrintToken(), 1, 0);
TokenHelper(lexer, new IdentifierToken("X"), 1, 6);
TokenHelper(lexer, new SemicolonToken(), 1, 7);
Assert.AreEqual(0, reporter.Errors.Count);
}
public void LexerBackTrackingThrowsIfCalledWithNonFullBufferTooManyTimes()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\example_program.txt", reporter);
bool throws = false;
try
{
lexer.Backtrack();
}
catch (InternalCompilerError e)
{
throws = true;
}
Assert.IsTrue(throws);
Assert.AreEqual(0, reporter.Errors.Count);
}
public void LexerBackTrackingThrowsIfCalledTooManyTimes()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\example_program.txt", reporter);
Assert.AreEqual(new PrintToken(), lexer.NextToken());
Assert.AreEqual(new TextToken("Give a number"), lexer.NextToken());
Assert.AreEqual(new SemicolonToken(), lexer.NextToken());
lexer.NextToken();
for (int i = 0; i < Lexer.BACKTRACK_BUFFER_SIZE; ++i)
{
lexer.Backtrack();
}
bool throws = false;
try
{
lexer.Backtrack();
} catch (InternalCompilerError e)
{
throws = true;
}
Assert.IsTrue(throws);
Assert.AreEqual(0, reporter.Errors.Count);
}
public void GetTokensHandlesInvalidIntegers()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\invalid_integers.txt", reporter);
Assert.AreEqual(new NumberToken(567), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("Hello"), lexer.NextToken());
Assert.AreEqual(new NumberToken(1), lexer.NextToken());
Assert.AreEqual(2, reporter.Errors.Count);
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type);
Assert.AreEqual(0, reporter.Errors[0].Line);
Assert.AreEqual(3, reporter.Errors[0].Column);
Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid character"));
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type);
Assert.AreEqual(1, reporter.Errors[1].Line);
Assert.AreEqual(0, reporter.Errors[1].Column);
Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("number does not fit"));
}
public void GetTokensHandlesIntegers()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\valid_integers.txt", reporter);
Assert.AreEqual(new NumberToken(123), lexer.NextToken());
Assert.AreEqual(new NumberToken(0), lexer.NextToken());
Assert.AreEqual(new NumberToken(9876543210), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}
public void GetTokenSetsLineAndColumnCorrectlyWith8SpacesPerTab()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\line_and_column.txt", reporter, 8);
Token token;
token = lexer.NextToken();
Assert.AreEqual(0, token.Line);
Assert.AreEqual(0, token.Column);
token = lexer.NextToken();
Assert.AreEqual(0, token.Line);
Assert.AreEqual(6, token.Column);
token = lexer.NextToken();
Assert.AreEqual(2, token.Line);
Assert.AreEqual(0, token.Column);
token = lexer.NextToken();
Assert.AreEqual(4, token.Line);
Assert.AreEqual(8, token.Column);
token = lexer.NextToken();
Assert.AreEqual(5, token.Line);
Assert.AreEqual(16, token.Column);
token = lexer.NextToken();
Assert.AreEqual(7, token.Line);
Assert.AreEqual(8, token.Column);
token = lexer.NextToken();
Assert.AreEqual(8, token.Line);
Assert.AreEqual(8, token.Column);
Assert.AreEqual(0, reporter.Errors.Count);
}
public void GetTokenHandlesInvalidStrings()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\invalid_strings.txt", reporter);
Assert.AreEqual(new TextToken("invaliddescape"), lexer.NextToken());
Assert.AreEqual(new TextToken("unterminated"), lexer.NextToken());
Assert.AreEqual(new TextToken("valid for checking unterminated"), lexer.NextToken());
Assert.AreEqual(2, reporter.Errors.Count);
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type);
Assert.AreEqual(0, reporter.Errors[0].Line);
Assert.AreEqual(9, reporter.Errors[0].Column);
Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid escape sequence"));
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type);
Assert.AreEqual(1, reporter.Errors[1].Line);
Assert.AreEqual(13, reporter.Errors[1].Column);
Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("unmatched"));
}
public void GetTokenParsesOperatorsCorrectly()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\operators.txt", reporter);
Assert.AreEqual(new PlusToken(), lexer.NextToken());
Assert.AreEqual(new MinusToken(), lexer.NextToken());
Assert.AreEqual(new MultiplyToken(), lexer.NextToken());
Assert.AreEqual(new DivideToken(), lexer.NextToken());
Assert.AreEqual(new AssignmentToken(), lexer.NextToken());
Assert.AreEqual(new LessThanToken(), lexer.NextToken());
Assert.AreEqual(new ComparisonToken(), lexer.NextToken());
Assert.AreEqual(new AndToken(), lexer.NextToken());
Assert.AreEqual(new NotToken(), lexer.NextToken());
Assert.AreEqual(new SemicolonToken(), lexer.NextToken());
Assert.AreEqual(new ColonToken(), lexer.NextToken());
Assert.AreEqual(new RangeToken(), lexer.NextToken());
Assert.AreEqual(new LParenToken(), lexer.NextToken());
Assert.AreEqual(new RParenToken(), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}
public void GetTokenParsesHandlesInvalidOperators()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\invalid_operators.txt", reporter);
Assert.AreEqual(new IdentifierToken("this"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("contains"), lexer.NextToken());
Assert.AreEqual(new RangeToken(), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("invalid_operators"), lexer.NextToken());
Assert.AreEqual(2, reporter.Errors.Count);
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type);
Assert.AreEqual(0, reporter.Errors[0].Line);
Assert.AreEqual(5, reporter.Errors[0].Column);
Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid operator"));
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type);
Assert.AreEqual(0, reporter.Errors[1].Line);
Assert.AreEqual(17, reporter.Errors[1].Column);
Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("invalid operator"));
}
public void GetTokenParsesHandlesInvalidCharacters()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\invalid_characters.txt", reporter);
Assert.AreEqual(new IdentifierToken("hello"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("there"), lexer.NextToken());
Assert.AreEqual(new TextToken("just"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("testing"), lexer.NextToken());
Assert.AreEqual(new NumberToken(1234), lexer.NextToken());
Assert.AreEqual(4, reporter.Errors.Count);
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type);
Assert.AreEqual(0, reporter.Errors[0].Line);
Assert.AreEqual(0, reporter.Errors[0].Column);
Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid start"));
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type);
Assert.AreEqual(0, reporter.Errors[1].Line);
Assert.AreEqual(6, reporter.Errors[1].Column);
Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("invalid start"));
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[2].Type);
Assert.AreEqual(0, reporter.Errors[2].Line);
Assert.AreEqual(18, reporter.Errors[2].Column);
Assert.IsTrue(reporter.Errors[2].Message.ToLower().Contains("invalid start"));
Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[3].Type);
Assert.AreEqual(0, reporter.Errors[3].Line);
Assert.AreEqual(26, reporter.Errors[3].Column);
Assert.IsTrue(reporter.Errors[3].Message.ToLower().Contains("invalid start"));
}
public void GetTokenParsesCommentsCorrectly()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\comments.txt", reporter);
Assert.AreEqual(new IdentifierToken("valid_token"), lexer.NextToken());
Assert.AreEqual(new DivideToken(), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("anothertoken"), lexer.NextToken());
Assert.AreEqual(new NumberToken(1234), lexer.NextToken());
Assert.AreEqual(new TextToken("test"), lexer.NextToken());
Assert.AreEqual(new EOFToken(), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}
public void GetTokenHandlesValidStrings()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\valid_strings.txt", reporter);
Assert.AreEqual(new TextToken("hello"), lexer.NextToken());
Assert.AreEqual(new TextToken("wor\tld"), lexer.NextToken());
Assert.AreEqual(new TextToken("thi\ns"), lexer.NextToken());
Assert.AreEqual(new TextToken("\"is\""), lexer.NextToken());
Assert.AreEqual(new TextToken("a\rtest"), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}
public void GetTokenHandlesReservedWords()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\keywords.txt", reporter);
Assert.AreEqual(new VarToken(), lexer.NextToken());
Assert.AreEqual(new ForToken(), lexer.NextToken());
Assert.AreEqual(new EndToken(), lexer.NextToken());
Assert.AreEqual(new InToken(), lexer.NextToken());
Assert.AreEqual(new DoToken(), lexer.NextToken());
Assert.AreEqual(new ReadToken(), lexer.NextToken());
Assert.AreEqual(new PrintToken(), lexer.NextToken());
Assert.AreEqual(new IntToken(), lexer.NextToken());
Assert.AreEqual(new StringToken(), lexer.NextToken());
Assert.AreEqual(new BoolToken(), lexer.NextToken());
Assert.AreEqual(new AssertToken(), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}
public void LexerPeekingWorksWithBacktracking()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\example_program.txt", reporter);
lexer.NextToken();
lexer.NextToken();
lexer.NextToken();
Assert.AreEqual(new VarToken(), lexer.PeekToken());
lexer.Backtrack();
Assert.AreEqual(new SemicolonToken(), lexer.PeekToken());
Assert.AreEqual(new SemicolonToken(), lexer.NextToken());
Assert.AreEqual(new VarToken(), lexer.PeekToken());
Assert.AreEqual(new VarToken(), lexer.PeekToken());
lexer.Backtrack();
lexer.Backtrack();
Assert.AreEqual(new TextToken("Give a number"), lexer.PeekToken());
Assert.AreEqual(new TextToken("Give a number"), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}
public void GetTokenReturnsEOFOnEndOfFile()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\empty.txt", reporter);
Assert.AreEqual(new EOFToken(), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}
private void TokenHelper(Lexer lexer, Token expected, int line, int column)
{
var actual = lexer.NextToken();
Assert.AreEqual(expected, actual);
Assert.AreEqual(line, actual.Line);
Assert.AreEqual(column, actual.Column);
}
public void GetTokenHandlesIdentifiers()
{
var reporter = new ErrorReporter();
var lexer = new Lexer(@"..\..\Lexing\valid_identifiers.txt", reporter);
Assert.AreEqual(new IdentifierToken("hello"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("w_or_ld"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("this_"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("i123s"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("a987"), lexer.NextToken());
Assert.AreEqual(new IdentifierToken("t_12_es_190t"), lexer.NextToken());
Assert.AreEqual(0, reporter.Errors.Count);
}