public void LexerPeekingWorksWithBacktracking() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\example_program.txt", reporter); lexer.NextToken(); lexer.NextToken(); lexer.NextToken(); Assert.AreEqual(new VarToken(), lexer.PeekToken()); lexer.Backtrack(); Assert.AreEqual(new SemicolonToken(), lexer.PeekToken()); Assert.AreEqual(new SemicolonToken(), lexer.NextToken()); Assert.AreEqual(new VarToken(), lexer.PeekToken()); Assert.AreEqual(new VarToken(), lexer.PeekToken()); lexer.Backtrack(); lexer.Backtrack(); Assert.AreEqual(new TextToken("Give a number"), lexer.PeekToken()); Assert.AreEqual(new TextToken("Give a number"), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }
private void TokenHelper(Lexer lexer, Token expected, int line, int column) { var actual = lexer.NextToken(); Assert.AreEqual(expected, actual); Assert.AreEqual(line, actual.Line); Assert.AreEqual(column, actual.Column); }
public void GetTokensHandlesInvalidIntegers() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\invalid_integers.txt", reporter); Assert.AreEqual(new NumberToken(567), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("Hello"), lexer.NextToken()); Assert.AreEqual(new NumberToken(1), lexer.NextToken()); Assert.AreEqual(2, reporter.Errors.Count); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type); Assert.AreEqual(0, reporter.Errors[0].Line); Assert.AreEqual(3, reporter.Errors[0].Column); Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid character")); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type); Assert.AreEqual(1, reporter.Errors[1].Line); Assert.AreEqual(0, reporter.Errors[1].Column); Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("number does not fit")); }
public void LexerBackTrackingThrowsIfCalledTooManyTimes() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\example_program.txt", reporter); Assert.AreEqual(new PrintToken(), lexer.NextToken()); Assert.AreEqual(new TextToken("Give a number"), lexer.NextToken()); Assert.AreEqual(new SemicolonToken(), lexer.NextToken()); lexer.NextToken(); for (int i = 0; i < Lexer.BACKTRACK_BUFFER_SIZE; ++i) { lexer.Backtrack(); } bool throws = false; try { lexer.Backtrack(); } catch (InternalCompilerError e) { throws = true; } Assert.IsTrue(throws); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokenSetsLineAndColumnCorrectlyWith8SpacesPerTab() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\line_and_column.txt", reporter, 8); Token token; token = lexer.NextToken(); Assert.AreEqual(0, token.Line); Assert.AreEqual(0, token.Column); token = lexer.NextToken(); Assert.AreEqual(0, token.Line); Assert.AreEqual(6, token.Column); token = lexer.NextToken(); Assert.AreEqual(2, token.Line); Assert.AreEqual(0, token.Column); token = lexer.NextToken(); Assert.AreEqual(4, token.Line); Assert.AreEqual(8, token.Column); token = lexer.NextToken(); Assert.AreEqual(5, token.Line); Assert.AreEqual(16, token.Column); token = lexer.NextToken(); Assert.AreEqual(7, token.Line); Assert.AreEqual(8, token.Column); token = lexer.NextToken(); Assert.AreEqual(8, token.Line); Assert.AreEqual(8, token.Column); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokensHandlesIntegers() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\valid_integers.txt", reporter); Assert.AreEqual(new NumberToken(123), lexer.NextToken()); Assert.AreEqual(new NumberToken(0), lexer.NextToken()); Assert.AreEqual(new NumberToken(9876543210), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokenParsesOperatorsCorrectly() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\operators.txt", reporter); Assert.AreEqual(new PlusToken(), lexer.NextToken()); Assert.AreEqual(new MinusToken(), lexer.NextToken()); Assert.AreEqual(new MultiplyToken(), lexer.NextToken()); Assert.AreEqual(new DivideToken(), lexer.NextToken()); Assert.AreEqual(new AssignmentToken(), lexer.NextToken()); Assert.AreEqual(new LessThanToken(), lexer.NextToken()); Assert.AreEqual(new ComparisonToken(), lexer.NextToken()); Assert.AreEqual(new AndToken(), lexer.NextToken()); Assert.AreEqual(new NotToken(), lexer.NextToken()); Assert.AreEqual(new SemicolonToken(), lexer.NextToken()); Assert.AreEqual(new ColonToken(), lexer.NextToken()); Assert.AreEqual(new RangeToken(), lexer.NextToken()); Assert.AreEqual(new LParenToken(), lexer.NextToken()); Assert.AreEqual(new RParenToken(), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokenReturnsEOFOnEndOfFile() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\empty.txt", reporter); Assert.AreEqual(new EOFToken(), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokenParsesHandlesInvalidCharacters() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\invalid_characters.txt", reporter); Assert.AreEqual(new IdentifierToken("hello"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("there"), lexer.NextToken()); Assert.AreEqual(new TextToken("just"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("testing"), lexer.NextToken()); Assert.AreEqual(new NumberToken(1234), lexer.NextToken()); Assert.AreEqual(4, reporter.Errors.Count); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type); Assert.AreEqual(0, reporter.Errors[0].Line); Assert.AreEqual(0, reporter.Errors[0].Column); Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid start")); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type); Assert.AreEqual(0, reporter.Errors[1].Line); Assert.AreEqual(6, reporter.Errors[1].Column); Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("invalid start")); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[2].Type); Assert.AreEqual(0, reporter.Errors[2].Line); Assert.AreEqual(18, reporter.Errors[2].Column); Assert.IsTrue(reporter.Errors[2].Message.ToLower().Contains("invalid start")); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[3].Type); Assert.AreEqual(0, reporter.Errors[3].Line); Assert.AreEqual(26, reporter.Errors[3].Column); Assert.IsTrue(reporter.Errors[3].Message.ToLower().Contains("invalid start")); }
public void GetTokenParsesHandlesInvalidOperators() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\invalid_operators.txt", reporter); Assert.AreEqual(new IdentifierToken("this"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("contains"), lexer.NextToken()); Assert.AreEqual(new RangeToken(), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("invalid_operators"), lexer.NextToken()); Assert.AreEqual(2, reporter.Errors.Count); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type); Assert.AreEqual(0, reporter.Errors[0].Line); Assert.AreEqual(5, reporter.Errors[0].Column); Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid operator")); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type); Assert.AreEqual(0, reporter.Errors[1].Line); Assert.AreEqual(17, reporter.Errors[1].Column); Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("invalid operator")); }
public void GetTokenParsesCommentsCorrectly() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\comments.txt", reporter); Assert.AreEqual(new IdentifierToken("valid_token"), lexer.NextToken()); Assert.AreEqual(new DivideToken(), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("anothertoken"), lexer.NextToken()); Assert.AreEqual(new NumberToken(1234), lexer.NextToken()); Assert.AreEqual(new TextToken("test"), lexer.NextToken()); Assert.AreEqual(new EOFToken(), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokenHandlesValidStrings() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\valid_strings.txt", reporter); Assert.AreEqual(new TextToken("hello"), lexer.NextToken()); Assert.AreEqual(new TextToken("wor\tld"), lexer.NextToken()); Assert.AreEqual(new TextToken("thi\ns"), lexer.NextToken()); Assert.AreEqual(new TextToken("\"is\""), lexer.NextToken()); Assert.AreEqual(new TextToken("a\rtest"), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokenHandlesReservedWords() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\keywords.txt", reporter); Assert.AreEqual(new VarToken(), lexer.NextToken()); Assert.AreEqual(new ForToken(), lexer.NextToken()); Assert.AreEqual(new EndToken(), lexer.NextToken()); Assert.AreEqual(new InToken(), lexer.NextToken()); Assert.AreEqual(new DoToken(), lexer.NextToken()); Assert.AreEqual(new ReadToken(), lexer.NextToken()); Assert.AreEqual(new PrintToken(), lexer.NextToken()); Assert.AreEqual(new IntToken(), lexer.NextToken()); Assert.AreEqual(new StringToken(), lexer.NextToken()); Assert.AreEqual(new BoolToken(), lexer.NextToken()); Assert.AreEqual(new AssertToken(), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }
public void GetTokenHandlesInvalidStrings() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\invalid_strings.txt", reporter); Assert.AreEqual(new TextToken("invaliddescape"), lexer.NextToken()); Assert.AreEqual(new TextToken("unterminated"), lexer.NextToken()); Assert.AreEqual(new TextToken("valid for checking unterminated"), lexer.NextToken()); Assert.AreEqual(2, reporter.Errors.Count); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[0].Type); Assert.AreEqual(0, reporter.Errors[0].Line); Assert.AreEqual(9, reporter.Errors[0].Column); Assert.IsTrue(reporter.Errors[0].Message.ToLower().Contains("invalid escape sequence")); Assert.AreEqual(Error.LEXICAL_ERROR, reporter.Errors[1].Type); Assert.AreEqual(1, reporter.Errors[1].Line); Assert.AreEqual(13, reporter.Errors[1].Column); Assert.IsTrue(reporter.Errors[1].Message.ToLower().Contains("unmatched")); }
public void GetTokenHandlesIdentifiers() { var reporter = new ErrorReporter(); var lexer = new Lexer(@"..\..\Lexing\valid_identifiers.txt", reporter); Assert.AreEqual(new IdentifierToken("hello"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("w_or_ld"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("this_"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("i123s"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("a987"), lexer.NextToken()); Assert.AreEqual(new IdentifierToken("t_12_es_190t"), lexer.NextToken()); Assert.AreEqual(0, reporter.Errors.Count); }