public void SimpleTest() { var text = "var a = 20"; var source = new Shovel.SourceFile () { FileName = "test.sho", Content = text }; var tokenizer = new Shovel.Compiler.Tokenizer (source); // Expect 5 tokens because a 'filename' token is inserted. Assert.AreEqual (5, tokenizer.Tokens.Count); var sb = new StringBuilder (); sb.AppendLine (); foreach (var token in tokenizer.Tokens) { sb.AppendFormat ( "[Token: Type={0}, StartPos={1}, EndPos={2}, Content='{3}']\n", token.Type, token.StartPos, token.EndPos, token.Content); } Assert.AreEqual (@" [Token: Type=FileName, StartPos=0, EndPos=0, Content='test.sho'] [Token: Type=Keyword, StartPos=0, EndPos=2, Content='var'] [Token: Type=Identifier, StartPos=4, EndPos=4, Content='a'] [Token: Type=Punctuation, StartPos=6, EndPos=6, Content='='] [Token: Type=Number, StartPos=8, EndPos=9, Content='20'] ", sb.ToString().TrimCarriageReturn()); }
public void SimpleTest() { var text = "var a = 20"; var source = new Shovel.SourceFile() { FileName = "test.sho", Content = text }; var tokenizer = new Shovel.Compiler.Tokenizer(source); // Expect 5 tokens because a 'filename' token is inserted. Assert.AreEqual(5, tokenizer.Tokens.Count); var sb = new StringBuilder(); sb.AppendLine(); foreach (var token in tokenizer.Tokens) { sb.AppendFormat( "[Token: Type={0}, StartPos={1}, EndPos={2}, Content='{3}']\n", token.Type, token.StartPos, token.EndPos, token.Content); } Assert.AreEqual(@" [Token: Type=FileName, StartPos=0, EndPos=0, Content='test.sho'] [Token: Type=Keyword, StartPos=0, EndPos=2, Content='var'] [Token: Type=Identifier, StartPos=4, EndPos=4, Content='a'] [Token: Type=Punctuation, StartPos=6, EndPos=6, Content='='] [Token: Type=Number, StartPos=8, EndPos=9, Content='20'] ", sb.ToString().TrimCarriageReturn()); }
public void TestUnfinishedLiteralString() { var text = "var a = '20"; var source = new Shovel.SourceFile () { FileName = "test.sho", Content = text }; Utils.ExpectException<ShovelException> (() => { Assert.IsNotNull (new Shovel.Compiler.Tokenizer (source).Tokens); }, (ex) => { Assert.IsNotNull (ex); Assert.AreEqual ("Expected an end quote, but reached the end of file.", ex.Message); Assert.AreEqual ("test.sho", ex.FileName); Assert.AreEqual (true, ex.AtEof); } ); }
public void TestUnfinishedLiteralString() { var text = "var a = '20"; var source = new Shovel.SourceFile() { FileName = "test.sho", Content = text }; Utils.ExpectException <ShovelException> (() => { Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens); }, (ex) => { Assert.IsNotNull(ex); Assert.AreEqual("Expected an end quote, but reached the end of file.", ex.Message); Assert.AreEqual("test.sho", ex.FileName); Assert.AreEqual(true, ex.AtEof); } ); }
public void TestUnfinishedMultilineComment() { var text = "var a = 10 /* this is a comment"; var source = new Shovel.SourceFile() { FileName = "test.sho", Content = text }; Utils.ExpectException <ShovelException>(() => { Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens); }, (ex) => { Assert.IsNotNull(ex); Assert.AreEqual("Reached the end of the file while parsing a multiline comment.", ex.Message); Assert.AreEqual("test.sho", ex.FileName); Assert.AreEqual(true, ex.AtEof); } ); }
public void TestUnknownCharacter() { var text = "var a = `"; var source = new Shovel.SourceFile() { FileName = "test.sho", Content = text }; Utils.ExpectException <ShovelException> (() => { Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens); }, (ex) => { Assert.IsNotNull(ex); Assert.AreEqual(@"Unexpected character '`'. file 'test.sho' line 1: var a = ` file 'test.sho' line 1: ^", ex.Message); Assert.AreEqual("test.sho", ex.FileName); Assert.AreEqual(false, ex.AtEof); Assert.AreEqual(1, ex.Line); Assert.AreEqual(9, ex.Column); } ); }
public void TestUnfinishedMultilineComment() { var text = "var a = 10 /* this is a comment"; var source = new Shovel.SourceFile() { FileName = "test.sho", Content = text }; Utils.ExpectException<ShovelException>(() => { Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens); }, (ex) => { Assert.IsNotNull(ex); Assert.AreEqual("Reached the end of the file while parsing a multiline comment.", ex.Message); Assert.AreEqual("test.sho", ex.FileName); Assert.AreEqual(true, ex.AtEof); } ); }
public void TestUnknownCharacter() { var text = "var a = `"; var source = new Shovel.SourceFile () { FileName = "test.sho", Content = text }; Utils.ExpectException<ShovelException> (() => { Assert.IsNotNull (new Shovel.Compiler.Tokenizer (source).Tokens); }, (ex) => { Assert.IsNotNull (ex); Assert.AreEqual (@"Unexpected character '`'. file 'test.sho' line 1: var a = ` file 'test.sho' line 1: ^", ex.Message); Assert.AreEqual ("test.sho", ex.FileName); Assert.AreEqual (false, ex.AtEof); Assert.AreEqual (1, ex.Line); Assert.AreEqual (9, ex.Column); } ); }