public void BadEol() { string text = "alpha epsi\n\rfoo"; var scanner = new Scanner(text); while (scanner.Token.IsValid()) { scanner.Advance(); } }
public void Basics() { string text = @"define Run() if Scope is Type then Process(scope) "; var scanner = new Scanner(text); var tokens = new List<Token>(); while (scanner.Token.IsValid()) { tokens.Add(scanner.Token); scanner.Advance(); } Assert.AreEqual("define", tokens[0].Text()); Assert.AreEqual("Run", tokens[1].Text()); Assert.AreEqual("(", tokens[2].Text()); Assert.AreEqual(")", tokens[3].Text()); Assert.AreEqual("if", tokens[4].Text()); Assert.AreEqual("Scope", tokens[5].Text()); Assert.AreEqual("is", tokens[6].Text()); Assert.AreEqual("Type", tokens[7].Text()); Assert.AreEqual("then", tokens[8].Text()); Assert.AreEqual("Process", tokens[9].Text()); Assert.AreEqual("(", tokens[10].Text()); Assert.AreEqual("scope", tokens[11].Text()); Assert.AreEqual(")", tokens[12].Text()); Assert.AreEqual(13, tokens.Count); Assert.AreEqual(TokenKind.Keyword, tokens[0].Kind); Assert.AreEqual(TokenKind.Identifier, tokens[1].Kind); Assert.AreEqual(TokenKind.Punct, tokens[2].Kind); Assert.AreEqual(TokenKind.Punct, tokens[3].Kind); Assert.AreEqual(1, tokens[0].Line); Assert.AreEqual(1, tokens[2].Line); Assert.AreEqual(1, tokens[3].Line); Assert.AreEqual(2, tokens[4].Line); Assert.AreEqual(2, tokens[5].Line); Assert.AreEqual(2, tokens[6].Line); Assert.AreEqual(2, tokens[7].Line); Assert.AreEqual(2, tokens[8].Line); Assert.AreEqual(3, tokens[9].Line); Assert.AreEqual(3, tokens[10].Line); Assert.AreEqual(3, tokens[11].Line); Assert.AreEqual(3, tokens[12].Line); }
public void Identifier() { string text = @"alpha _beta"; var scanner = new Scanner(text); Assert.AreEqual("alpha", scanner.Token.Text()); Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind); scanner.Advance(); Assert.AreEqual("_beta", scanner.Token.Text()); Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind); scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind); }
public void String() { string text = @"foo ""some text goes here"" yyy"; var scanner = new Scanner(text); Assert.AreEqual("foo", scanner.Token.Text()); scanner.Advance(); Assert.AreEqual("\"some\ntext\n goes here\"", scanner.Token.Text()); Assert.AreEqual(1, scanner.Token.Line); scanner.Advance(); Assert.AreEqual("yyy", scanner.Token.Text()); Assert.AreEqual(3, scanner.Token.Line); scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind); text = "aa \"bb\"\"cc\" dd"; scanner = new Scanner(text); Assert.AreEqual("aa", scanner.Token.Text()); scanner.Advance(); Assert.AreEqual("\"bb\"\"cc\"", scanner.Token.Text()); scanner.Advance(); Assert.AreEqual("dd", scanner.Token.Text()); }
public void SingleLineComment() { string text = @"alpha # text goes here some more # foo"; var scanner = new Scanner(text); Assert.AreEqual("alpha", scanner.Token.Text()); scanner.Advance(); Assert.AreEqual("some", scanner.Token.Text()); scanner.Advance(); Assert.AreEqual("more", scanner.Token.Text()); scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind); }