public void BadTokenPosition() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new IntegerLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); try { tokenizer.Tokenize("5 A"); } catch (UnknownTokenException ex) { Assert.AreEqual(3, ex.Position.Column); Assert.AreEqual(1, ex.Position.Line); Assert.AreEqual("A", ex.Token); } try { tokenizer.Tokenize("5 4\r\n2\r\n X\r\n5"); } catch (UnknownTokenException ex) { Assert.AreEqual(4, ex.Position.Column); Assert.AreEqual(3, ex.Position.Line); Assert.AreEqual("X", ex.Token); } }
public void NumericLiterals() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new IntegerLiteralMatcher()); tokenizer.AddTokenMatcher(new DecimalLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); Token[] tokens; tokens = tokenizer.Tokenize("10 10.0"); Assert.AreEqual(3, tokens.Length); Assert.AreEqual("10", tokens[0].Text); Assert.AreEqual("10.0", tokens[2].Text); tokens = tokenizer.Tokenize("10m 10ul"); Assert.AreEqual(3, tokens.Length); Assert.AreEqual("10m", tokens[0].Text); Assert.AreEqual("10ul", tokens[2].Text); tokens = tokenizer.Tokenize("10f 10l"); Assert.AreEqual(3, tokens.Length); Assert.AreEqual("10f", tokens[0].Text); Assert.AreEqual("10l", tokens[2].Text); }
public void NumericLiterals() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new IntegerLiteralMatcher()); tokenizer.AddTokenMatcher(new DecimalLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); Token[] tokens; tokens = tokenizer.Tokenize("10 10.0"); Assert.AreEqual(3,tokens.Length); Assert.AreEqual("10",tokens[0].Text); Assert.AreEqual("10.0",tokens[2].Text); tokens = tokenizer.Tokenize("10m 10ul"); Assert.AreEqual(3, tokens.Length); Assert.AreEqual("10m", tokens[0].Text); Assert.AreEqual("10ul", tokens[2].Text); tokens = tokenizer.Tokenize("10f 10l"); Assert.AreEqual(3, tokens.Length); Assert.AreEqual("10f", tokens[0].Text); Assert.AreEqual("10l", tokens[2].Text); }
public void BadTokenPosition() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new IntegerLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); try { tokenizer.Tokenize("5 A"); } catch (UnknownTokenException ex) { Assert.AreEqual(3, ex.Position.Column); Assert.AreEqual(1, ex.Position.Line); Assert.AreEqual("A", ex.Token); } try { tokenizer.Tokenize("5 4\r\n2\r\n X\r\n5"); } catch (UnknownTokenException ex) { Assert.AreEqual(4, ex.Position.Column); Assert.AreEqual(3, ex.Position.Line); Assert.AreEqual("X",ex.Token); } }
public void BadToken() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new IntegerLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); tokenizer.Tokenize("5 A"); }
public void StartsAndEndsWithToken() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new StartsAndEndsWithMatcher("<!--", "-->", '"')); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); Token[] tokens; tokens = tokenizer.Tokenize("<!--test--> <!-- test 2 -->"); Assert.AreEqual(3, tokens.Length); tokens = tokenizer.Tokenize("<!--test \"-->\"--> <!-- test 2 -->"); Assert.AreEqual(3, tokens.Length); }
//[ExpectedException(typeof(UnknownTokenException))] public void BadToken() { try { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new IntegerLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); tokenizer.Tokenize("5 A"); Assert.Fail(); } catch (UnknownTokenException ex) { } }
public void BadToken() { try { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new IntegerLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); tokenizer.Tokenize("5 A"); Assert.Fail(); } catch(UnknownTokenException ex) { } }
public void TestFallback() { ITokenMatcher matcher1 = new CharMatcher('('); ITokenMatcher matcher2 = new CharMatcher(')'); ITokenMatcher matcher3 = new StringMatcher("(test)"); ITokenMatcher matcher4 = new AnyCharMatcher("abcdefghijklmnopqrstuvwxyz"); Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(matcher1); tokenizer.AddTokenMatcher(matcher2); tokenizer.AddTokenMatcher(matcher3); tokenizer.AddTokenMatcher(matcher4); Token[] tokens = tokenizer.Tokenize("(test)(x)"); Assert.AreEqual(4, tokens.Length); }
public void TestStringLiteral() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new StringLiteralMatcher()); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); tokenizer.AddTokenMatcher(new CharMatcher('+')); Token[] tokens = tokenizer.Tokenize("\"test1\" + \"test2\""); Assert.AreEqual(5, tokens.Length); Assert.AreEqual("\"test1\"", tokens[0].Text); Assert.AreEqual("\"test2\"", tokens[4].Text); tokens = tokenizer.Tokenize("\"test1\" + \"test\\\"2\""); Assert.AreEqual(5, tokens.Length); Assert.AreEqual("\"test1\"", tokens[0].Text); Assert.AreEqual("\"test\\\"2\"", tokens[4].Text); }
public void TestAnySequence() { ITokenMatcher matcher = new CompositeMatcher( new AnyCharMatcher("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"), new SequenceOfAnyCharMatcher("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789") ); ITokenMatcher alphaMatcher = new SequenceOfAnyCharMatcher("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789"); Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(matcher); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); tokenizer.AddTokenMatcher(new CharMatcher('.')); Token[] tokens = tokenizer.Tokenize("Math.Max"); Assert.AreEqual(3, tokens.Length); Assert.AreEqual("Math", tokens[0].Text); Assert.AreEqual(".", tokens[1].Text); Assert.AreEqual("Max", tokens[2].Text); }
public void TestFallback() { ITokenMatcher matcher1 = new CharMatcher('('); ITokenMatcher matcher2 = new CharMatcher(')'); ITokenMatcher matcher3 = new StringMatcher("(test)"); ITokenMatcher matcher4 = new AnyCharMatcher("abcdefghijklmnopqrstuvwxyz"); Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(matcher1); tokenizer.AddTokenMatcher(matcher2); tokenizer.AddTokenMatcher(matcher3); tokenizer.AddTokenMatcher(matcher4); Token[] tokens = tokenizer.Tokenize("(test)(x)"); Assert.AreEqual(4,tokens.Length); }
public void StartsAndEndsWithToken() { Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(new StartsAndEndsWithMatcher("<!--","-->",'"')); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); Token[] tokens; tokens = tokenizer.Tokenize("<!--test--> <!-- test 2 -->"); Assert.AreEqual(3,tokens.Length); tokens = tokenizer.Tokenize("<!--test \"-->\"--> <!-- test 2 -->"); Assert.AreEqual(3, tokens.Length); }
public void TestAnySequence() { ITokenMatcher matcher = new CompositeMatcher( new AnyCharMatcher("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"), new SequenceOfAnyCharMatcher("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789") ); ITokenMatcher alphaMatcher = new SequenceOfAnyCharMatcher("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789"); Tokenizer tokenizer = new Tokenizer(); tokenizer.AddTokenMatcher(matcher); tokenizer.AddTokenMatcher(new WhiteSpaceMatcher()); tokenizer.AddTokenMatcher(new CharMatcher('.')); Token[] tokens = tokenizer.Tokenize("Math.Max"); Assert.AreEqual(3,tokens.Length); Assert.AreEqual("Math",tokens[0].Text); Assert.AreEqual(".", tokens[1].Text); Assert.AreEqual("Max", tokens[2].Text); }