public void Test5() { var text = "{}"; var split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Comment[0,2]", viewSplit(split)); text = "(*comment 1*)"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Comment[0,13]", viewSplit(split)); text = "{comment 1\ncomment 2}"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Comment[0,21]", viewSplit(split)); text = "//comment 1"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Comment[0,11]", viewSplit(split)); text = "//comment 1\ncode 2"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Comment[0,11]Symbol[11,12]Identifier[12,16]Symbol[16,17]Number[17,18]", viewSplit(split)); text = "//comment 1 (*comment 2*)"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Comment[0,25]:Comment[12,25];", viewSplit(split)); text = "//comment 1 //comment"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Comment[0,21]:Comment[12,21];", viewSplit(split)); }
public void Test4() { // zero or more, no \n or \r quotes or control stings var text = "''"; var split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("CharacterString[0,2]", viewSplit(split)); text = "'hello world!'"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("CharacterString[0,14]", viewSplit(split)); text = "'hello world!\n'"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Symbol[0,1]Identifier[1,6]Symbol[6,7]Identifier[7,12]Symbol[12,13]Symbol[13,14]Symbol[14,15]", viewSplit(split)); text = "#34#56"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("CharacterString[0,6]", viewSplit(split)); text = "'hello'#34#56'world'"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("CharacterString[0,20]", viewSplit(split)); text = "'hello'#34#56"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("CharacterString[0,13]", viewSplit(split)); }
public void Test2() { // Identifier: starts with letter, _ or &, includes letters and digits, length from 1 to 127 var text = "a"; var split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Identifier[0,1]", viewSplit(split)); text = "a__34"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Identifier[0,5]", viewSplit(split)); text = "_____"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Identifier[0,5]", viewSplit(split)); text = "&a__34"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Identifier[0,6]", viewSplit(split)); text = "&a_=_34"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Identifier[0,3]Symbol[3,4]Identifier[4,7]", viewSplit(split)); text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; Console.Write(text.Length); split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Identifier[0,127]Identifier[127,128]", viewSplit(split)); }
public void Test3() { // Numbers: sign (or no) unsigned real (digits . digits eE sign digits) or integer (dec, bin, hex, oct) // integer var text = "2"; var split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Number[0,1]", viewSplit(split)); text = "$267af"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Number[0,6]", viewSplit(split)); text = "-$267afk"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Number[0,7]Identifier[7,8]", viewSplit(split)); text = "&267af"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Number[0,4]Identifier[4,6]", viewSplit(split)); text = "%1001010"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Number[0,8]", viewSplit(split)); text = "267.01e8"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Number[0,8]", viewSplit(split)); text = "+267.01E+8"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Number[0,10]", viewSplit(split)); }
public void Test1() { // ’ + - * / = < > [ ] . , ( ) : ^ @ { } $ # & % << >> ** <> >< <= >= := += -= *= /= (* *) (. .) // var text = "*"; var split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Symbol[0,1]", viewSplit(split)); text = "(*"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Symbol[0,1]Symbol[1,2]", viewSplit(split)); text = "(*'"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Symbol[0,1]Symbol[1,2]Symbol[2,3]", viewSplit(split)); text = "&(*'"; split = SimpleLexer.SplitToLexeme(text); Assert.AreEqual("Symbol[0,1]Symbol[1,2]Symbol[2,3]Symbol[3,4]", viewSplit(split)); }