public void TestFloatTokenizer() { var test = @"1.01"; var tokens = new Lexers.Lexer(test).Lex().ToList(); Assert.IsTrue(tokens.Count == 1); Assert.IsTrue(tokens.First().TokenType == TokenType.Float); }
public LanguageParser(Lexers.Lexer lexer) { TokenStream = new ParseableTokenStream(lexer); // we'll tag all lambdas we find starting from 1000 here // later when we iterate over scope and create anonnymous lambdas // we need the lambdas to have the SAME name even if we iterate over // the syntax tree multiple times. this is hacky, i know. // curried functions will be labeled from 0 to 1000 LambdaDeclr.LambdaCount = 1000; }
public void TestTokenizer() { var test = @"function void int ""void int"" {} ->*/test^void,5,6,7 8.0"; var tokens = new Lexers.Lexer(test).Lex().ToList(); foreach (var token in tokens) { Console.WriteLine(token.TokenType + " - " + token.TokenValue); } }
public void TestTokenizer1() { var test = @"fun function = 1 print"; var tokens = new Lexers.Lexer(test).Lex().ToList(); }