Example #1
0
 public static List<Token> AnalyzeString(Language language, string value)
 {
     List<Token> toks = new List<Token>();
       Tokenizer tkz = new Tokenizer(language);
       tkz.SetText(value);
       for (; ; ) {
     Token tk = tkz.ReadToken();
     if (tk == null)
       break;
     if (tk.Type == (int)TokenType.Comments ||
     tk.Type == (int)TokenType.DocComments)
       continue;
     toks.Add(tk);
       }
       return toks;
 }
Example #2
0
File: Program.cs Project: AxFab/amy
        static void ExtractTokenFromFile(string path)
        {
            try {
            Tokenizer fr = new Tokenizer(path, Language.CLanguage());
            while (true) {
              Token t = fr.ReadToken();
              if (t == null)
            break;
              TokenType type = (TokenType)t.Type;
              Console.WriteLine("- {0} '{1}'", type, t.Litteral);
            }

              } catch (Exception e) {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.Error.WriteLine(e);
            Console.ForegroundColor = ConsoleColor.Gray;
              }
        }