C# (CSharp) Amy.Lexer Tokenizer.SetText Examples

Example #1

File: Tokenizer.cs Project: AxFab/amy

 public static List<Token> AnalyzeString(Language language, string value)
 {
     List<Token> toks = new List<Token>();
       Tokenizer tkz = new Tokenizer(language);
       tkz.SetText(value);
       for (; ; ) {
     Token tk = tkz.ReadToken();
     if (tk == null)
       break;
     if (tk.Type == (int)TokenType.Comments ||
     tk.Type == (int)TokenType.DocComments)
       continue;
     toks.Add(tk);
       }
       return toks;
 }