public static List<Token> AnalyzeString(Language language, string value)
{
List<Token> toks = new List<Token>();
Tokenizer tkz = new Tokenizer(language);
tkz.SetText(value);
for (; ; ) {
Token tk = tkz.ReadToken();
if (tk == null)
break;
if (tk.Type == (int)TokenType.Comments ||
tk.Type == (int)TokenType.DocComments)
continue;
toks.Add(tk);
}
return toks;
}
static void ExtractTokenFromFile(string path)
{
try {
Tokenizer fr = new Tokenizer(path, Language.CLanguage());
while (true) {
Token t = fr.ReadToken();
if (t == null)
break;
TokenType type = (TokenType)t.Type;
Console.WriteLine("- {0} '{1}'", type, t.Litteral);
}
} catch (Exception e) {
Console.ForegroundColor = ConsoleColor.Red;
Console.Error.WriteLine(e);
Console.ForegroundColor = ConsoleColor.Gray;
}
}