Tokenize() public method

public Tokenize ( String script ) : IList
script String
return IList
Tokenizer Class Documentation
Beispiel #1
0
 public IList<Token> tokenize(string script)
 {
     var tokenizer = new Tokenizer(ctx, this);
     return tokenizer.Tokenize(script);
 }
Beispiel #2
0
 // token.h
 // static void parseSyntaxRule(CTX, const char *rule, kline_t pline, kArray *a);
 public void parseSyntaxRule(string rule, LineInfo pline, out List<Token> adst)
 {
     var tokenizer = new Tokenizer(ctx, this);
     var tokens = tokenizer.Tokenize(rule);
     makeSyntaxRule(tokens, 0, tokens.Count, out adst);
 }