C# (CSharp) Joke.Front.Pony.Lexing PonyTokens示例

示例#1

        public PonyTokenizer(Errors errors, ISource source)
        {
            Errors = errors;
            Source = source;
            Tokens = new PonyTokens(Source, new PonyToken[] { });

            content = Source.Content;
            limit   = content.Length;
            next    = 0;
            clutter = 0;
            payload = 0;
        }

示例#2

 public void Tokenize()
 {
     Tokens = new PonyTokens(Source, GetTokens());
 }