public TokenizerResult Tokenize(ITracker tracker) { int count = 0; while (tracker.ReadWithOffset(count, out c) && range.Contains(c)) { count++; } tracker.AddToken(null, count); return(TokenizerResult.Success); }
public TokenizerResult Tokenize(ITracker tracker) { int count = 0; while (tracker.ReadWithOffset(count, out c) && range.Contains(c)) { if (c == '\n') // Newline is used as Seperator { break; } count++; } tracker.AddToken(null, count); return(TokenizerResult.Success); }