public static IEnumerable <string> GetTokens(this string value)
{
StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(value));
return(new TextStreamParser(tokenizer));
}
internal TextStreamParser(StreamTokenizer tokenizer)
{
this.tokenizer = tokenizer;
}
public static IEnumerable <string> GetTokens(this StreamReader reader)
{
StreamTokenizer tokenizer = new StreamTokenizer(reader);
return(new TextStreamParser(tokenizer));
}