public virtual IEnumerable<Token> Tokenize(params string[] args)
{
TokenWriter tokenWriter = new TokenWriter();
Task.Factory.StartNew(() =>
{
try
{
using (StringReader reader = new StringReader(string.Join(" ", args)))
using (BasicReader basicReader = new BasicReader(reader, StringComparer.Ordinal))
{
this.Parse(tokenWriter, basicReader);
tokenWriter.Close();
}
}
catch (Exception ex)
{
tokenWriter.Abort(ex);
}
});
return tokenWriter.GetConsumingEnumerable();
}