public static IEnumerable <ParsingToken> StreamToTokens(this IEnumerable <char[]> enumerable)
{
var enumerator = enumerable.SelectMany(x => x).GetEnumerator();
using (var seeker = new BufferedSeeker(enumerator))
{
ParsingToken currentToken = new MessageStream();
ParsingToken lastToken = null;
ParsingToken tempToken;
while ((tempToken = currentToken.Process(seeker, lastToken)) != null)
{
seeker.SeekToPeek();
lastToken = currentToken;
yield return(currentToken);
currentToken = tempToken;
}
}
}