/// <summary>
/// Returns the next object. Calling this method repeatedly until it returns,
/// null will return each object from the underlying source exactly once.
/// </summary>
/// <returns>
/// The next object or null to signal that the stream is exhausted.
/// </returns>
public TokenSample Read()
{
AdSentence paragraph;
if (monitor != null)
{
monitor.Token.ThrowIfCancellationRequested();
}
while ((paragraph = adSentenceStream.Read()) != null)
{
var root = paragraph.Root;
var sentence = new List <string>();
Process(root, sentence);
return(new TokenSample(detokenizer, sentence.ToArray()));
}
return(null);
}
