public TokenizedText(GraphemeString graphemeString, TokenRange tokenRange)
{
GraphemeString = graphemeString;
TokenRange = tokenRange;
for (TokenNode start = tokenRange.Start, node = tokenRange.Start; ;)
{
if (node.Value.Type == TokenType.FullStop || node.Value.Type == TokenType.QuestionMark || node.Value.Type == TokenType.ExclamationPoint)
{
for (node = node.Next; node.Value.Type == TokenType.FullStop; node = node.Next)
{
;
}
Sentences.Add(new TokenizedSentence(new TokenRange(start, node)));
start = node;
continue;
}
if (node.Value.Type == TokenType.Eof)
{
if (start != node)
{
Sentences.Add(new TokenizedSentence(new TokenRange(start, node)));
}
break;
}
node = node.Next;
}
}
public TokenizedSentence(TokenRange tokenRange)
{
TokenRange = tokenRange;
}
