public PhraseTokenizer(string text)
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
reader = new PhraseReader(text);
}
public PhraseTokenizer(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException($"Argument '{nameof(stream)}' cannot be read.", nameof(stream));
}
reader = new PhraseReader(stream);
}
internal void AddNext(string phrase, PhraseReader reader)
{
var token = reader.Read();
if (token == null)
{
this.Phrase = phrase;
return;
}
var nextPortion = (PhrasePortion)this.NextPortions[token.Portion];
if (nextPortion == null)
{
nextPortion = new PhrasePortion(token.Portion);
this.NextPortions[token.Portion] = nextPortion;
}
nextPortion.AddNext(phrase, reader);
}
public void AddPhrase(string phrase)
{
if (string.IsNullOrWhiteSpace(phrase))
{
return;
}
phrase = phrase.Trim();
var reader = new PhraseReader(phrase);
var token = reader.Read();
var portion = (PhrasePortion)dictionary[token.Portion];
if (portion == null)
{
portion = new PhrasePortion(token.Portion);
dictionary[token.Portion] = portion;
}
portion.AddNext(phrase, reader);
}
