public IEnumerable <Token> Traverse(string text)
{
if (string.IsNullOrEmpty(text))
{
yield break;
}
using (var reader = new RewindStringReader(text, this.OptionOutputOriginalCase))
{
var breaker = new MaximumMatchTokenBreaker(_dawg, reader);
var token = breaker.Next();
do
{
yield return(token);
} while ((token = breaker.Next()) != null);
}
}
public IEnumerable<Token> Traverse(string text)
{
if (string.IsNullOrEmpty(text))
{
yield break;
}
using (var reader = new RewindStringReader(text, this.OptionOutputOriginalCase))
{
var breaker = new MaximumMatchTokenBreaker(_dawg, reader);
var token = breaker.Next();
do
{
yield return token;
} while ((token = breaker.Next()) != null);
}
}
public MaximumMatchTokenBreaker(Dawg dawg, RewindStringReader reader)
: base(reader)
{
_reader = reader;
_dawg = dawg;
}
public MaximumMatchTokenBreaker(Dawg dawg, RewindStringReader reader)
: base(reader)
{
_reader = reader;
_dawg = dawg;
}
public BigramTokenBreaker(RewindStringReader reader)
: base(reader)
{
_reader = reader;
}
public BigramTokenBreaker(RewindStringReader reader)
: base(reader)
{
_reader = reader;
}
public WhiteSpaceTokenBreaker(RewindStringReader stringReader)
{
_reader = stringReader;
}
public WhiteSpaceTokenBreaker(RewindStringReader stringReader)
{
_reader = stringReader;
}
