public AbstractTokenizer(ITokenObserver <T> observer, IAllocationStrategy allocationStrategy,
ITokenEncoding encoding)
{
_observer = observer;
_allocationStrategy = allocationStrategy;
_encoding = encoding;
}
public SimpleTokenizer(ITokenObserver <TokenType> observer, IAllocationStrategy allocationStrategy,
ITokenEncoding encoding) : base(observer, allocationStrategy, encoding)
{
}
public SimpleTokenizer(ITokenObserver <TokenType> observer, ITokenEncoding encoding) : base(observer, encoding)
{
}
public AbstractTokenizer(ITokenObserver <T> observer, ITokenEncoding encoding) : this(observer,
new ArrayPoolAllocationStrategy(), encoding)
{
}
