public AbstractTokenizer(ITokenObserver <T> observer, IAllocationStrategy allocationStrategy, ITokenEncoding encoding) { _observer = observer; _allocationStrategy = allocationStrategy; _encoding = encoding; }
public SimpleTokenizer(ITokenObserver <TokenType> observer, IAllocationStrategy allocationStrategy, ITokenEncoding encoding) : base(observer, allocationStrategy, encoding) { }
public SimpleTokenizer(ITokenObserver <TokenType> observer, ITokenEncoding encoding) : base(observer, encoding) { }
public AbstractTokenizer(ITokenObserver <T> observer, ITokenEncoding encoding) : this(observer, new ArrayPoolAllocationStrategy(), encoding) { }