public TokSpanEventStream(IObjectStream <TokenSample> samples, bool skipAlphaNumerics, ITokenContextGenerator cg)
: base(samples)
{
alphaNumeric = new Regex(Factory.GetAlphanumeric(null), RegexOptions.Compiled);
this.skipAlphaNumerics = skipAlphaNumerics;
this.cg = cg;
}
/// <summary>
/// Initializes a new instance of the <see cref="TokSpanEventStream"/> class.
/// </summary>
/// <param name="samples">The samples.</param>
/// <param name="skipAlphaNumerics">if set to <c>true</c> [skip alpha numerics].</param>
/// <param name="alphaNumericPattern">The alpha numeric pattern.</param>
/// <param name="cg">The token context generator.</param>
public TokSpanEventStream(IObjectStream <TokenSample> samples, bool skipAlphaNumerics, string alphaNumericPattern,
ITokenContextGenerator cg)
: base(samples)
{
alphaNumeric = new Regex(alphaNumericPattern, RegexOptions.Compiled);
this.skipAlphaNumerics = skipAlphaNumerics;
this.cg = cg;
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenizerME"/> class.
/// </summary>
/// <param name="model">The tokenizer model.</param>
public TokenizerME(TokenizerModel model) {
this.model = model.MaxentModel;
var factory = model.Factory;
alphanumeric = new Regex(factory.AlphaNumericPattern, RegexOptions.Compiled);
cg = factory.ContextGenerator;
useAlphaNumericOptimization = model.UseAlphaNumericOptimization;
newTokens = new List<Span>();
tokProbs = new List<double>(50);
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenizerME"/> class.
/// </summary>
/// <param name="model">The tokenizer model.</param>
public TokenizerME(TokenizerModel model)
{
this.model = model.MaxentModel;
var factory = model.Factory;
alphanumeric = new Regex(factory.AlphaNumericPattern, RegexOptions.Compiled);
cg = factory.ContextGenerator;
useAlphaNumericOptimization = model.UseAlphaNumericOptimization;
newTokens = new List <Span>();
tokProbs = new List <double>(50);
}
