/// <summary>
/// Trains a model for the <see cref="TokenizerME"/>.
/// </summary>
/// <param name="samples">The samples used for the training.</param>
/// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
/// <param name="parameters">The machine learning train parameters.</param>
/// <param name="monitor">
/// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation.
/// This argument can be a <c>null</c> value.
/// </param>
/// <returns>The trained <see cref="TokenizerModel"/>.</returns>
public static TokenizerModel Train(IObjectStream <TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters, Monitor monitor)
{
var manifestInfoEntries = new Dictionary <string, string>();
var eventStream = new TokSpanEventStream(samples, factory.UseAlphaNumericOptimization,
factory.AlphaNumericPattern, factory.ContextGenerator);
var trainer = TrainerFactory.GetEventTrainer(parameters, manifestInfoEntries, monitor);
var model = trainer.Train(eventStream);
return(new TokenizerModel(model, manifestInfoEntries, factory));
}
/// <summary>
/// Trains a model for the <see cref="TokenizerME"/>.
/// </summary>
/// <param name="samples">The samples used for the training.</param>
/// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
/// <param name="parameters">The machine learning train parameters.</param>
/// <returns>The trained <see cref="TokenizerModel"/>.</returns>
public static TokenizerModel Train(IObjectStream <TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters)
{
return(Train(samples, factory, parameters, null));
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenizerModel"/> class.
/// </summary>
/// <param name="tokenizerModel">The tokenizer model.</param>
/// <param name="manifestInfoEntries">The manifest information entries.</param>
/// <param name="tokenizerFactory">The tokenizer factory.</param>
public TokenizerModel(IMaxentModel tokenizerModel, Dictionary <string, string> manifestInfoEntries, TokenizerFactory tokenizerFactory)
: base(ComponentName, tokenizerFactory.LanguageCode, manifestInfoEntries, tokenizerFactory)
{
artifactMap.Add(TokenizerModelEntry, tokenizerModel);
CheckArtifactMap();
}
/// <summary>
/// Trains a model for the <see cref="TokenizerME"/>.
/// </summary>
/// <param name="samples">The samples used for the training.</param>
/// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
/// <param name="parameters">The machine learning train parameters.</param>
/// <param name="monitor">
/// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation.
/// This argument can be a <c>null</c> value.
/// </param>
/// <returns>The trained <see cref="TokenizerModel"/>.</returns>
public static TokenizerModel Train(IObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters, Monitor monitor) {
var manifestInfoEntries = new Dictionary<string, string>();
var eventStream = new TokSpanEventStream(samples, factory.UseAlphaNumericOptimization,
factory.AlphaNumericPattern, factory.ContextGenerator);
var trainer = TrainerFactory.GetEventTrainer(parameters, manifestInfoEntries, monitor);
var model = trainer.Train(eventStream);
return new TokenizerModel(model, manifestInfoEntries, factory);
}
/// <summary>
/// Trains a model for the <see cref="TokenizerME"/>.
/// </summary>
/// <param name="samples">The samples used for the training.</param>
/// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
/// <param name="parameters">The machine learning train parameters.</param>
/// <returns>The trained <see cref="TokenizerModel"/>.</returns>
public static TokenizerModel Train(IObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters) {
return Train(samples, factory, parameters, null);
}
private static TokenizerModel Train(TokenizerFactory factory) {
return TokenizerME.Train(CreateSampleStream(), factory, TrainingParameters.DefaultParameters());
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenizerModel"/> class.
/// </summary>
/// <param name="tokenizerModel">The tokenizer model.</param>
/// <param name="manifestInfoEntries">The manifest information entries.</param>
/// <param name="tokenizerFactory">The tokenizer factory.</param>
public TokenizerModel(IMaxentModel tokenizerModel, Dictionary<string, string> manifestInfoEntries, TokenizerFactory tokenizerFactory)
: base(ComponentName, tokenizerFactory.LanguageCode, manifestInfoEntries, tokenizerFactory) {
artifactMap.Add(TokenizerModelEntry, tokenizerModel);
CheckArtifactMap();
}
