/// <summary> /// Trains a model for the <see cref="TokenizerME"/>. /// </summary> /// <param name="samples">The samples used for the training.</param> /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param> /// <param name="parameters">The machine learning train parameters.</param> /// <param name="monitor"> /// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation. /// This argument can be a <c>null</c> value. /// </param> /// <returns>The trained <see cref="TokenizerModel"/>.</returns> public static TokenizerModel Train(IObjectStream <TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters, Monitor monitor) { var manifestInfoEntries = new Dictionary <string, string>(); var eventStream = new TokSpanEventStream(samples, factory.UseAlphaNumericOptimization, factory.AlphaNumericPattern, factory.ContextGenerator); var trainer = TrainerFactory.GetEventTrainer(parameters, manifestInfoEntries, monitor); var model = trainer.Train(eventStream); return(new TokenizerModel(model, manifestInfoEntries, factory)); }
/// <summary> /// Trains a model for the <see cref="TokenizerME"/>. /// </summary> /// <param name="samples">The samples used for the training.</param> /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param> /// <param name="parameters">The machine learning train parameters.</param> /// <returns>The trained <see cref="TokenizerModel"/>.</returns> public static TokenizerModel Train(IObjectStream <TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters) { return(Train(samples, factory, parameters, null)); }
/// <summary> /// Initializes a new instance of the <see cref="TokenizerModel"/> class. /// </summary> /// <param name="tokenizerModel">The tokenizer model.</param> /// <param name="manifestInfoEntries">The manifest information entries.</param> /// <param name="tokenizerFactory">The tokenizer factory.</param> public TokenizerModel(IMaxentModel tokenizerModel, Dictionary <string, string> manifestInfoEntries, TokenizerFactory tokenizerFactory) : base(ComponentName, tokenizerFactory.LanguageCode, manifestInfoEntries, tokenizerFactory) { artifactMap.Add(TokenizerModelEntry, tokenizerModel); CheckArtifactMap(); }
/// <summary> /// Trains a model for the <see cref="TokenizerME"/>. /// </summary> /// <param name="samples">The samples used for the training.</param> /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param> /// <param name="parameters">The machine learning train parameters.</param> /// <param name="monitor"> /// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation. /// This argument can be a <c>null</c> value. /// </param> /// <returns>The trained <see cref="TokenizerModel"/>.</returns> public static TokenizerModel Train(IObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters, Monitor monitor) { var manifestInfoEntries = new Dictionary<string, string>(); var eventStream = new TokSpanEventStream(samples, factory.UseAlphaNumericOptimization, factory.AlphaNumericPattern, factory.ContextGenerator); var trainer = TrainerFactory.GetEventTrainer(parameters, manifestInfoEntries, monitor); var model = trainer.Train(eventStream); return new TokenizerModel(model, manifestInfoEntries, factory); }
/// <summary> /// Trains a model for the <see cref="TokenizerME"/>. /// </summary> /// <param name="samples">The samples used for the training.</param> /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param> /// <param name="parameters">The machine learning train parameters.</param> /// <returns>The trained <see cref="TokenizerModel"/>.</returns> public static TokenizerModel Train(IObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters) { return Train(samples, factory, parameters, null); }
private static TokenizerModel Train(TokenizerFactory factory) { return TokenizerME.Train(CreateSampleStream(), factory, TrainingParameters.DefaultParameters()); }
/// <summary> /// Initializes a new instance of the <see cref="TokenizerModel"/> class. /// </summary> /// <param name="tokenizerModel">The tokenizer model.</param> /// <param name="manifestInfoEntries">The manifest information entries.</param> /// <param name="tokenizerFactory">The tokenizer factory.</param> public TokenizerModel(IMaxentModel tokenizerModel, Dictionary<string, string> manifestInfoEntries, TokenizerFactory tokenizerFactory) : base(ComponentName, tokenizerFactory.LanguageCode, manifestInfoEntries, tokenizerFactory) { artifactMap.Add(TokenizerModelEntry, tokenizerModel); CheckArtifactMap(); }