Beispiel #1
0
        /// <summary>
        /// Trains a model for the <see cref="TokenizerME"/>.
        /// </summary>
        /// <param name="samples">The samples used for the training.</param>
        /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
        /// <param name="parameters">The machine learning train parameters.</param>
        /// <param name="monitor">
        /// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation.
        /// This argument can be a <c>null</c> value.
        /// </param>
        /// <returns>The trained <see cref="TokenizerModel"/>.</returns>
        public static TokenizerModel Train(IObjectStream <TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters, Monitor monitor)
        {
            var manifestInfoEntries = new Dictionary <string, string>();

            var eventStream = new TokSpanEventStream(samples, factory.UseAlphaNumericOptimization,
                                                     factory.AlphaNumericPattern, factory.ContextGenerator);

            var trainer = TrainerFactory.GetEventTrainer(parameters, manifestInfoEntries, monitor);
            var model   = trainer.Train(eventStream);

            return(new TokenizerModel(model, manifestInfoEntries, factory));
        }
Beispiel #2
0
 /// <summary>
 /// Trains a model for the <see cref="TokenizerME"/>.
 /// </summary>
 /// <param name="samples">The samples used for the training.</param>
 /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
 /// <param name="parameters">The machine learning train parameters.</param>
 /// <returns>The trained <see cref="TokenizerModel"/>.</returns>
 public static TokenizerModel Train(IObjectStream <TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters)
 {
     return(Train(samples, factory, parameters, null));
 }
 /// <summary>
 /// Initializes a new instance of the <see cref="TokenizerModel"/> class.
 /// </summary>
 /// <param name="tokenizerModel">The tokenizer model.</param>
 /// <param name="manifestInfoEntries">The manifest information entries.</param>
 /// <param name="tokenizerFactory">The tokenizer factory.</param>
 public TokenizerModel(IMaxentModel tokenizerModel, Dictionary <string, string> manifestInfoEntries, TokenizerFactory tokenizerFactory)
     : base(ComponentName, tokenizerFactory.LanguageCode, manifestInfoEntries, tokenizerFactory)
 {
     artifactMap.Add(TokenizerModelEntry, tokenizerModel);
     CheckArtifactMap();
 }
Beispiel #4
0
        /// <summary>
        /// Trains a model for the <see cref="TokenizerME"/>.
        /// </summary>
        /// <param name="samples">The samples used for the training.</param>
        /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
        /// <param name="parameters">The machine learning train parameters.</param>
        /// <param name="monitor">
        /// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation.
        /// This argument can be a <c>null</c> value.
        /// </param>
        /// <returns>The trained <see cref="TokenizerModel"/>.</returns>
        public static TokenizerModel Train(IObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters, Monitor monitor) {
            var manifestInfoEntries = new Dictionary<string, string>();

            var eventStream = new TokSpanEventStream(samples, factory.UseAlphaNumericOptimization,
                factory.AlphaNumericPattern, factory.ContextGenerator);

            var trainer = TrainerFactory.GetEventTrainer(parameters, manifestInfoEntries, monitor);
            var model = trainer.Train(eventStream);

            return new TokenizerModel(model, manifestInfoEntries, factory);
        }
Beispiel #5
0
 /// <summary>
 /// Trains a model for the <see cref="TokenizerME"/>.
 /// </summary>
 /// <param name="samples">The samples used for the training.</param>
 /// <param name="factory">A <see cref="TokenizerFactory"/> to get resources from.</param>
 /// <param name="parameters">The machine learning train parameters.</param>
 /// <returns>The trained <see cref="TokenizerModel"/>.</returns>
 public static TokenizerModel Train(IObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters parameters) {
     return Train(samples, factory, parameters, null);
 }
 private static TokenizerModel Train(TokenizerFactory factory) {
     return TokenizerME.Train(CreateSampleStream(), factory, TrainingParameters.DefaultParameters());
 }
Beispiel #7
0
 /// <summary>
 /// Initializes a new instance of the <see cref="TokenizerModel"/> class.
 /// </summary>
 /// <param name="tokenizerModel">The tokenizer model.</param>
 /// <param name="manifestInfoEntries">The manifest information entries.</param>
 /// <param name="tokenizerFactory">The tokenizer factory.</param>
 public TokenizerModel(IMaxentModel tokenizerModel, Dictionary<string, string> manifestInfoEntries, TokenizerFactory tokenizerFactory)
     : base(ComponentName, tokenizerFactory.LanguageCode, manifestInfoEntries, tokenizerFactory) {
     
     artifactMap.Add(TokenizerModelEntry, tokenizerModel);
     CheckArtifactMap();
 }