/// <summary> /// Initializes a new instance of the <see cref="ChunkerME"/> with the specified <see cref="ChunkerModel"/>. /// </summary> /// <param name="model">The chunker model.</param> public ChunkerME(ChunkerModel model) { contextGenerator = model.Factory.GetContextGenerator(); sequenceValidator = model.Factory.GetSequenceValidator(); this.model = model.ChunkerSequenceModel ?? new BeamSearch(model.BeamSize, model.MaxentModel); }
/// <summary> /// Initializes the current instance with the specified model and the specified beam size. /// </summary> /// <param name="model">The model for this chunker</param> /// <param name="beamSize">The size of the beam that should be used when decoding sequences.</param> /// <param name="sequenceValidator">The <see cref="ISequenceValidator{String}"/> to determines whether the outcome is valid for the preceding sequence. This can be used to implement constraints on what sequences are valid..</param> /// <param name="contextGenerator">The context generator.</param> internal ChunkerME(ChunkerModel model, int beamSize, ISequenceValidator<string> sequenceValidator, IChunkerContextGenerator contextGenerator) { // This method is marked as deprecated in the OpenNLP, but it is required in the Parser, // I could change the cg in the factory, but its not ideal in this situation (i think) :P this.sequenceValidator = sequenceValidator; this.contextGenerator = contextGenerator; this.model = model.ChunkerSequenceModel ?? new BeamSearch(beamSize, model.MaxentModel); }
/// <summary> /// Initializes the current instance with the specified model and the specified beam size. /// </summary> /// <param name="model">The model for this chunker</param> /// <param name="beamSize">The size of the beam that should be used when decoding sequences.</param> /// <param name="sequenceValidator">The <see cref="ISequenceValidator{String}"/> to determines whether the outcome is valid for the preceding sequence. This can be used to implement constraints on what sequences are valid..</param> /// <param name="contextGenerator">The context generator.</param> internal ChunkerME(ChunkerModel model, int beamSize, ISequenceValidator <string> sequenceValidator, IChunkerContextGenerator contextGenerator) { // This method is marked as deprecated in the OpenNLP, but it is required in the Parser, // I could change the cg in the factory, but its not ideal in this situation (i think) :P this.sequenceValidator = sequenceValidator; this.contextGenerator = contextGenerator; this.model = model.ChunkerSequenceModel ?? new BeamSearch(beamSize, model.MaxentModel); }
/// <summary> /// Finds the sequence with the highest probability. /// </summary> /// <param name="sequence">The sequence.</param> /// <param name="additionalContext">The additional context.</param> /// <param name="beamSearch">The beam search.</param> /// <param name="validator">The validator.</param> /// <returns>The sequence with the highest probability.</returns> public Sequence BestSequence(T[] sequence, object[] additionalContext, IBeamSearchContextGenerator <T> beamSearch, ISequenceValidator <T> validator) { var sequences = BestSequences(1, sequence, additionalContext, beamSearch, validator); if (sequence.Length > 0) { return(sequences[0]); } return(null); }
public LemmatizerME(LemmatizerModel model) { if (model == null) { throw new ArgumentNullException(nameof(model)); } contextGenerator = model.Factory.GetContextGenerator(); sequenceValidator = model.Factory.GetSequenceValidator(); // Knuppe: In the original implementation there is condition to recreate the beamsearch object, but // the condition is impossible to occur, due to the getLemmatizerSequenceModel() method logic this.model = model.LemmatizerSequenceModel; }
/// <summary> /// Creates new search object. /// </summary> /// <param name="size">The size of the beam (k).</param> /// <param name="cg">The context generator for the model.</param> /// <param name="model">The model for assigning probabilities to the sequence outcomes.</param> /// <param name="validator">The sequence validator.</param> /// <param name="cacheSize">Size of the cache.</param> public BeamSearch(int size, IBeamSearchContextGenerator <T> cg, IMaxentModel model, ISequenceValidator <T> validator, int cacheSize) { this.cg = cg; this.size = size; this.model = model; this.validator = validator; if (cacheSize > 0) { contextsCache = new Cache(cacheSize); } probs = new double[model.GetNumOutcomes()]; }
/// <summary> /// Initializes a new instance of the <see cref="NameFinderME"/> using the given <see cref="TokenNameFinderModel"/>. /// </summary> /// <param name="model">The model.</param> /// <exception cref="System.ArgumentNullException">model</exception> public NameFinderME(TokenNameFinderModel model) { if (model == null) { throw new ArgumentNullException(nameof(model)); } sequenceCodec = model.Factory.CreateSequenceCodec(); sequenceValidator = sequenceCodec.CreateSequenceValidator(); this.model = model.NameFinderSequenceModel; contextGenerator = model.Factory.CreateContextGenerator(); // TODO: We should deprecate this. And come up with a better solution! additionalContextFeatureGenerator = new AdditionalContextFeatureGenerator(); contextGenerator.AddFeatureGenerator( new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8)); }
/// <summary> /// Finds the n most probable sequences. /// </summary> /// <param name="numSequences">The number sequences.</param> /// <param name="sequence">The sequence.</param> /// <param name="additionalContext">The additional context.</param> /// <param name="minSequenceScore">The minimum sequence score.</param> /// <param name="beamSearch">The beam search.</param> /// <param name="validator">The validator.</param> public Sequence[] BestSequences(int numSequences, T[] sequence, object[] additionalContext, double minSequenceScore, IBeamSearchContextGenerator <T> beamSearch, ISequenceValidator <T> validator) { IHeap <Sequence> prev = new ListHeap <Sequence>(size); IHeap <Sequence> next = new ListHeap <Sequence>(size); prev.Add(new Sequence()); if (additionalContext == null) { additionalContext = new object[] {}; // EMPTY_ADDITIONAL_CONTEXT } for (var i = 0; i < sequence.Length; i++) { var sz = Math.Min(size, prev.Size()); for (var sc = 0; prev.Size() > 0 && sc < sz; sc++) { var top = prev.Extract(); var tmpOutcomes = top.Outcomes; var outcomes = tmpOutcomes.ToArray(); var contexts = beamSearch.GetContext(i, sequence, outcomes, additionalContext); double[] scores; if (contextsCache != null) { scores = (double[])contextsCache.Get(contexts); if (scores == null) { scores = model.Eval(contexts, probs); contextsCache.Put(contexts, scores); } } else { scores = model.Eval(contexts, probs); } var tempScores = new double[scores.Length]; for (var c = 0; c < scores.Length; c++) { tempScores[c] = scores[c]; } Array.Sort(tempScores); var min = tempScores[Math.Max(0, scores.Length - size)]; for (var p = 0; p < scores.Length; p++) { if (scores[p] < min) { continue; //only advance first "size" outcomes } var outcome = model.GetOutcome(p); if (validator.ValidSequence(i, sequence, outcomes, outcome)) { var ns = new Sequence(top, outcome, scores[p]); if (ns.Score > minSequenceScore) { next.Add(ns); } } } if (next.Size() == 0) { //if no advanced sequences, advance all valid for (var p = 0; p < scores.Length; p++) { var outcome = model.GetOutcome(p); if (validator.ValidSequence(i, sequence, outcomes, outcome)) { var ns = new Sequence(top, outcome, scores[p]); if (ns.Score > minSequenceScore) { next.Add(ns); } } } } } // make prev = next; and re-init next (we reuse existing prev set once we clear it) prev.Clear(); var tmp = prev; prev = next; next = tmp; } var numSeq = Math.Min(numSequences, prev.Size()); var topSequences = new Sequence[numSeq]; for (var seqIndex = 0; seqIndex < numSeq; seqIndex++) { topSequences[seqIndex] = prev.Extract(); } return(topSequences); }
/// <summary> /// Finds the n most probable sequences. /// </summary> /// <param name="numSequences">The number sequences.</param> /// <param name="sequence">The sequence.</param> /// <param name="additionalContext">The additional context.</param> /// <param name="beamSearch">The beam search.</param> /// <param name="validator">The validator.</param> /// <returns>The n most probable sequences.</returns> public Sequence[] BestSequences(int numSequences, T[] sequence, object[] additionalContext, IBeamSearchContextGenerator <T> beamSearch, ISequenceValidator <T> validator) { return(BestSequences(numSequences, sequence, additionalContext, zeroLog, beamSearch, validator)); }
/// <summary> /// Initializes a new instance of the <see cref="NameFinderME"/> using the given <see cref="TokenNameFinderModel"/>. /// </summary> /// <param name="model">The model.</param> /// <exception cref="System.ArgumentNullException">model</exception> public NameFinderME(TokenNameFinderModel model) { if (model == null) throw new ArgumentNullException("model"); sequenceCodec = model.Factory.CreateSequenceCodec(); sequenceValidator = sequenceCodec.CreateSequenceValidator(); this.model = model.NameFinderSequenceModel; contextGenerator = model.Factory.CreateContextGenerator(); // TODO: We should deprecate this. And come up with a better solution! additionalContextFeatureGenerator = new AdditionalContextFeatureGenerator(); contextGenerator.AddFeatureGenerator( new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8)); }