/// <summary> /// Class constructor which takes the string locations of the /// information which the maxent model needs. /// </summary> public MaximumEntropyTokenizer(SharpEntropy.IMaximumEntropyModel model) { mContextGenerator = new TokenContextGenerator(); mAlphaNumericOptimization = false; mModel = model; mNewTokens = new List <Util.Span>(); mTokenProbabilities = new List <double>(50); }
public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader <string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator <Util.Pair <System.Text.StringBuilder, int> > contextGenerator) { mDataReader = dataReader; mScanner = scanner; mContextGenerator = contextGenerator; if (mDataReader.HasNext()) { string current = mDataReader.NextToken(); if (mDataReader.HasNext()) { mNext = mDataReader.NextToken(); } AddNewEvents(current); } }
/// <summary> /// Creates a new <code>MaximumEntropySentenceDetector</code> instance. /// </summary> /// <param name="model"> /// The IMaximumEntropyModel which this MaximumEntropySentenceDetector will use to /// evaluate end-of-sentence decisions. /// </param> /// <param name="contextGenerator">The IContextGenerator object which this MaximumEntropySentenceDetector /// will use to turn strings into contexts for the model to /// evaluate. /// </param> /// <param name="scanner">the EndOfSentenceScanner which this MaximumEntropySentenceDetector /// will use to locate end of sentence indexes. /// </param> public MaximumEntropySentenceDetector(SharpEntropy.IMaximumEntropyModel model, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator, IEndOfSentenceScanner scanner) { mModel = model; mContextGenerator = contextGenerator; mScanner = scanner; }
/// <summary> /// Class constructor which takes the string locations of the /// information which the maxent model needs. /// </summary> public MaximumEntropyTokenizer(SharpEntropy.IMaximumEntropyModel model) { mContextGenerator = new TokenContextGenerator(); mAlphaNumericOptimization = false; mModel = model; mNewTokens = new List<Util.Span>(); mTokenProbabilities = new List<double>(50); }
/// <summary> /// Constructor which takes a IMaximumEntropyModel and a IContextGenerator. /// calls the three-arg constructor with a default ed of sentence scanner. /// </summary> /// <param name="model"> /// The MaxentModel which this SentenceDetectorME will use to /// evaluate end-of-sentence decisions. /// </param> /// <param name="contextGenerator"> /// The IContextGenerator object which this MaximumEntropySentenceDetector /// will use to turn strings into contexts for the model to /// evaluate. /// </param> public MaximumEntropySentenceDetector(SharpEntropy.IMaximumEntropyModel model, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator) : this(model, contextGenerator, new DefaultEndOfSentenceScanner()) { }
/// <summary> /// Creates a new <code>MaximumEntropySentenceDetector</code> instance. /// </summary> /// <param name="model"> /// The IMaximumEntropyModel which this MaximumEntropySentenceDetector will use to /// evaluate end-of-sentence decisions. /// </param> /// <param name="contextGenerator">The IContextGenerator object which this MaximumEntropySentenceDetector /// will use to turn strings into contexts for the model to /// evaluate. /// </param> /// <param name="scanner">the EndOfSentenceScanner which this MaximumEntropySentenceDetector /// will use to locate end of sentence indexes. /// </param> public MaximumEntropySentenceDetector(SharpEntropy.IMaximumEntropyModel model, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator, IEndOfSentenceScanner scanner) { mModel = model; mContextGenerator = contextGenerator; mScanner = scanner; }
public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader<string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator) { mDataReader = dataReader; mScanner = scanner; mContextGenerator = contextGenerator; if (mDataReader.HasNext()) { string current = mDataReader.NextToken(); if (mDataReader.HasNext()) { mNext = mDataReader.NextToken(); } AddNewEvents(current); } }