/// <summary>
/// Class constructor which takes the string locations of the
/// information which the maxent model needs.
/// </summary>
public MaximumEntropyTokenizer(SharpEntropy.IMaximumEntropyModel model)
{
mContextGenerator = new TokenContextGenerator();
mAlphaNumericOptimization = false;
mModel = model;
mNewTokens = new List <Util.Span>();
mTokenProbabilities = new List <double>(50);
}
public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader <string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator <Util.Pair <System.Text.StringBuilder, int> > contextGenerator)
{
mDataReader = dataReader;
mScanner = scanner;
mContextGenerator = contextGenerator;
if (mDataReader.HasNext())
{
string current = mDataReader.NextToken();
if (mDataReader.HasNext())
{
mNext = mDataReader.NextToken();
}
AddNewEvents(current);
}
}
/// <summary>
/// Creates a new <code>MaximumEntropySentenceDetector</code> instance.
/// </summary>
/// <param name="model">
/// The IMaximumEntropyModel which this MaximumEntropySentenceDetector will use to
/// evaluate end-of-sentence decisions.
/// </param>
/// <param name="contextGenerator">The IContextGenerator object which this MaximumEntropySentenceDetector
/// will use to turn strings into contexts for the model to
/// evaluate.
/// </param>
/// <param name="scanner">the EndOfSentenceScanner which this MaximumEntropySentenceDetector
/// will use to locate end of sentence indexes.
/// </param>
public MaximumEntropySentenceDetector(SharpEntropy.IMaximumEntropyModel model, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator, IEndOfSentenceScanner scanner)
{
mModel = model;
mContextGenerator = contextGenerator;
mScanner = scanner;
}
/// <summary>
/// Class constructor which takes the string locations of the
/// information which the maxent model needs.
/// </summary>
public MaximumEntropyTokenizer(SharpEntropy.IMaximumEntropyModel model)
{
mContextGenerator = new TokenContextGenerator();
mAlphaNumericOptimization = false;
mModel = model;
mNewTokens = new List<Util.Span>();
mTokenProbabilities = new List<double>(50);
}
/// <summary>
/// Constructor which takes a IMaximumEntropyModel and a IContextGenerator.
/// calls the three-arg constructor with a default ed of sentence scanner.
/// </summary>
/// <param name="model">
/// The MaxentModel which this SentenceDetectorME will use to
/// evaluate end-of-sentence decisions.
/// </param>
/// <param name="contextGenerator">
/// The IContextGenerator object which this MaximumEntropySentenceDetector
/// will use to turn strings into contexts for the model to
/// evaluate.
/// </param>
public MaximumEntropySentenceDetector(SharpEntropy.IMaximumEntropyModel model, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator)
: this(model, contextGenerator, new DefaultEndOfSentenceScanner())
{
}
/// <summary>
/// Creates a new <code>MaximumEntropySentenceDetector</code> instance.
/// </summary>
/// <param name="model">
/// The IMaximumEntropyModel which this MaximumEntropySentenceDetector will use to
/// evaluate end-of-sentence decisions.
/// </param>
/// <param name="contextGenerator">The IContextGenerator object which this MaximumEntropySentenceDetector
/// will use to turn strings into contexts for the model to
/// evaluate.
/// </param>
/// <param name="scanner">the EndOfSentenceScanner which this MaximumEntropySentenceDetector
/// will use to locate end of sentence indexes.
/// </param>
public MaximumEntropySentenceDetector(SharpEntropy.IMaximumEntropyModel model, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator, IEndOfSentenceScanner scanner)
{
mModel = model;
mContextGenerator = contextGenerator;
mScanner = scanner;
}
public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader<string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator)
{
mDataReader = dataReader;
mScanner = scanner;
mContextGenerator = contextGenerator;
if (mDataReader.HasNext())
{
string current = mDataReader.NextToken();
if (mDataReader.HasNext())
{
mNext = mDataReader.NextToken();
}
AddNewEvents(current);
}
}
