/// <summary>
/// Class constructor which takes the string locations of the
/// information which the maxent model needs.
/// </summary>
public MaximumEntropyTokenizer(IMaximumEntropyModel model)
{
_contextGenerator = new TokenContextGenerator();
AlphaNumericOptimization = false;
this._model = model;
_newTokens = new List <Span>();
_tokenProbabilities = new List <double>(50);
}
/// <summary>
/// Constructor sets up the training event reader based on a stream of training data.
/// </summary>
/// <param name="dataReader">
/// Stream of training data.
/// </param>
public BasicEventReader(ITrainingDataReader<string> dataReader)
{
mContext = new BasicContextGenerator();
mDataReader = dataReader;
if (mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
}
/// <summary>
/// Constructor sets up the training event reader based on a stream of training data.
/// </summary>
/// <param name="dataReader">
/// Stream of training data.
/// </param>
public BasicEventReader(ITrainingDataReader <string> dataReader)
{
mContext = new BasicContextGenerator();
mDataReader = dataReader;
if (mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
}
/// <summary>
/// Initializes a new instance of the <see cref="BasicEventStream"/> class.
/// </summary>
/// <param name="dataStream">The data stream.</param>
/// <param name="separator">The separator.</param>
public BasicEventStream(IDataStream dataStream, string separator)
{
_dataStream = dataStream;
_separator = separator;
_contextGenerator = new BasicContextGenerator(_separator);
if (_dataStream.HasNext())
{
_nextEvent = CreateEvent((string)_dataStream.NextToken());
}
}
/// <summary>
/// Initializes a new instance of the <see cref="BasicEventStream"/> class.
/// </summary>
/// <param name="dataStream">The data stream.</param>
/// <param name="separator">The separator.</param>
public BasicEventStream(IDataStream dataStream, string separator)
{
_dataStream = dataStream;
_separator = separator;
_contextGenerator = new BasicContextGenerator(_separator);
if (_dataStream.HasNext())
{
_nextEvent = CreateEvent((string)_dataStream.NextToken());
}
}
public SentenceDetectionEventReader(ITrainingDataReader<string> dataReader, IEndOfSentenceScanner scanner,
IContextGenerator<Tuple<StringBuilder, int>> contextGenerator)
{
_dataReader = dataReader;
_scanner = scanner;
_contextGenerator = contextGenerator;
if (_dataReader.HasNext())
{
string current = _dataReader.NextToken();
if (_dataReader.HasNext())
{
_next = _dataReader.NextToken();
}
AddNewEvents(current);
}
}
public SentenceDetectionEventReader(ITrainingDataReader <string> dataReader, IEndOfSentenceScanner scanner,
IContextGenerator <Tuple <StringBuilder, int> > contextGenerator)
{
_dataReader = dataReader;
_scanner = scanner;
_contextGenerator = contextGenerator;
if (_dataReader.HasNext())
{
string current = _dataReader.NextToken();
if (_dataReader.HasNext())
{
_next = _dataReader.NextToken();
}
AddNewEvents(current);
}
}
/// <summary>
/// Creates a new <code>MaximumEntropySentenceDetector</code> instance.
/// </summary>
/// <param name="model">
/// The IMaximumEntropyModel which this MaximumEntropySentenceDetector will use to
/// evaluate end-of-sentence decisions.
/// </param>
/// <param name="contextGenerator">The IContextGenerator object which this MaximumEntropySentenceDetector
/// will use to turn strings into contexts for the model to
/// evaluate.
/// </param>
/// <param name="scanner">the EndOfSentenceScanner which this MaximumEntropySentenceDetector
/// will use to locate end of sentence indexes.
/// </param>
public MaximumEntropySentenceDetector(IMaximumEntropyModel model, IContextGenerator<Tuple<StringBuilder, int>> contextGenerator, IEndOfSentenceScanner scanner)
{
_model = model;
_contextGenerator = contextGenerator;
_scanner = scanner;
}
/// <summary>
/// Creates a new <code>MaximumEntropySentenceDetector</code> instance.
/// </summary>
/// <param name="model">
/// The IMaximumEntropyModel which this MaximumEntropySentenceDetector will use to
/// evaluate end-of-sentence decisions.
/// </param>
/// <param name="contextGenerator">The IContextGenerator object which this MaximumEntropySentenceDetector
/// will use to turn strings into contexts for the model to
/// evaluate.
/// </param>
/// <param name="scanner">the EndOfSentenceScanner which this MaximumEntropySentenceDetector
/// will use to locate end of sentence indexes.
/// </param>
public MaximumEntropySentenceDetector(IMaximumEntropyModel model, IContextGenerator <Tuple <StringBuilder, int> > contextGenerator, IEndOfSentenceScanner scanner)
{
_model = model;
_contextGenerator = contextGenerator;
_scanner = scanner;
}
/// <summary>
/// Constructor which takes a IMaximumEntropyModel and a IContextGenerator.
/// calls the three-arg constructor with a default ed of sentence scanner.
/// </summary>
/// <param name="model">
/// The MaxentModel which this SentenceDetectorME will use to
/// evaluate end-of-sentence decisions.
/// </param>
/// <param name="contextGenerator">
/// The IContextGenerator object which this MaximumEntropySentenceDetector
/// will use to turn strings into contexts for the model to
/// evaluate.
/// </param>
public MaximumEntropySentenceDetector(IMaximumEntropyModel model, IContextGenerator <Tuple <StringBuilder, int> > contextGenerator) :
this(model, contextGenerator, new DefaultEndOfSentenceScanner())
{
}
public InvalidEmailDetectionEventReader(ITrainingDataReader <string> dataReader,
IContextGenerator <string> contextGenerator)
{
_dataReader = dataReader;
_contextGenerator = contextGenerator;
}
// Constructors ------------------
/// <summary>
/// Constructor which takes a IMaximumEntropyModel and calls the three-arg
/// constructor with that model, a SentenceDetectionContextGenerator, and the
/// default end of sentence scanner.
/// </summary>
/// <param name="model">
/// The MaxentModel which this SentenceDetectorME will use to
/// evaluate end-of-sentence decisions.
/// </param>
public MaximumEntropyInvalidEmailDetector(IMaximumEntropyModel model)
{
_contextGenerator = new InvalidEmailDetectionContextGenerator();
_model = model;
}
public JsonDataContextEnricher (IContextGenerator contextGenerator)
{
this.contextGenerator = contextGenerator;
}
/// <summary>
/// Class constructor which takes the string locations of the
/// information which the maxent model needs.
/// </summary>
public MaximumEntropyTokenizer(IMaximumEntropyModel model)
{
_contextGenerator = new TokenContextGenerator();
AlphaNumericOptimization = false;
this._model = model;
}
/// <summary>
/// Constructor which takes a IMaximumEntropyModel and a IContextGenerator.
/// calls the three-arg constructor with a default ed of sentence scanner.
/// </summary>
/// <param name="model">
/// The MaxentModel which this SentenceDetectorME will use to
/// evaluate end-of-sentence decisions.
/// </param>
/// <param name="contextGenerator">
/// The IContextGenerator object which this MaximumEntropySentenceDetector
/// will use to turn strings into contexts for the model to
/// evaluate.
/// </param>
public MaximumEntropySentenceDetector(IMaximumEntropyModel model, IContextGenerator<Tuple<StringBuilder, int>> contextGenerator):
this(model, contextGenerator, new DefaultEndOfSentenceScanner()){}
/// <summary>
/// Class constructor which takes the string locations of the
/// information which the maxent model needs.
/// </summary>
public MaximumEntropyTokenizer(IMaximumEntropyModel model)
{
_contextGenerator = new TokenContextGenerator();
AlphaNumericOptimization = false;
this._model = model;
}
public JsonDataContextEnricher(IContextGenerator contextGenerator)
{
this.contextGenerator = contextGenerator;
}
public ContextCheckerService(IContextGenerator contextGenerator)
{
_contextGenerator = contextGenerator;
}
public JsonTelemetryEnricher(IContextGenerator contextGenerator)
{
this.contextGenerator = contextGenerator;
}
/// <summary>
/// Class constructor which takes the string locations of the
/// information which the maxent model needs.
/// </summary>
public MaximumEntropyTokenizer(IMaximumEntropyModel model)
{
_contextGenerator = new TokenContextGenerator();
AlphaNumericOptimization = false;
this._model = model;
_newTokens = new List<Span>();
_tokenProbabilities = new List<double>(50);
}
public InvalidEmailDetectionDataEventReader(IEnumerable <EmailAndValidity> emailsAndValidities)
{
this._currentIndex = 0;
this._emailsAndValidities = emailsAndValidities.ToList();
this._contextGenerator = new InvalidEmailDetectionContextGenerator();
}
public JsonTelemetryEnricher (IContextGenerator contextGenerator)
{
this.contextGenerator = contextGenerator;
}
public ContextScaffolder(ScaffolderDependencies dependencies, IContextGenerator generator) : base(dependencies)
{
Generator = generator;
}
