/// <summary> Use this training method if you wish to supply an end of
/// sentence scanner which provides a different set of ending chars
/// other than the default ones. They are "\\.|!|\\?|\\\"|\\)".
/// </summary>
public static SharpEntropy.GisModel TrainModel(string inFile, int iterations, int cut, IEndOfSentenceScanner scanner)
{
SharpEntropy.ITrainingEventReader eventReader;
SharpEntropy.ITrainingDataReader <string> dataReader;
System.IO.StreamReader streamReader;
using (streamReader = new System.IO.StreamReader(inFile, System.Text.Encoding.UTF7))
{
dataReader = new SharpEntropy.PlainTextByLineDataReader(streamReader);
eventReader = new SentenceDetectionEventReader(dataReader, scanner);
SharpEntropy.GisTrainer trainer = new SharpEntropy.GisTrainer();
trainer.TrainModel(eventReader, iterations, cut);
return(new SharpEntropy.GisModel(trainer));
}
}
public static GisModel TrainModel(IEnumerable <string> files, int iterations, int cut, IEndOfSentenceScanner scanner)
{
var trainer = new GisTrainer();
foreach (var file in files)
{
using (var streamReader = new StreamReader(file))
{
ITrainingDataReader <string> dataReader = new PlainTextByLineDataReader(streamReader);
ITrainingEventReader eventReader = new SentenceDetectionEventReader(dataReader, scanner);
trainer.TrainModel(eventReader, iterations, cut);
}
}
return(new GisModel(trainer));
}
public static GisModel TrainModel(IEnumerable <string> filePaths, int iterations, int cut, IEndOfSentenceScanner scanner)
{
var trainer = new GisTrainer();
#if DNF
var readers = filePaths.Select(path => new StreamReader(path)).ToList();
#else
var readers = filePaths.Select(path => new StreamReader(new FileStream(path, FileMode.OpenOrCreate))).ToList();
#endif
// train the model
ITrainingDataReader <string> dataReader = new MultipleFilesPlainTextByLineDataReader(readers);
ITrainingEventReader eventReader = new SentenceDetectionEventReader(dataReader, scanner);
trainer.TrainModel(eventReader, iterations, cut);
return(new GisModel(trainer));
}
/// <summary> Use this training method if you wish to supply an end of
/// sentence scanner which provides a different set of ending chars
/// other than the default ones. They are "\\.|!|\\?|\\\"|\\)".
/// </summary>
public static SharpEntropy.GisModel TrainModel(string inFile, int iterations, int cut, IEndOfSentenceScanner scanner)
{
SharpEntropy.ITrainingEventReader eventReader;
SharpEntropy.ITrainingDataReader<string> dataReader;
System.IO.StreamReader streamReader;
using (streamReader = new System.IO.StreamReader(inFile, System.Text.Encoding.UTF7))
{
dataReader = new SharpEntropy.PlainTextByLineDataReader(streamReader);
eventReader = new SentenceDetectionEventReader(dataReader, scanner);
SharpEntropy.GisTrainer trainer = new SharpEntropy.GisTrainer();
trainer.TrainModel(eventReader, iterations, cut);
return new SharpEntropy.GisModel(trainer);
}
}
public static GisModel TrainModel(IEnumerable<string> filePaths, int iterations, int cut, IEndOfSentenceScanner scanner)
{
var trainer = new GisTrainer();
var readers = filePaths.Select(path => new StreamReader(path)).ToList();
// train the model
ITrainingDataReader<string> dataReader = new MultipleFilesPlainTextByLineDataReader(readers);
ITrainingEventReader eventReader = new SentenceDetectionEventReader(dataReader, scanner);
trainer.TrainModel(eventReader, iterations, cut);
return new GisModel(trainer);
}
public static GisModel TrainModel(IEnumerable<string> files, int iterations, int cut, IEndOfSentenceScanner scanner)
{
var trainer = new GisTrainer();
foreach (var file in files)
{
using (var streamReader = new StreamReader(file))
{
ITrainingDataReader<string> dataReader = new PlainTextByLineDataReader(streamReader);
ITrainingEventReader eventReader = new SentenceDetectionEventReader(dataReader, scanner);
trainer.TrainModel(eventReader, iterations, cut);
}
}
return new GisModel(trainer);
}
