public void testSimpleTraining() {
IObjectStream<DocumentSample> samples = new GenericObjectStream<DocumentSample>(new[] {
new DocumentSample("1", new[] {"a", "b", "c"}),
new DocumentSample("1", new[] {"a", "b", "c", "1", "2"}),
new DocumentSample("1", new[] {"a", "b", "c", "3", "4"}),
new DocumentSample("0", new[] {"x", "y", "z"}),
new DocumentSample("0", new[] {"x", "y", "z", "5", "6"}),
new DocumentSample("0", new[] {"x", "y", "z", "7", "8"})
});
var param = new TrainingParameters();
param.Set(Parameters.Iterations, "100");
param.Set(Parameters.Cutoff, "0");
var model = DocumentCategorizerME.Train("x-unspecified", samples, param, new DocumentCategorizerFactory());
var doccat = new DocumentCategorizerME(model);
var aProbs = doccat.Categorize("a");
Assert.AreEqual("1", doccat.GetBestCategory(aProbs));
var bProbs = doccat.Categorize("x");
Assert.AreEqual("0", doccat.GetBestCategory(bProbs));
//test to make sure sorted map's last key is cat 1 because it has the highest score.
var sortedScoreMap = doccat.SortedScoreMap("a");
foreach (var pair in sortedScoreMap) {
Assert.AreEqual("1", pair.Value[0]);
break;
}
}
public void Setup() {
mlParams = new TrainingParameters();
mlParams.Set(Parameters.Algorithm, GIS.MaxEntropy);
mlParams.Set(Parameters.Iterations, "10");
mlParams.Set(Parameters.Cutoff, "5");
TrainerFactory.RegisterTrainer("Dummy", typeof(DummyTrainer));
}
public static TokenizerModel CreateMaxentTokenModel() {
using (var data = Tests.OpenFile("/opennlp/tools/tokenize/token.train")) {
var samples = new TokenSampleStream(new PlainTextByLineStream(data));
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
return TokenizerME.Train(samples, new TokenizerFactory("en", null, true), mlParams);
}
}
public void Setup() {
var p = new TrainingParameters();
p.Set(Parameters.Iterations, "70");
p.Set(Parameters.Cutoff, "1");
var chunkerModel = ChunkerME.Train("en", CreateSampleStream(), p, new ChunkerFactory());
chunker = new ChunkerME(chunkerModel);
}
public void TestNameFinder() {
using (var file = Tests.OpenFile("opennlp/tools/namefind/AnnotatedSentences.txt")) {
var sampleStream = new NameSampleStream(new PlainTextByLineStream(file, "ISO-8859-1"));
var param = new TrainingParameters();
param.Set(Parameters.Iterations, "70");
param.Set(Parameters.Cutoff, "1");
var model = NameFinderME.Train(
"en",
sampleStream,
param,
new TokenNameFinderFactory(null, new Dictionary<string, object>()));
var nameFinder = new NameFinderME(model);
// now test if it can detect the sample sentences
var sentence = new[] {
"Alisa",
"appreciated",
"the",
"hint",
"and",
"enjoyed",
"a",
"delicious",
"traditional",
"meal."
};
var names = nameFinder.Find(sentence);
Assert.AreEqual(1, names.Length);
Assert.AreEqual(new Span(0, 1, Type), names[0]);
sentence = new[] {
"Hi",
"Mike",
",",
"it's",
"Stefanie",
"Schmidt",
"."
};
names = nameFinder.Find(sentence);
Assert.AreEqual(2, names.Length);
Assert.AreEqual(new Span(1, 2, Type), names[0]);
Assert.AreEqual(new Span(4, 6, Type), names[1]);
}
}
public static TokenizerModel TrainModel(string path)
{
FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read);
TokenSampleStream stream = new TokenSampleStream(new PlainTextByLineStream(fs));
TrainingParameters trainParams = new TrainingParameters();
trainParams.Set(Parameters.Iterations, "100");
trainParams.Set(Parameters.Cutoff, "0");
return(TokenizerME.Train(stream, new TokenizerFactory(TRAINING_LANGUAGE, null, true), trainParams));
}
public void TestPerceptronOnPrepAttachDataWithSkippedAveraging()
{
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.Perceptron);
trainParams.Set(Parameters.Cutoff, "1");
trainParams.Set(Parameters.UseSkippedAveraging, "true");
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.773706362961129);
}
public void TestMaxentOnPrepAttachDataWithParams() {
var reportMap = new Dictionary<string, string>();
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEnt);
trainParams.Set(Parameters.DataIndexer, Parameters.DataIndexers.TwoPass);
trainParams.Set(Parameters.Cutoff, "1");
var trainer = TrainerFactory.GetEventTrainer(trainParams, reportMap, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.7997028967566229d);
}
public void TestPerceptronOnPrepAttachDataWithTolerance() {
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.Perceptron);
trainParams.Set(Parameters.Cutoff, "1");
trainParams.Set(Parameters.Iterations, "500");
trainParams.Set(Parameters.Tolerance, "0.0001");
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.7677642980935875);
}
public void TestPerceptronOnPrepAttachDataWithSkippedAveraging() {
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.Perceptron);
trainParams.Set(Parameters.Cutoff, "1");
trainParams.Set(Parameters.UseSkippedAveraging, "true");
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.773706362961129);
}
public void TestPerceptronOnPrepAttachDataWithTolerance()
{
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.Perceptron);
trainParams.Set(Parameters.Cutoff, "1");
trainParams.Set(Parameters.Iterations, "500");
trainParams.Set(Parameters.Tolerance, "0.0001");
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.7677642980935875);
}
public void TestQnOnPrepAttachDataInParallel()
{
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEntQn);
//trainParams.Set(Parameters.Iterations, "100");
trainParams.Set(Parameters.Threads, "2");
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
TestModel(model, 0.8115870264917059);
}
public static LemmatizerModel TrainModel(string path)
{
FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read);
TrainingParameters trainParams = new TrainingParameters();
trainParams.Set(Parameters.Iterations, "1");
trainParams.Set(Parameters.Cutoff, "0");
LemmatizerFactory lemmatizerFactory = new LemmatizerFactory();
LemmaSampleStream sampleStream = new LemmaSampleStream(new PlainTextByLineStream(fs));
return(LemmatizerME.Train(TRAINING_LANGUAGE, sampleStream, trainParams, lemmatizerFactory));
}
public void TestMaxentOnPrepAttachDataWithParams()
{
var reportMap = new Dictionary <string, string>();
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEnt);
trainParams.Set(Parameters.DataIndexer, Parameters.DataIndexers.TwoPass);
trainParams.Set(Parameters.Cutoff, "1");
var trainer = TrainerFactory.GetEventTrainer(trainParams, reportMap, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.7997028967566229d);
}
public static SentenceModel TrainModel(string path)
{
FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read);
TrainingParameters trainParams = new TrainingParameters();
trainParams.Set(Parameters.Iterations, "100");
trainParams.Set(Parameters.Cutoff, "0");
SentenceDetectorFactory detectorFactory = new SentenceDetectorFactory(TRAINING_LANGUAGE, true, null, null);
SentenceSampleStream sampleStream = new SentenceSampleStream(new PlainTextByLineStream(fs));
return(SentenceDetectorME.Train(TRAINING_LANGUAGE, sampleStream, detectorFactory, trainParams));
}
public void TestQnOnPrepAttachDataWithL2Params()
{
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEntQn);
trainParams.Set(Parameters.DataIndexer, Parameters.DataIndexers.TwoPass);
trainParams.Set(Parameters.Cutoff, "1");
trainParams.Set(Parameters.L1Cost, "0");
trainParams.Set(Parameters.L2Cost, "1.0");
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
TestModel(model, 0.8227283981183461);
}
public void TestWithNameEvaluationErrorListener()
{
using (var file = Tests.OpenFile("opennlp/tools/namefind/AnnotatedSentences.txt")) {
var sampleStream = new NameSampleStream(new PlainTextByLineStream(file, "ISO-8859-1"));
var parameters = new TrainingParameters();
parameters.Set(Parameters.Iterations, "70");
parameters.Set(Parameters.Cutoff, "1");
parameters.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEnt);
var cv = new TokenNameFinderCrossValidator("en", Type, parameters, new NameEvaluationErrorListener());
cv.Evaluate(sampleStream, 2);
Assert.NotNull(cv.FMeasure);
}
}
public void TestWithNameEvaluationErrorListener() {
using (var file = Tests.OpenFile("opennlp/tools/namefind/AnnotatedSentences.txt")) {
var sampleStream = new NameSampleStream(new PlainTextByLineStream(file, "ISO-8859-1"));
var parameters = new TrainingParameters();
parameters.Set(Parameters.Iterations, "70");
parameters.Set(Parameters.Cutoff, "1");
parameters.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEnt);
var cv = new TokenNameFinderCrossValidator("en", Type, parameters, new NameEvaluationErrorListener());
cv.Evaluate(sampleStream, 2);
Assert.NotNull(cv.FMeasure);
}
}
public void AbbreviationDefaultBehaviorTest()
{
var samples =
"Test E-mail met zowel letsel als 12. Toedracht in het onderwerp." + Environment.NewLine +
"Dit is een 2e regel met een tel. 011-4441444 erin." + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine + Environment.NewLine +
"Dit is een 2e regel met een tel. 033-1333123 erin!" + Environment.NewLine +
"Test E-mail met zowel winst als 12. toedracht in het onderwerp." + Environment.NewLine +
"Dit is een 2e regel!" + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine;
var stringsToIgnoreDictionary = new SharpNL.Dictionary.Dictionary(false)
{
{ "12. Toedracht" },
{ "Tel." },
};
var trainingParameters = new TrainingParameters();
trainingParameters.Set(Parameters.Algorithm, "MAXENT");
trainingParameters.Set(Parameters.TrainerType, "Event");
trainingParameters.Set(Parameters.Iterations, "100");
trainingParameters.Set(Parameters.Cutoff, "5");
char[] eos = { '.', '?', '!' };
var sdFactory = new SentenceDetectorFactory("nl", true, stringsToIgnoreDictionary, eos);
var stringReader = new StringReader(samples);
var stream = new SentenceSampleStream(new PlainTextByLineStream(stringReader));
var sentenceModel = SentenceDetectorME.Train("nl", stream, sdFactory, trainingParameters);
var sentenceDetectorMe = new SentenceDetectorME(sentenceModel);
var sentences = sentenceDetectorMe.SentDetect(samples);
var expected = samples.Split(new [] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
Assert.AreEqual(8, sentences.Length);
for (var i = 0; i < sentences.Length; i++)
{
Assert.AreEqual(expected[i], sentences[i]);
}
}
public void TestCrossCompatibility()
{
using (var data = Tests.OpenFile("/opennlp/tools/tokenize/token.train")) {
var samples = new TokenSampleStream(new PlainTextByLineStream(data));
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
var model = TokenizerME.Train(samples, new TokenizerFactory("en", null, true), mlParams);
var sMe = new TokenizerME(model);
TokenizerMETest.TestTokenizer(sMe);
var sProbs = sMe.TokenProbabilities;
// --- java \/
var sFile = Path.GetTempFileName();
model.Serialize(new FileStream(sFile, FileMode.Create));
var jModel = new opennlp.tools.tokenize.TokenizerModel(
OpenNLP.CreateInputStream(sFile)
);
var jMe = new opennlp.tools.tokenize.TokenizerME(jModel);
TestJavaTokenizer(jMe);
var jProbs = jMe.getTokenProbabilities();
Assert.AreEqual(jProbs.Length, sProbs.Length);
for (int i = 0; i < jProbs.Length; i++)
{
// one difference :(
// -0.00000000000000011102230246251565
//
// but still "insignificant" :)
Assert.AreEqual(jProbs[i], sProbs[i], 0.0000000001d);
}
}
}
public static TokenizerModel CreateSimpleMaxentTokenModel() {
var samples = new List<TokenSample> {
new TokenSample("year", new[] {new Span(0, 4)}),
new TokenSample("year,", new[] {new Span(0, 4), new Span(4, 5)}),
new TokenSample("it,", new[] {new Span(0, 2), new Span(2, 3)}),
new TokenSample("it", new[] {new Span(0, 2)}),
new TokenSample("yes", new[] {new Span(0, 3)}),
new TokenSample("yes,", new[] {new Span(0, 3), new Span(3, 4)})
};
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
return TokenizerME.Train(
new CollectionObjectStream<TokenSample>(samples),
new TokenizerFactory("en", null, true),
mlParams);
}
public void TestSentenceDetector()
{
using (var file = Tests.OpenFile("/opennlp/tools/sentdetect/Sentences.txt")) {
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
var sdFactory = new SentenceDetectorFactory("en", true, null, null);
var stream = new SentenceSampleStream(new PlainTextByLineStream(file));
var model = SentenceDetectorME.Train("en", stream, sdFactory, mlParams);
Assert.AreEqual("en", model.Language);
Assert.AreEqual(model.UseTokenEnd, true);
EvalSentences(new SentenceDetectorME(model));
}
}
internal static POSModel TrainPOSModel(ModelType type = ModelType.Maxent) {
var p = new TrainingParameters();
switch (type) {
case ModelType.Maxent:
p.Set(Parameters.Algorithm, "MAXENT");
break;
case ModelType.Perceptron:
p.Set(Parameters.Algorithm, "PERCEPTRON");
break;
default:
throw new NotSupportedException();
}
p.Set(Parameters.Iterations, "100");
p.Set(Parameters.Cutoff, "5");
return POSTaggerME.Train("en", CreateSampleStream(), p, new POSTaggerFactory());
}
public void TestSentenceDetector() {
using (var file = Tests.OpenFile("/opennlp/tools/sentdetect/Sentences.txt")) {
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
var sdFactory = new SentenceDetectorFactory("en", true, null, null);
var stream = new SentenceSampleStream(new PlainTextByLineStream(file));
var model = SentenceDetectorME.Train("en", stream, sdFactory, mlParams);
Assert.AreEqual("en", model.Language);
Assert.AreEqual(model.UseTokenEnd, true);
EvalSentences(new SentenceDetectorME(model));
}
}
public void AbbreviationDefaultBehaviorTest() {
var samples =
"Test E-mail met zowel letsel als 12. Toedracht in het onderwerp." + Environment.NewLine +
"Dit is een 2e regel met een tel. 011-4441444 erin." + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine + Environment.NewLine +
"Dit is een 2e regel met een tel. 033-1333123 erin!" + Environment.NewLine +
"Test E-mail met zowel winst als 12. toedracht in het onderwerp." + Environment.NewLine +
"Dit is een 2e regel!" + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine;
var stringsToIgnoreDictionary = new SharpNL.Dictionary.Dictionary(false) {
{"12. Toedracht"},
{"Tel."},
};
var trainingParameters = new TrainingParameters();
trainingParameters.Set(Parameters.Algorithm, "MAXENT");
trainingParameters.Set(Parameters.TrainerType, "Event");
trainingParameters.Set(Parameters.Iterations, "100");
trainingParameters.Set(Parameters.Cutoff, "5");
char[] eos = { '.', '?', '!' };
var sdFactory = new SentenceDetectorFactory("nl", true, stringsToIgnoreDictionary, eos);
var stringReader = new StringReader(samples);
var stream = new SentenceSampleStream(new PlainTextByLineStream(stringReader));
var sentenceModel = SentenceDetectorME.Train("nl", stream, sdFactory, trainingParameters);
var sentenceDetectorMe = new SentenceDetectorME(sentenceModel);
var sentences = sentenceDetectorMe.SentDetect(samples);
var expected = samples.Split(new []{ Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
Assert.AreEqual(8, sentences.Length);
for (var i = 0; i < sentences.Length; i++)
Assert.AreEqual(expected[i], sentences[i]);
}
public void TestMaxentOnPrepAttachDataWithParamsDefault() {
var reportMap = new Dictionary<string, string>();
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEnt);
var trainer = TrainerFactory.GetEventTrainer(trainParams, reportMap, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.8086159940579352d);
}
public void Setup() {
var sParams = new TrainingParameters();
sParams.Set(Parameters.Iterations, "70");
sParams.Set(Parameters.Cutoff, "1");
var jParams = new opennlp.tools.util.TrainingParameters();
jParams.put("Iterations", "70");
jParams.put("Cutoff", "1");
var sModel = ChunkerME.Train("en", ChunkerMETest.CreateSampleStream(), sParams, new ChunkerFactory());
var jModel = opennlp.tools.chunker.ChunkerME.train("en", JavaSampleStream(), jParams,
new opennlp.tools.chunker.ChunkerFactory());
Assert.NotNull(sModel);
Assert.NotNull(jModel);
sChunker = new ChunkerME(sModel);
jChunker = new opennlp.tools.chunker.ChunkerME(jModel);
}
public void TestMaxentOnPrepAttachDataWithParamsDefault()
{
var reportMap = new Dictionary <string, string>();
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEnt);
var trainer = TrainerFactory.GetEventTrainer(trainParams, reportMap, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
PrepAttachDataUtility.TestModel(model, 0.8086159940579352d);
}
public void TestQnOnPrepAttachDataWithParamsDefault()
{
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.MaxEntQn);
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
TestModel(model, 0.8115870264917059);
}
public static TokenizerModel CreateSimpleMaxentTokenModel()
{
var samples = new List <TokenSample> {
new TokenSample("year", new[] { new Span(0, 4) }),
new TokenSample("year,", new[] { new Span(0, 4), new Span(4, 5) }),
new TokenSample("it,", new[] { new Span(0, 2), new Span(2, 3) }),
new TokenSample("it", new[] { new Span(0, 2) }),
new TokenSample("yes", new[] { new Span(0, 3) }),
new TokenSample("yes,", new[] { new Span(0, 3), new Span(3, 4) })
};
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
return(TokenizerME.Train(
new CollectionObjectStream <TokenSample>(samples),
new TokenizerFactory("en", null, true),
mlParams));
}
public void TestSimpleTraining()
{
IObjectStream <DocumentSample> samples = new GenericObjectStream <DocumentSample>(new[] {
new DocumentSample("1", new[] { "a", "b", "c" }),
new DocumentSample("1", new[] { "a", "b", "c", "1", "2" }),
new DocumentSample("1", new[] { "a", "b", "c", "3", "4" }),
new DocumentSample("0", new[] { "x", "y", "z" }),
new DocumentSample("0", new[] { "x", "y", "z", "5", "6" }),
new DocumentSample("0", new[] { "x", "y", "z", "7", "8" })
});
var param = new TrainingParameters();
param.Set(Parameters.Iterations, "100");
param.Set(Parameters.Cutoff, "0");
var model = DocumentCategorizerME.Train("x-unspecified", samples, param, new DocumentCategorizerFactory());
var doccat = new DocumentCategorizerME(model);
var aProbs = doccat.Categorize("a");
Assert.AreEqual("1", doccat.GetBestCategory(aProbs));
var bProbs = doccat.Categorize("x");
Assert.AreEqual("0", doccat.GetBestCategory(bProbs));
//test to make sure sorted map's last key is cat 1 because it has the highest score.
var sortedScoreMap = doccat.SortedScoreMap("a");
// first 0
// second 1 (last)
foreach (var pair in sortedScoreMap)
{
Assert.AreEqual("0", pair.Value[0]);
break;
}
}
internal static POSModel TrainPosModel(ModelType type = ModelType.Maxent)
{
var p = new TrainingParameters();
switch (type)
{
case ModelType.Maxent:
p.Set(Parameters.Algorithm, "MAXENT");
break;
case ModelType.Perceptron:
p.Set(Parameters.Algorithm, "PERCEPTRON");
break;
default:
throw new NotSupportedException();
}
p.Set(Parameters.Iterations, "100");
p.Set(Parameters.Cutoff, "5");
return(POSTaggerME.Train("en", CreateSampleStream(), p, new POSTaggerFactory()));
}
public void Setup()
{
var sParams = new TrainingParameters();
sParams.Set(Parameters.Iterations, "70");
sParams.Set(Parameters.Cutoff, "1");
var jParams = new opennlp.tools.util.TrainingParameters();
jParams.put("Iterations", "70");
jParams.put("Cutoff", "1");
var sModel = ChunkerME.Train("en", ChunkerMETest.CreateSampleStream(), sParams, new ChunkerFactory());
var jModel = opennlp.tools.chunker.ChunkerME.train("en", JavaSampleStream(), jParams,
new opennlp.tools.chunker.ChunkerFactory());
Assert.NotNull(sModel);
Assert.NotNull(jModel);
sChunker = new ChunkerME(sModel);
jChunker = new opennlp.tools.chunker.ChunkerME(jModel);
}
/// <summary>
/// Trains a parser model with the given parameters.
/// </summary>
/// <param name="languageCode">The language code.</param>
/// <param name="samples">The data samples.</param>
/// <param name="rules">The head rules.</param>
/// <param name="iterations">The number of training iterations.</param>
/// <param name="cutoff">The min number of times a feature must be seen.</param>
/// <param name="monitor">
/// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation.
/// This argument can be a <c>null</c> value.
/// </param>
/// <returns>The trained <see cref="ParserModel"/> object.</returns>
/// <exception cref="System.NotSupportedException">Trainer type is not supported.</exception>
public static ParserModel Train(string languageCode, IObjectStream <Parse> samples, AbstractHeadRules rules, int iterations, int cutoff, Monitor monitor)
{
var param = new TrainingParameters();
param.Set("dict", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("tagger", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("tagger", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
param.Set("chunker", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("chunker", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
param.Set("check", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("check", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
param.Set("build", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("build", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
return(Train(languageCode, samples, rules, param, monitor));
}
public void TestNameFinderWithTypes()
{
using (var file = Tests.OpenFile("opennlp/tools/namefind/AnnotatedSentencesWithTypes.txt")) {
var sampleStream = new NameSampleStream(new PlainTextByLineStream(file, "ISO-8859-1"));
var param = new TrainingParameters();
param.Set(Parameters.Iterations, "70");
param.Set(Parameters.Cutoff, "1");
var model = NameFinderME.Train(
"en",
sampleStream,
param,
new TokenNameFinderFactory(null, new Dictionary <string, object>()));
var nameFinder = new NameFinderME(model);
// now test if it can detect the sample sentences
var sentence = new[] { "Alisa", "appreciated", "the", "hint", "and", "enjoyed", "a", "delicious", "traditional", "meal." };
var names = nameFinder.Find(sentence);
Assert.AreEqual(1, names.Length);
Assert.AreEqual(new Span(0, 1, "person"), names[0]);
Assert.True(HasOtherAsOutcome(model));
sentence = new[] { "Hi", "Mike", ",", "it's", "Stefanie", "Schmidt", "." };
names = nameFinder.Find(sentence);
Assert.AreEqual(2, names.Length);
Assert.AreEqual(new Span(1, 2, "person"), names[0]);
Assert.AreEqual(new Span(4, 6, "person"), names[1]);
Assert.AreEqual("person", names[0].Type);
Assert.AreEqual("person", names[1].Type);
}
}
public void TestPerceptronOnPrepAttachDataWithStepSizeDecrease() {
var trainParams = new TrainingParameters();
trainParams.Set(Parameters.Algorithm, Parameters.Algorithms.Perceptron);
trainParams.Set(Parameters.Cutoff, "1");
trainParams.Set(Parameters.Iterations, "500");
trainParams.Set(Parameters.StepSizeDecrease, "0.06");
var trainer = TrainerFactory.GetEventTrainer(trainParams, null, null);
var model = trainer.Train(PrepAttachDataUtility.CreateTrainingStream());
/*
* The java test gives an error too, soo.... for now i'll assume that is correct :P
*
* java.lang.AssertionError: expected:<0.7756870512503095> but was:<0.7766773953948998>
at org.junit.Assert.fail(Assert.java:91)
at org.junit.Assert.failNotEquals(Assert.java:645)
at org.junit.Assert.assertEquals(Assert.java:441)
at org.junit.Assert.assertEquals(Assert.java:510)
*/
//PrepAttachDataUtility.TestModel(model, 0.7756870512503095); < OpenNLP value
PrepAttachDataUtility.TestModel(model, 0.77742015350334237);
}
public void TestEverything() {
using (var file = Tests.OpenFile("/opennlp/tools/sentdetect/Sentences.txt")) {
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
var sdFactory = new SentenceDetectorFactory("en", true, null, null);
var stream = new SentenceSampleStream(new PlainTextByLineStream(file));
var model = SentenceDetectorME.Train("en", stream, sdFactory, mlParams);
Assert.AreEqual("en", model.Language);
Assert.AreEqual(model.UseTokenEnd, true);
var sMe = new SentenceDetectorME(model);
// test the SharpNL sentences
SentenceDetectorMETest.EvalSentences(sMe);
var sFile = Path.GetTempFileName();
model.Serialize(new FileStream(sFile, FileMode.Create));
var jModel2 = new JavaModel(OpenNLP.CreateInputStream(sFile));
var jMe = new JavaSDME(jModel2);
// test the Java OpenNLP sentences.
JavaEvalSentences(jMe);
// first try?! Yes! ;-)
}
}
public void TestOnlyWithNamesWithTypes() {
using (var file = Tests.OpenFile("opennlp/tools/namefind/OnlyWithNamesWithTypes.train")) {
var sampleStream = new NameSampleStream(new PlainTextByLineStream(file));
var param = new TrainingParameters();
param.Set(Parameters.Iterations, "70");
param.Set(Parameters.Cutoff, "1");
var model = NameFinderME.Train(
"en",
sampleStream,
param,
new TokenNameFinderFactory(null, new Dictionary<string, object>()));
var nameFinder = new NameFinderME(model);
// now test if it can detect the sample sentences
var sentence = WhitespaceTokenizer.Instance.Tokenize(
"Neil Abercrombie Anibal Acevedo-Vila Gary Ackerman Robert Aderholt " +
"Daniel Akaka Todd Akin Lamar Alexander Rodney Alexander");
var names = nameFinder.Find(sentence);
Assert.AreEqual(new Span(0, 2, "person"), names[0]);
Assert.AreEqual(new Span(2, 4, "person"), names[1]);
Assert.AreEqual(new Span(4, 6, "person"), names[2]);
Assert.True(!HasOtherAsOutcome(model));
}
}
public void TestOnlyWithEntitiesWithTypes() {
using (var file = Tests.OpenFile("opennlp/tools/namefind/OnlyWithEntitiesWithTypes.train")) {
var sampleStream = new NameSampleStream(new PlainTextByLineStream(file));
var param = new TrainingParameters();
param.Set(Parameters.Iterations, "70");
param.Set(Parameters.Cutoff, "1");
var model = NameFinderME.Train(
"en",
sampleStream,
param,
new TokenNameFinderFactory(null, new Dictionary<string, object>()));
var nameFinder = new NameFinderME(model);
// now test if it can detect the sample sentences
var sentence = WhitespaceTokenizer.Instance.Tokenize("NATO United States Barack Obama");
var names = nameFinder.Find(sentence);
Assert.AreEqual(new Span(0, 1, "organization"), names[0]);
Assert.AreEqual(new Span(1, 3, "location"), names[1]);
Assert.AreEqual(new Span(3, 5, "person"), names[2]);
Assert.False(HasOtherAsOutcome(model));
}
}
public void TestNameFinderWithMultipleTypes() {
using (var file = Tests.OpenFile("opennlp/tools/namefind/voa1.train")) {
var sampleStream = new NameSampleStream(new PlainTextByLineStream(file));
var param = new TrainingParameters();
param.Set(Parameters.Iterations, "70");
param.Set(Parameters.Cutoff, "1");
var model = NameFinderME.Train(
"en",
sampleStream,
param,
new TokenNameFinderFactory(null, new Dictionary<string, object>()));
var nameFinder = new NameFinderME(model);
// now test if it can detect the sample sentences
var sentence = new [] { "U", ".", "S", ".", "President", "Barack", "Obama", "has",
"arrived", "in", "South", "Korea", ",", "where", "he", "is", "expected", "to",
"show", "solidarity", "with", "the", "country", "'", "s", "president", "in",
"demanding", "North", "Korea", "move", "toward", "ending", "its", "nuclear",
"weapons", "programs", "." };
var names = nameFinder.Find(sentence);
Assert.AreEqual(4, names.Length);
Assert.AreEqual(new Span(0, 4, "location"), names[0]);
Assert.AreEqual(new Span(5, 7, "person"), names[1]);
Assert.AreEqual(new Span(10, 12, "location"), names[2]);
Assert.AreEqual(new Span(28, 30, "location"), names[3]);
/*
These asserts are not needed because the equality comparer handles the Type
assertEquals("location", names1[0].getType());
assertEquals("person", names1[1].getType());
assertEquals("location", names1[2].getType());
assertEquals("location", names1[3].getType());
*/
sentence = new[] {
"Scott", "Snyder", "is", "the", "director", "of", "the",
"Center", "for", "U", ".", "S", ".", "Korea", "Policy", "."
};
names = nameFinder.Find(sentence);
Assert.AreEqual(2, names.Length);
Assert.AreEqual(new Span(0, 2, "person"), names[0]);
Assert.AreEqual(new Span(7, 15, "organization"), names[1]);
/*
assertEquals("person", names2[0].getType());
assertEquals("organization", names2[1].getType());
*/
}
}
public void TestCrossCompatibility() {
using (var data = Tests.OpenFile("/opennlp/tools/tokenize/token.train")) {
var samples = new TokenSampleStream(new PlainTextByLineStream(data));
var mlParams = new TrainingParameters();
mlParams.Set(Parameters.Iterations, "100");
mlParams.Set(Parameters.Cutoff, "0");
var model = TokenizerME.Train(samples, new TokenizerFactory("en", null, true), mlParams);
var sMe = new TokenizerME(model);
TokenizerMETest.TestTokenizer(sMe);
var sProbs = sMe.TokenProbabilities;
// --- java \/
var sFile = Path.GetTempFileName();
model.Serialize(new FileStream(sFile, FileMode.Create));
var jModel = new opennlp.tools.tokenize.TokenizerModel(
OpenNLP.CreateInputStream(sFile)
);
var jMe = new opennlp.tools.tokenize.TokenizerME(jModel);
TestJavaTokenizer(jMe);
var jProbs = jMe.getTokenProbabilities();
Assert.AreEqual(jProbs.Length, sProbs.Length);
for (int i = 0; i < jProbs.Length; i++) {
// one difference :(
// -0.00000000000000011102230246251565
//
// but still "insignificant" :)
Assert.AreEqual(jProbs[i], sProbs[i], 0.0000000001d);
}
}
}
/// <summary>
/// Trains a parser model with the given parameters.
/// </summary>
/// <param name="languageCode">The language code.</param>
/// <param name="samples">The data samples.</param>
/// <param name="rules">The head rules.</param>
/// <param name="iterations">The number of training iterations.</param>
/// <param name="cutoff">The min number of times a feature must be seen.</param>
/// <param name="monitor">
/// A evaluation monitor that can be used to listen the messages during the training or it can cancel the training operation.
/// This argument can be a <c>null</c> value.
/// </param>
/// <returns>The trained <see cref="ParserModel"/> object.</returns>
/// <exception cref="System.NotSupportedException">Trainer type is not supported.</exception>
public static ParserModel Train(string languageCode, IObjectStream<Parse> samples, AbstractHeadRules rules, int iterations, int cutoff, Monitor monitor) {
var param = new TrainingParameters();
param.Set("dict", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("tagger", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("tagger", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
param.Set("chunker", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("chunker", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
param.Set("check", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("check", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
param.Set("build", Parameters.Cutoff, cutoff.ToString(CultureInfo.InvariantCulture));
param.Set("build", Parameters.Iterations, iterations.ToString(CultureInfo.InvariantCulture));
return Train(languageCode, samples, rules, param, monitor);
}
public void TestCustomTrainer()
{
mlParams.Set(Parameters.Algorithm, "Dummy");
Assert.True(TrainerFactory.IsValid(mlParams));
}