public void AbbreviationDefaultBehaviorTest() {
var samples =
"Test E-mail met zowel letsel als 12. Toedracht in het onderwerp." + Environment.NewLine +
"Dit is een 2e regel met een tel. 011-4441444 erin." + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine + Environment.NewLine +
"Dit is een 2e regel met een tel. 033-1333123 erin!" + Environment.NewLine +
"Test E-mail met zowel winst als 12. toedracht in het onderwerp." + Environment.NewLine +
"Dit is een 2e regel!" + Environment.NewLine +
"Dit is een 2e regel." + Environment.NewLine;
var stringsToIgnoreDictionary = new SharpNL.Dictionary.Dictionary(false) {
{"12. Toedracht"},
{"Tel."},
};
var trainingParameters = new TrainingParameters();
trainingParameters.Set(Parameters.Algorithm, "MAXENT");
trainingParameters.Set(Parameters.TrainerType, "Event");
trainingParameters.Set(Parameters.Iterations, "100");
trainingParameters.Set(Parameters.Cutoff, "5");
char[] eos = { '.', '?', '!' };
var sdFactory = new SentenceDetectorFactory("nl", true, stringsToIgnoreDictionary, eos);
var stringReader = new StringReader(samples);
var stream = new SentenceSampleStream(new PlainTextByLineStream(stringReader));
var sentenceModel = SentenceDetectorME.Train("nl", stream, sdFactory, trainingParameters);
var sentenceDetectorMe = new SentenceDetectorME(sentenceModel);
var sentences = sentenceDetectorMe.SentDetect(samples);
var expected = samples.Split(new []{ Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
Assert.AreEqual(8, sentences.Length);
for (var i = 0; i < sentences.Length; i++)
Assert.AreEqual(expected[i], sentences[i]);
}
internal static void EvalSentences(SentenceDetectorME sentDetect) {
const string sampleSentences1 = "This is a test. There are many tests, this is the second.";
var sents = sentDetect.SentDetect(sampleSentences1);
Assert.AreEqual(sents.Length, 2);
Assert.AreEqual(sents[0], "This is a test.");
Assert.AreEqual(sents[1], "There are many tests, this is the second.");
var probs = sentDetect.GetSentenceProbabilities();
Assert.AreEqual(probs.Length, 2);
const string sampleSentences2 = "This is a test. There are many tests, this is the second";
sents = sentDetect.SentDetect(sampleSentences2);
Assert.AreEqual(sents.Length, 2);
probs = sentDetect.GetSentenceProbabilities();
Assert.AreEqual(probs.Length, 2);
Assert.AreEqual(sents[0], "This is a test.");
Assert.AreEqual(sents[1], "There are many tests, this is the second");
const string sampleSentences3 = "This is a \"test\". He said \"There are many tests, this is the second.\"";
sents = sentDetect.SentDetect(sampleSentences3);
Assert.AreEqual(sents.Length, 2);
probs = sentDetect.GetSentenceProbabilities();
Assert.AreEqual(probs.Length, 2);
Assert.AreEqual(sents[0], "This is a \"test\".");
Assert.AreEqual(sents[1], "He said \"There are many tests, this is the second.\"");
const string sampleSentences4 = "This is a \"test\". I said \"This is a test.\" Any questions?";
sents = sentDetect.SentDetect(sampleSentences4);
Assert.AreEqual(sents.Length, 3);
probs = sentDetect.GetSentenceProbabilities();
Assert.AreEqual(probs.Length, 3);
Assert.AreEqual(sents[0], "This is a \"test\".");
Assert.AreEqual(sents[1], "I said \"This is a test.\"");
Assert.AreEqual(sents[2], "Any questions?");
const string sampleSentences5 = "This is a one sentence test space at the end. ";
sents = sentDetect.SentDetect(sampleSentences5);
Assert.AreEqual(1, sentDetect.GetSentenceProbabilities().Length);
Assert.AreEqual(sents[0], "This is a one sentence test space at the end.");
const string sampleSentences6 = "This is a one sentences test with tab at the end. ";
sents = sentDetect.SentDetect(sampleSentences6);
Assert.AreEqual(sents[0], "This is a one sentences test with tab at the end.");
const string sampleSentences7 = "This is a test. With spaces between the two sentences.";
sents = sentDetect.SentDetect(sampleSentences7);
Assert.AreEqual(sents[0], "This is a test.");
Assert.AreEqual(sents[1], "With spaces between the two sentences.");
const string sampleSentences9 = "";
sents = sentDetect.SentDetect(sampleSentences9);
Assert.AreEqual(0, sents.Length);
const string sampleSentences10 = " "; // whitespaces and tabs
sents = sentDetect.SentDetect(sampleSentences10);
Assert.AreEqual(0, sents.Length);
const string sampleSentences11 = "This is test sentence without a dot at the end and spaces ";
sents = sentDetect.SentDetect(sampleSentences11);
Assert.AreEqual(sents[0], "This is test sentence without a dot at the end and spaces");
probs = sentDetect.GetSentenceProbabilities();
Assert.AreEqual(1, probs.Length);
const string sampleSentence12 = " This is a test.";
sents = sentDetect.SentDetect(sampleSentence12);
Assert.AreEqual(sents[0], "This is a test.");
const string sampleSentence13 = " This is a test";
sents = sentDetect.SentDetect(sampleSentence13);
Assert.AreEqual(sents[0], "This is a test");
// Test that sentPosDetect also works
var pos = sentDetect.SentPosDetect(sampleSentences2);
Assert.AreEqual(pos.Length, 2);
probs = sentDetect.GetSentenceProbabilities();
Assert.AreEqual(probs.Length, 2);
Assert.AreEqual(new Span(0, 15), pos[0]);
Assert.AreEqual(new Span(16, 56), pos[1]);
}
