public void TestEmptyString()
{
NLPSentenceDetectorOp sentenceDetectorOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
BreakIterator bi = new OpenNLPSentenceBreakIterator(sentenceDetectorOp);
bi.SetText("");
Test0Sentences(bi);
}
public void TestWhitespaceOnly()
{
NLPSentenceDetectorOp sentenceDetectorOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
BreakIterator bi = new OpenNLPSentenceBreakIterator(sentenceDetectorOp);
bi.SetText(" \n \n\n\r\n\t \n");
Test0Sentences(bi);
}
public void TestSliceStart()
{
NLPSentenceDetectorOp sentenceDetectorOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
BreakIterator bi = new OpenNLPSentenceBreakIterator(sentenceDetectorOp);
bi.SetText(GetCharArrayIterator(PADDING + SENTENCES[0], PADDING.Length, SENTENCES[0].Length));
Test1Sentence(bi, SENTENCES[0]);
}
public void TestSingleSentence()
{
NLPSentenceDetectorOp sentenceDetectorOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
BreakIterator bi = new OpenNLPSentenceBreakIterator(sentenceDetectorOp);
bi.SetText(GetCharArrayIterator(SENTENCES[0]));
Test1Sentence(bi, SENTENCES[0]);
}
public void TestFirstPosition()
{
NLPSentenceDetectorOp sentenceDetectorOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
BreakIterator bi = new OpenNLPSentenceBreakIterator(sentenceDetectorOp);
bi.SetText(GetCharArrayIterator(SENTENCES[0]));
assertEquals(SENTENCES[0].Length, bi.Last()); // side-effect: set current position to last()
Test1Sentence(bi, SENTENCES[0]);
}
public void TestThreeSentences()
{
NLPSentenceDetectorOp sentenceDetectorOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
BreakIterator bi = new OpenNLPSentenceBreakIterator(sentenceDetectorOp);
bi.SetText(TEXT); // String is converted to StringCharacterIterator
Do3SentenceTest(bi);
bi.SetText(GetCharArrayIterator(TEXT));
Do3SentenceTest(bi);
}
public override Tokenizer Create(AttributeFactory factory, TextReader reader)
{
try
{
NLPSentenceDetectorOp sentenceOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
NLPTokenizerOp tokenizerOp = OpenNLPOpsFactory.GetTokenizer(tokenizerModelFile);
return(new OpenNLPTokenizer(factory, reader, sentenceOp, tokenizerOp));
}
catch (IOException e)
{
throw new Exception(e.ToString(), e);
}
}
public override Tokenizer Create(AttributeFactory factory, TextReader reader)
{
try
{
NLPSentenceDetectorOp sentenceOp = OpenNLPOpsFactory.GetSentenceDetector(sentenceModelFile);
NLPTokenizerOp tokenizerOp = OpenNLPOpsFactory.GetTokenizer(tokenizerModelFile);
return(new OpenNLPTokenizer(factory, reader, sentenceOp, tokenizerOp));
}
catch (Exception e) when(e.IsIOException())
{
throw RuntimeException.Create(e);
}
}