private static void Tokenize_IncompleteSentence_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings("This is an incomplete sentence ").ToArray(),
new[] { "This is an incomplete sentence " });
}
public void Tokenize_CompleteSentenceWithSpaceAtEnd_ReturnsTokens()
{
var tokenizer = new LatinSentenceTokenizer();
Assert.That(tokenizer.TokenizeToStrings("\"This is a complete sentence.\" \n"),
Is.EqualTo(new[] { "\"This is a complete sentence.\"" }));
}
public void Tokenize_Abbreviation_ReturnsTokens()
{
var tokenizer = new LatinSentenceTokenizer(new[] { "mr", "dr", "ms" });
Assert.That(tokenizer.TokenizeToStrings("Mr. Smith went to Washington. This is the second sentence."),
Is.EqualTo(new[] { "Mr. Smith went to Washington.", "This is the second sentence." }));
}
private static void Tokenize_CompleteSentenceWithSpaceAtEnd_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings("\"This is a complete sentence.\" \n").ToArray(),
new[] { "\"This is a complete sentence.\"" });
}
public void Tokenize_IncompleteSentence_ReturnsTokens()
{
var tokenizer = new LatinSentenceTokenizer();
Assert.That(tokenizer.TokenizeToStrings("This is an incomplete sentence "),
Is.EqualTo(new[] { "This is an incomplete sentence " }));
}
public void Tokenize_QuotationInSentence_ReturnsTokens()
{
var tokenizer = new LatinSentenceTokenizer();
Assert.That(tokenizer.TokenizeToStrings("\"This is the first sentence!\" he said. This is the second sentence."),
Is.EqualTo(new[] { "\"This is the first sentence!\" he said.", "This is the second sentence." }));
}
public void Tokenize_Parens_ReturnsTokens()
{
var tokenizer = new LatinSentenceTokenizer();
Assert.That(tokenizer.TokenizeToStrings("This is the first sentence. (This is the second sentence.)"),
Is.EqualTo(new[] { "This is the first sentence.", "(This is the second sentence.)" }));
}
private static void Tokenize_Parens_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings("This is sentence one. (This is sentence two.)").ToArray(),
new[] { "This is sentence one.", "(This is sentence two.)" });
}
public void Tokenize_MultipleLines_ReturnsTokens()
{
var tokenizer = new LatinSentenceTokenizer();
Assert.That(tokenizer.TokenizeToStrings("This is the first sentence.\nThis is the second sentence."),
Is.EqualTo(new[] { "This is the first sentence.", "This is the second sentence." }));
}
private static void Tokenize_Abbreviation_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer(new[] { "mr", "dr", "ms" });
assert.DeepEqual(tokenizer.TokenizeToStrings(
"Mr. Smith went to Washington. This is sentence two.").ToArray(),
new[] { "Mr. Smith went to Washington.", "This is sentence two." });
}
private static void Tokenize_QuotationInSentence_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings(
"\"This is sentence one!\" he said. This is sentence two.").ToArray(),
new[] { "\"This is sentence one!\" he said.", "This is sentence two." });
}
private static void Tokenize_Quotes_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings(
"\"This is the first sentence.\" This is sentence two.").ToArray(),
new[] { "\"This is the first sentence.\"", "This is sentence two." });
}
private static void Tokenize_TwoSentences_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings(
"This is the first sentence. This is the second sentence.").ToArray(),
new[] { "This is the first sentence.", "This is the second sentence." });
}
public void Tokenize_Empty_ReturnsEmpty()
{
var tokenizer = new LatinSentenceTokenizer();
Assert.That(tokenizer.TokenizeToStrings(""), Is.Empty);
}
public void Tokenize_SingleLine_ReturnsTokens()
{
var tokenizer = new LatinSentenceTokenizer();
Assert.That(tokenizer.TokenizeToStrings("This is a test."), Is.EqualTo(new[] { "This is a test." }));
}
private static void Tokenize_Empty_ReturnsEmpty(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings("").ToArray(), new string[0]);
}
private static void Tokenize_SingleLine_ReturnsTokens(Assert assert)
{
var tokenizer = new LatinSentenceTokenizer();
assert.DeepEqual(tokenizer.TokenizeToStrings("This is a test.").ToArray(), new[] { "This is a test." });
}
