public void WordAndPuncts_initialSpace()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", "");
IEnumerable<WordAndPunct> words = cat.WordAndPuncts(" Dude ");
IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator();
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "Dude", " ", 1);
Assert.IsFalse(wordCollection.MoveNext());
}
public void WordAndPuncts_initialSpace()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", "");
IEnumerable <WordAndPunct> words = cat.WordAndPuncts(" Dude ");
IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator();
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "Dude", " ", 1);
Assert.IsFalse(wordCollection.MoveNext());
}
public void WordAndPuncts_numberInWord()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", "");
IEnumerable<WordAndPunct> words = cat.WordAndPuncts("This is test1.");
IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator();
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "This", " ", 0);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "is", " ", 5);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "test1", ".", 8);
Assert.IsFalse(wordCollection.MoveNext());
}
public void WordAndPuncts_numberInWord()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", "");
IEnumerable <WordAndPunct> words = cat.WordAndPuncts("This is test1.");
IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator();
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "This", " ", 0);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "is", " ", 5);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "test1", ".", 8);
Assert.IsFalse(wordCollection.MoveNext());
}
public void WordAndPuncts_initialSpaceFollowedByNumbers()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", "");
IEnumerable <WordAndPunct> words = cat.WordAndPuncts("1 2 3");
IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator();
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "1", " ", 0);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "2", " ", 2);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "3", "", 4);
Assert.IsFalse(wordCollection.MoveNext());
}
public void WordAndPuncts_simple()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "");
IEnumerable<WordAndPunct> words = cat.WordAndPuncts("This is my test.");
using (IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator())
{
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "This", " ", 0);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "is", " ", 5);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "my", " ", 8);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "test", ".", 11);
Assert.IsFalse(wordCollection.MoveNext());
}
}
public void WordAndPuncts_simple()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "");
IEnumerable <WordAndPunct> words = cat.WordAndPuncts("This is my test.");
using (IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator())
{
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "This", " ", 0);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "is", " ", 5);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "my", " ", 8);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "test", ".", 11);
Assert.IsFalse(wordCollection.MoveNext());
}
}
/// ------------------------------------------------------------------------------------
/// <summary>
/// Get all instances of the item being checked in the token list passed.
/// This includes both valid and invalid instances.
/// This is used 1) to create an inventory of these items.
/// To show the user all instance of an item with a specified key.
/// 2) With a "desiredKey" in order to fetch instance of a specific
/// item (e.g. all the places where "the" is a repeated word.
/// </summary>
/// <param name="tokens">Tokens for text to be scanned</param>
/// <param name="desiredKey">If you only want instance of a specific key (e.g. one word,
/// one punctuation pattern, one character, etc.) place it here. Empty string returns
/// all items.</param>
/// <returns>List of token substrings</returns>
/// ------------------------------------------------------------------------------------
public List<TextTokenSubstring> GetReferences(IEnumerable<ITextToken> tokens, string desiredKey)
{
#if DEBUG
List<ITextToken> AllTokens = new List<ITextToken>(tokens);
#endif
m_characterCategorizer = m_checksDataSource.CharacterCategorizer;
ValidItems = m_checksDataSource.GetParameterValue(kValidItemsParameter);
InvalidItems = m_checksDataSource.GetParameterValue(kInvalidItemsParameter);
string preferredLocale =
m_checksDataSource.GetParameterValue("PreferredLocale") ?? string.Empty;
m_mixedCapitalization = new List<TextTokenSubstring>();
ProcessMixedCapitalization processor =
new ProcessMixedCapitalization(m_checksDataSource, m_mixedCapitalization);
foreach (ITextToken tok in tokens)
{
if ((tok.Locale ?? string.Empty) != preferredLocale)
continue;
foreach (WordAndPunct wap in m_characterCategorizer.WordAndPuncts(tok.Text))
processor.ProcessWord(tok, wap, desiredKey);
}
return m_mixedCapitalization;
}
public void WordAndPuncts_initialSpaceFollowedByNumbers()
{
CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", "");
IEnumerable<WordAndPunct> words = cat.WordAndPuncts("1 2 3");
IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator();
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "1", " ", 0);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "2", " ", 2);
Assert.IsTrue(wordCollection.MoveNext());
CheckWordAndPunct(wordCollection.Current, "3", "", 4);
Assert.IsFalse(wordCollection.MoveNext());
}