public void ProcessToken(ITextToken tok)
{
foreach (WordAndPunct wap in characterCategorizer.WordAndPuncts(tok.Text))
{
ProcessWord(tok, wap);
}
}
/// ------------------------------------------------------------------------------------
/// <summary>
/// Get all instances of the item being checked in the token list passed.
/// This includes both valid and invalid instances.
/// This is used 1) to create an inventory of these items.
/// To show the user all instance of an item with a specified key.
/// 2) With a "desiredKey" in order to fetch instance of a specific
/// item (e.g. all the places where "the" is a repeated word.
/// </summary>
/// <param name="tokens">Tokens for text to be scanned</param>
/// <param name="desiredKey">If you only want instance of a specific key (e.g. one word,
/// one punctuation pattern, one character, etc.) place it here. Empty string returns
/// all items.</param>
/// <returns>List of token substrings</returns>
/// ------------------------------------------------------------------------------------
public List <TextTokenSubstring> GetReferences(IEnumerable <ITextToken> tokens, string desiredKey)
{
#if DEBUG
List <ITextToken> AllTokens = new List <ITextToken>(tokens);
if (AllTokens.Count == 0)
{
// Keep the compiler from complaining about assigning to a variable, but not using it.
}
#endif
m_characterCategorizer = m_checksDataSource.CharacterCategorizer;
ValidItems = m_checksDataSource.GetParameterValue(kValidItemsParameter);
InvalidItems = m_checksDataSource.GetParameterValue(kInvalidItemsParameter);
string preferredLocale =
m_checksDataSource.GetParameterValue("PreferredLocale") ?? string.Empty;
m_mixedCapitalization = new List <TextTokenSubstring>();
ProcessMixedCapitalization processor =
new ProcessMixedCapitalization(m_checksDataSource, m_mixedCapitalization);
foreach (ITextToken tok in tokens)
{
if ((tok.Locale ?? string.Empty) != preferredLocale)
{
continue;
}
foreach (WordAndPunct wap in m_characterCategorizer.WordAndPuncts(tok.Text))
{
processor.ProcessWord(tok, wap, desiredKey);
}
}
return(m_mixedCapitalization);
}