public WordAndPuncts ( string text ) : List |
||
text | string | |
Résultat | List |
public void WordAndPuncts_initialSpace() { CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", ""); IEnumerable<WordAndPunct> words = cat.WordAndPuncts(" Dude "); IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator(); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "Dude", " ", 1); Assert.IsFalse(wordCollection.MoveNext()); }
public void WordAndPuncts_initialSpace() { CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", ""); IEnumerable <WordAndPunct> words = cat.WordAndPuncts(" Dude "); IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator(); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "Dude", " ", 1); Assert.IsFalse(wordCollection.MoveNext()); }
public void WordAndPuncts_numberInWord() { CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", ""); IEnumerable<WordAndPunct> words = cat.WordAndPuncts("This is test1."); IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator(); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "This", " ", 0); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "is", " ", 5); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "test1", ".", 8); Assert.IsFalse(wordCollection.MoveNext()); }
public void WordAndPuncts_numberInWord() { CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", ""); IEnumerable <WordAndPunct> words = cat.WordAndPuncts("This is test1."); IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator(); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "This", " ", 0); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "is", " ", 5); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "test1", ".", 8); Assert.IsFalse(wordCollection.MoveNext()); }
public void WordAndPuncts_initialSpaceFollowedByNumbers() { CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", ""); IEnumerable <WordAndPunct> words = cat.WordAndPuncts("1 2 3"); IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator(); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "1", " ", 0); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "2", " ", 2); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "3", "", 4); Assert.IsFalse(wordCollection.MoveNext()); }
public void WordAndPuncts_simple() { CharacterCategorizer cat = new CharacterCategorizer("", "", ""); IEnumerable<WordAndPunct> words = cat.WordAndPuncts("This is my test."); using (IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator()) { Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "This", " ", 0); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "is", " ", 5); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "my", " ", 8); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "test", ".", 11); Assert.IsFalse(wordCollection.MoveNext()); } }
public void WordAndPuncts_simple() { CharacterCategorizer cat = new CharacterCategorizer("", "", ""); IEnumerable <WordAndPunct> words = cat.WordAndPuncts("This is my test."); using (IEnumerator <WordAndPunct> wordCollection = words.GetEnumerator()) { Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "This", " ", 0); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "is", " ", 5); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "my", " ", 8); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "test", ".", 11); Assert.IsFalse(wordCollection.MoveNext()); } }
/// ------------------------------------------------------------------------------------ /// <summary> /// Get all instances of the item being checked in the token list passed. /// This includes both valid and invalid instances. /// This is used 1) to create an inventory of these items. /// To show the user all instance of an item with a specified key. /// 2) With a "desiredKey" in order to fetch instance of a specific /// item (e.g. all the places where "the" is a repeated word. /// </summary> /// <param name="tokens">Tokens for text to be scanned</param> /// <param name="desiredKey">If you only want instance of a specific key (e.g. one word, /// one punctuation pattern, one character, etc.) place it here. Empty string returns /// all items.</param> /// <returns>List of token substrings</returns> /// ------------------------------------------------------------------------------------ public List<TextTokenSubstring> GetReferences(IEnumerable<ITextToken> tokens, string desiredKey) { #if DEBUG List<ITextToken> AllTokens = new List<ITextToken>(tokens); #endif m_characterCategorizer = m_checksDataSource.CharacterCategorizer; ValidItems = m_checksDataSource.GetParameterValue(kValidItemsParameter); InvalidItems = m_checksDataSource.GetParameterValue(kInvalidItemsParameter); string preferredLocale = m_checksDataSource.GetParameterValue("PreferredLocale") ?? string.Empty; m_mixedCapitalization = new List<TextTokenSubstring>(); ProcessMixedCapitalization processor = new ProcessMixedCapitalization(m_checksDataSource, m_mixedCapitalization); foreach (ITextToken tok in tokens) { if ((tok.Locale ?? string.Empty) != preferredLocale) continue; foreach (WordAndPunct wap in m_characterCategorizer.WordAndPuncts(tok.Text)) processor.ProcessWord(tok, wap, desiredKey); } return m_mixedCapitalization; }
public void WordAndPuncts_initialSpaceFollowedByNumbers() { CharacterCategorizer cat = new CharacterCategorizer("", "", "", "", ""); IEnumerable<WordAndPunct> words = cat.WordAndPuncts("1 2 3"); IEnumerator<WordAndPunct> wordCollection = words.GetEnumerator(); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "1", " ", 0); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "2", " ", 2); Assert.IsTrue(wordCollection.MoveNext()); CheckWordAndPunct(wordCollection.Current, "3", "", 4); Assert.IsFalse(wordCollection.MoveNext()); }