public void BackwardsTest10MaxLength2() { WordTokenizer tokenizer = new WordTokenizer(message); WordTokenCollection tokens = tokenizer.Backwards(10, 2); Assert.AreEqual("morn", tokens [0].Message, "morn"); Assert.AreEqual(" ", tokens [1].Message, "3 spaces."); Assert.AreEqual(2, tokens.Count, "Count"); }
public void ForwardMissingTest() { string localMessage = "one, two, three, four"; WordTokenizer tokenizer = new WordTokenizer(localMessage); WordTokenCollection tokens = tokenizer.Forward(0, 10); Assert.AreEqual(7, tokens.Count, "Count"); Assert.AreEqual(tokens [0].Message, localMessage.Substring(tokens [0].Index, tokens [0].Message.Length), "one,"); Assert.AreEqual("one,", tokens [0].Message, "one,"); Assert.AreEqual(tokens [1].Message, localMessage.Substring(tokens [1].Index, tokens [1].Message.Length), "1 space"); Assert.AreEqual(" ", tokens [1].Message, "1 space"); Assert.AreEqual(tokens [2].Message, localMessage.Substring(tokens [2].Index, tokens [2].Message.Length), "two,"); Assert.AreEqual("two,", tokens [2].Message, "two,"); Assert.AreEqual(tokens [3].Message, localMessage.Substring(tokens [3].Index, tokens [3].Message.Length), "2 spaces"); Assert.AreEqual(" ", tokens [3].Message, "2 spaces,"); Assert.AreEqual(tokens [4].Message, localMessage.Substring(tokens [4].Index, tokens [4].Message.Length), "three,"); Assert.AreEqual("three,", tokens [4].Message, "three,"); Assert.AreEqual(tokens [5].Message, localMessage.Substring(tokens [5].Index, tokens [5].Message.Length), "3 spaces"); Assert.AreEqual(" ", tokens [5].Message, "3 spaces"); Assert.AreEqual(tokens [6].Message, localMessage.Substring(tokens [6].Index, tokens [6].Message.Length), "four"); Assert.AreEqual("four", tokens [6].Message, "four"); tokens = tokenizer.Forward(localMessage.Length); Assert.AreEqual(0, tokens.Count, "Count"); }
private bool AddSpaces(WordTokenCollection tokens, int index, ref bool wasSpace, ref int spaces) { bool previousValue = wasSpace; if (wasSpace) { tokens.Add(new WordToken(new String(' ', spaces), index - spaces)); spaces = 1; wasSpace = false; } else if (tokens.Count > 0) { tokens.Add(new WordToken(" ", index - spaces)); } return(previousValue); }
public void BackwardsTestEmptyWord() { string localMessage = "One morning, when Gregor Samsa "; WordTokenizer tokenizer = new WordTokenizer(localMessage); WordTokenCollection tokens = tokenizer.Backwards(localMessage.Length - 1, 2); Assert.AreEqual(tokens [0].Message, localMessage.Substring(tokens [0].Index, tokens [0].Message.Length), "3 spaces"); Assert.AreEqual(" ", tokens [0].Message, "3 spaces"); Assert.AreEqual(tokens [1].Message, localMessage.Substring(tokens [1].Index, tokens [1].Message.Length), "Samsa"); Assert.AreEqual("Samsa", tokens [1].Message, "Samsa"); Assert.AreEqual(2, tokens.Count, "Count"); }
public void ForwardIndexTest() { string localMessage = "one, two, three, four "; WordTokenizer tokenizer = new WordTokenizer(localMessage); WordTokenCollection tokens = tokenizer.Forward(4); Assert.AreEqual(7, tokens.Count, "Count"); tokens = tokenizer.Forward(4, 2); Assert.AreEqual(2, tokens.Count, "Count"); tokens = tokenizer.Forward(0); Assert.AreEqual(tokens [tokens.Count - 1].Message, localMessage.Substring(tokens [tokens.Count - 1].Index, tokens [tokens.Count - 1].Message.Length), "Empty spaces"); }
public void ForwardTest10() { WordTokenizer tokenizer = new WordTokenizer(message); WordTokenCollection tokens = tokenizer.Forward(10); Assert.AreEqual(tokens [0].Message, message.Substring(tokens [0].Index, tokens [0].Message.Length), "ning,"); Assert.AreEqual("ning,", tokens [0].Message, "ning,"); Assert.AreEqual(tokens [1].Message, message.Substring(tokens [1].Index, tokens [1].Message.Length), "4 spaces"); Assert.AreEqual(" ", tokens [1].Message, "4 spaces"); Assert.AreEqual(tokens [2].Message, message.Substring(tokens [2].Index, tokens [2].Message.Length), "when"); Assert.AreEqual("when", tokens [2].Message, "when"); }
public void BackwardsTest10() { WordTokenizer tokenizer = new WordTokenizer(message); WordTokenCollection tokens = tokenizer.Backwards(10); Assert.AreEqual(tokens [0].Message, message.Substring(tokens [0].Index, tokens [0].Message.Length), "morn"); Assert.AreEqual("morn", tokens [0].Message, "morn"); Assert.AreEqual(" ", tokens [1].Message, "3 spaces."); Assert.AreEqual(tokens [1].Message, message.Substring(tokens [1].Index, tokens [1].Message.Length), "4 spaces"); Assert.AreEqual("One", tokens [2].Message, "One"); Assert.AreEqual(tokens [2].Message, message.Substring(tokens [2].Index, tokens [2].Message.Length), "One"); }
public void ForwardTest() { WordTokenizer tokenizer = new WordTokenizer(message); WordTokenCollection tokens = tokenizer.Forward(0); Assert.AreEqual(tokens [0].Message, message.Substring(tokens [0].Index, tokens [0].Message.Length), "One"); Assert.AreEqual(tokens [1].Message, message.Substring(tokens [1].Index, tokens [1].Message.Length), "3 spaces"); Assert.AreEqual(tokens [2].Message, message.Substring(tokens [2].Index, tokens [2].Message.Length), "morning"); Assert.AreEqual(tokens [3].Message, message.Substring(tokens [3].Index, tokens [3].Message.Length), "4 spaces"); Assert.AreEqual(tokens [4].Message, message.Substring(tokens [4].Index, tokens [4].Message.Length), "when"); Assert.AreEqual(tokens [5].Message, message.Substring(tokens [5].Index, tokens [5].Message.Length), "1 space"); Assert.AreEqual(tokens [6].Message, message.Substring(tokens [6].Index, tokens [6].Message.Length), "Gregor"); Assert.AreEqual(tokens [7].Message, message.Substring(tokens [7].Index, tokens [7].Message.Length), "1 space"); Assert.AreEqual(tokens [8].Message, message.Substring(tokens [8].Index, tokens [8].Message.Length), "Samsa"); Assert.AreEqual(tokens [9].Message, message.Substring(tokens [9].Index, tokens [9].Message.Length), "1 space"); Assert.AreEqual(tokens [10].Message, message.Substring(tokens [10].Index, tokens [10].Message.Length), "woke"); Assert.AreEqual(tokens [11].Message, message.Substring(tokens [11].Index, tokens [11].Message.Length), "1 space"); Assert.AreEqual(tokens [12].Message, message.Substring(tokens [12].Index, tokens [12].Message.Length), "from"); Assert.AreEqual(tokens [13].Message, message.Substring(tokens [13].Index, tokens [13].Message.Length), "1 space"); Assert.AreEqual(tokens [14].Message, message.Substring(tokens [14].Index, tokens [14].Message.Length), "troubled"); Assert.AreEqual(tokens [15].Message, message.Substring(tokens [15].Index, tokens [15].Message.Length), "1 space"); Assert.AreEqual(tokens [16].Message, message.Substring(tokens [16].Index, tokens [16].Message.Length), "dreams,"); Assert.AreEqual(tokens [17].Message, message.Substring(tokens [17].Index, tokens [17].Message.Length), "1 space"); Assert.AreEqual(tokens [18].Message, message.Substring(tokens [18].Index, tokens [18].Message.Length), "he"); Assert.AreEqual(tokens [19].Message, message.Substring(tokens [19].Index, tokens [19].Message.Length), "1 space"); Assert.AreEqual(tokens [20].Message, message.Substring(tokens [20].Index, tokens [20].Message.Length), "found"); Assert.AreEqual(tokens [21].Message, message.Substring(tokens [21].Index, tokens [21].Message.Length), "1 space"); Assert.AreEqual(tokens [22].Message, message.Substring(tokens [22].Index, tokens [22].Message.Length), "himself"); Assert.AreEqual(tokens [23].Message, message.Substring(tokens [23].Index, tokens [23].Message.Length), "1 space"); Assert.AreEqual(tokens [24].Message, message.Substring(tokens [24].Index, tokens [24].Message.Length), "transformed"); Assert.AreEqual(tokens [25].Message, message.Substring(tokens [25].Index, tokens [25].Message.Length), "1 space"); Assert.AreEqual(tokens [26].Message, message.Substring(tokens [26].Index, tokens [26].Message.Length), "in"); Assert.AreEqual(tokens [27].Message, message.Substring(tokens [27].Index, tokens [27].Message.Length), "1 space"); Assert.AreEqual(tokens [28].Message, message.Substring(tokens [28].Index, tokens [28].Message.Length), "his"); Assert.AreEqual(tokens [29].Message, message.Substring(tokens [29].Index, tokens [29].Message.Length), "1 space"); Assert.AreEqual(tokens [30].Message, message.Substring(tokens [30].Index, tokens [30].Message.Length), "bed"); Assert.AreEqual(tokens [31].Message, message.Substring(tokens [31].Index, tokens [31].Message.Length), "1 space"); Assert.AreEqual(tokens [32].Message, message.Substring(tokens [32].Index, tokens [32].Message.Length), "into"); Assert.AreEqual(tokens [33].Message, message.Substring(tokens [33].Index, tokens [33].Message.Length), "1 space"); Assert.AreEqual(tokens [34].Message, message.Substring(tokens [34].Index, tokens [34].Message.Length), "a"); Assert.AreEqual(tokens [35].Message, message.Substring(tokens [35].Index, tokens [35].Message.Length), "1 space"); Assert.AreEqual(tokens [36].Message, message.Substring(tokens [36].Index, tokens [36].Message.Length), "horrible"); Assert.AreEqual(tokens [37].Message, message.Substring(tokens [37].Index, tokens [37].Message.Length), "1 space"); Assert.AreEqual(tokens [38].Message, message.Substring(tokens [38].Index, tokens [38].Message.Length), "vermin."); Assert.AreEqual(tokens [39].Message, message.Substring(tokens [39].Index, tokens [39].Message.Length), "3 spaces"); }
public WordTokenCollection Backwards(int index, int maxLength) { WordTokenCollection tokens = new WordTokenCollection(); int newIndex = 0; int oldIndex = 0; char separator = ' '; bool wasSpace = false; int spacesCounter = 1; if (index <= 0) { return(tokens); } do { if (index == -1) //We have reached last found space { if (AddSpaces(tokens, newIndex + spacesCounter, ref wasSpace, ref spacesCounter) == false) { if (tokens.Count == maxLength) { break; } tokens.Add(new WordToken(message.Substring(newIndex + 1, index - newIndex), newIndex - 1)); } break; } newIndex = message.LastIndexOf(separator, index, index + 1); if (newIndex == -1) { AddSpaces(tokens, oldIndex + spacesCounter, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) { break; } tokens.Add(new WordToken(message.Substring(0, index + 1), 0)); break; } if (oldIndex - newIndex == 1) //Previous index is also space { wasSpace = true; oldIndex = newIndex; index = newIndex - 1; spacesCounter++; continue; } else if (newIndex - index == 0) //First index is space { wasSpace = true; oldIndex = newIndex; index = newIndex - 1; continue; } AddSpaces(tokens, oldIndex + spacesCounter, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) { break; } tokens.Add(new WordToken(message.Substring(newIndex + 1, index - newIndex), newIndex + 1)); if (tokens.Count == maxLength) { break; } oldIndex = newIndex; index = newIndex - 1; } while (true); return(tokens); }
public WordTokenCollection Forward(int index, int maxLength) { WordTokenCollection tokens = new WordTokenCollection(); int newIndex = 0; int oldIndex = 0; int spacesCounter = 1; char separator = ' '; bool wasSpace = false; if (index >= message.Length) { return(tokens); } do { newIndex = message.IndexOf(separator, index); if (newIndex == -1) { //We only add substring if there aren't spaces AddSpaces(tokens, index, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) { break; } if (index != message.Length) { tokens.Add(new WordToken(message.Substring(index), index)); } break; } if (newIndex - oldIndex == 1) //Previous index is also space { wasSpace = true; oldIndex = newIndex; index = newIndex + 1; spacesCounter++; continue; } else if (newIndex - index == 0) //First index is space { wasSpace = true; oldIndex = newIndex; index = newIndex + 1; continue; } AddSpaces(tokens, index, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) { break; } tokens.Add(new WordToken(message.Substring(index, newIndex - index), index)); if (tokens.Count == maxLength) { break; } oldIndex = newIndex; index = newIndex + 1; } while (true); return(tokens); }
public WordTokenCollection Backwards (int index, int maxLength) { WordTokenCollection tokens = new WordTokenCollection (); int newIndex = 0; int oldIndex = 0; char separator = ' '; bool wasSpace = false; int spacesCounter = 1; if (index <= 0) return tokens; do { if (index == -1) { //We have reached last found space if (AddSpaces (tokens, newIndex + spacesCounter, ref wasSpace, ref spacesCounter) == false) { if (tokens.Count == maxLength) break; tokens.Add (new WordToken (message.Substring (newIndex + 1, index - newIndex), newIndex - 1)); } break; } newIndex = message.LastIndexOf (separator, index, index + 1); if (newIndex == -1) { AddSpaces (tokens, oldIndex + spacesCounter, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) break; tokens.Add (new WordToken (message.Substring (0, index + 1), 0)); break; } if (oldIndex - newIndex == 1) { //Previous index is also space wasSpace = true; oldIndex = newIndex; index = newIndex - 1; spacesCounter++; continue; } else if (newIndex - index == 0) { //First index is space wasSpace = true; oldIndex = newIndex; index = newIndex - 1; continue; } AddSpaces (tokens, oldIndex + spacesCounter, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) break; tokens.Add (new WordToken (message.Substring (newIndex + 1, index - newIndex), newIndex + 1)); if (tokens.Count == maxLength) break; oldIndex = newIndex; index = newIndex - 1; } while (true); return tokens; }
private bool AddSpaces (WordTokenCollection tokens, int index, ref bool wasSpace, ref int spaces) { bool previousValue = wasSpace; if (wasSpace) { tokens.Add (new WordToken (new String (' ', spaces), index - spaces)); spaces = 1; wasSpace = false; } else if (tokens.Count > 0) tokens.Add (new WordToken (" ", index - spaces)); return previousValue; }
public WordTokenCollection Forward (int index, int maxLength) { WordTokenCollection tokens = new WordTokenCollection (); int newIndex = 0; int oldIndex = 0; int spacesCounter = 1; char separator = ' '; bool wasSpace = false; if (index >= message.Length) return tokens; do { newIndex = message.IndexOf (separator, index); if (newIndex == -1) { //We only add substring if there aren't spaces AddSpaces (tokens, index, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) break; if (index != message.Length) tokens.Add (new WordToken (message.Substring (index), index)); break; } if (newIndex - oldIndex == 1) { //Previous index is also space wasSpace = true; oldIndex = newIndex; index = newIndex + 1; spacesCounter++; continue; } else if (newIndex - index == 0) { //First index is space wasSpace = true; oldIndex = newIndex; index = newIndex + 1; continue; } AddSpaces (tokens, index, ref wasSpace, ref spacesCounter); if (tokens.Count == maxLength) break; tokens.Add (new WordToken (message.Substring (index, newIndex - index), index)); if (tokens.Count == maxLength) break; oldIndex = newIndex; index = newIndex + 1; } while (true); return tokens; }