public void BackwardsTest10MaxLength2()
{
WordTokenizer tokenizer = new WordTokenizer(message);
WordTokenCollection tokens = tokenizer.Backwards(10, 2);
Assert.AreEqual("morn", tokens [0].Message, "morn");
Assert.AreEqual(" ", tokens [1].Message, "3 spaces.");
Assert.AreEqual(2, tokens.Count, "Count");
}
public void ForwardMissingTest()
{
string localMessage = "one, two, three, four";
WordTokenizer tokenizer = new WordTokenizer(localMessage);
WordTokenCollection tokens = tokenizer.Forward(0, 10);
Assert.AreEqual(7, tokens.Count, "Count");
Assert.AreEqual(tokens [0].Message,
localMessage.Substring(tokens [0].Index, tokens [0].Message.Length),
"one,");
Assert.AreEqual("one,", tokens [0].Message, "one,");
Assert.AreEqual(tokens [1].Message,
localMessage.Substring(tokens [1].Index, tokens [1].Message.Length),
"1 space");
Assert.AreEqual(" ", tokens [1].Message, "1 space");
Assert.AreEqual(tokens [2].Message,
localMessage.Substring(tokens [2].Index, tokens [2].Message.Length),
"two,");
Assert.AreEqual("two,", tokens [2].Message, "two,");
Assert.AreEqual(tokens [3].Message,
localMessage.Substring(tokens [3].Index, tokens [3].Message.Length),
"2 spaces");
Assert.AreEqual(" ", tokens [3].Message, "2 spaces,");
Assert.AreEqual(tokens [4].Message,
localMessage.Substring(tokens [4].Index, tokens [4].Message.Length),
"three,");
Assert.AreEqual("three,", tokens [4].Message, "three,");
Assert.AreEqual(tokens [5].Message,
localMessage.Substring(tokens [5].Index, tokens [5].Message.Length),
"3 spaces");
Assert.AreEqual(" ", tokens [5].Message, "3 spaces");
Assert.AreEqual(tokens [6].Message,
localMessage.Substring(tokens [6].Index, tokens [6].Message.Length),
"four");
Assert.AreEqual("four", tokens [6].Message, "four");
tokens = tokenizer.Forward(localMessage.Length);
Assert.AreEqual(0, tokens.Count, "Count");
}
private bool AddSpaces(WordTokenCollection tokens, int index,
ref bool wasSpace, ref int spaces)
{
bool previousValue = wasSpace;
if (wasSpace)
{
tokens.Add(new WordToken(new String(' ', spaces), index - spaces));
spaces = 1;
wasSpace = false;
}
else if (tokens.Count > 0)
{
tokens.Add(new WordToken(" ", index - spaces));
}
return(previousValue);
}
public void BackwardsTestEmptyWord()
{
string localMessage = "One morning, when Gregor Samsa ";
WordTokenizer tokenizer = new WordTokenizer(localMessage);
WordTokenCollection tokens = tokenizer.Backwards(localMessage.Length - 1, 2);
Assert.AreEqual(tokens [0].Message,
localMessage.Substring(tokens [0].Index, tokens [0].Message.Length),
"3 spaces");
Assert.AreEqual(" ", tokens [0].Message, "3 spaces");
Assert.AreEqual(tokens [1].Message,
localMessage.Substring(tokens [1].Index, tokens [1].Message.Length),
"Samsa");
Assert.AreEqual("Samsa", tokens [1].Message, "Samsa");
Assert.AreEqual(2, tokens.Count, "Count");
}
public void ForwardIndexTest()
{
string localMessage = "one, two, three, four ";
WordTokenizer tokenizer = new WordTokenizer(localMessage);
WordTokenCollection tokens = tokenizer.Forward(4);
Assert.AreEqual(7, tokens.Count, "Count");
tokens = tokenizer.Forward(4, 2);
Assert.AreEqual(2, tokens.Count, "Count");
tokens = tokenizer.Forward(0);
Assert.AreEqual(tokens [tokens.Count - 1].Message,
localMessage.Substring(tokens [tokens.Count - 1].Index,
tokens [tokens.Count - 1].Message.Length),
"Empty spaces");
}
public void ForwardTest10()
{
WordTokenizer tokenizer = new WordTokenizer(message);
WordTokenCollection tokens = tokenizer.Forward(10);
Assert.AreEqual(tokens [0].Message,
message.Substring(tokens [0].Index, tokens [0].Message.Length),
"ning,");
Assert.AreEqual("ning,", tokens [0].Message, "ning,");
Assert.AreEqual(tokens [1].Message,
message.Substring(tokens [1].Index, tokens [1].Message.Length),
"4 spaces");
Assert.AreEqual(" ", tokens [1].Message, "4 spaces");
Assert.AreEqual(tokens [2].Message,
message.Substring(tokens [2].Index, tokens [2].Message.Length),
"when");
Assert.AreEqual("when", tokens [2].Message, "when");
}
public void BackwardsTest10()
{
WordTokenizer tokenizer = new WordTokenizer(message);
WordTokenCollection tokens = tokenizer.Backwards(10);
Assert.AreEqual(tokens [0].Message,
message.Substring(tokens [0].Index, tokens [0].Message.Length),
"morn");
Assert.AreEqual("morn", tokens [0].Message, "morn");
Assert.AreEqual(" ", tokens [1].Message, "3 spaces.");
Assert.AreEqual(tokens [1].Message,
message.Substring(tokens [1].Index, tokens [1].Message.Length),
"4 spaces");
Assert.AreEqual("One", tokens [2].Message, "One");
Assert.AreEqual(tokens [2].Message,
message.Substring(tokens [2].Index, tokens [2].Message.Length),
"One");
}
public void ForwardTest()
{
WordTokenizer tokenizer = new WordTokenizer(message);
WordTokenCollection tokens = tokenizer.Forward(0);
Assert.AreEqual(tokens [0].Message,
message.Substring(tokens [0].Index, tokens [0].Message.Length),
"One");
Assert.AreEqual(tokens [1].Message,
message.Substring(tokens [1].Index, tokens [1].Message.Length),
"3 spaces");
Assert.AreEqual(tokens [2].Message,
message.Substring(tokens [2].Index, tokens [2].Message.Length),
"morning");
Assert.AreEqual(tokens [3].Message,
message.Substring(tokens [3].Index, tokens [3].Message.Length),
"4 spaces");
Assert.AreEqual(tokens [4].Message,
message.Substring(tokens [4].Index, tokens [4].Message.Length),
"when");
Assert.AreEqual(tokens [5].Message,
message.Substring(tokens [5].Index, tokens [5].Message.Length),
"1 space");
Assert.AreEqual(tokens [6].Message,
message.Substring(tokens [6].Index, tokens [6].Message.Length),
"Gregor");
Assert.AreEqual(tokens [7].Message,
message.Substring(tokens [7].Index, tokens [7].Message.Length),
"1 space");
Assert.AreEqual(tokens [8].Message,
message.Substring(tokens [8].Index, tokens [8].Message.Length),
"Samsa");
Assert.AreEqual(tokens [9].Message,
message.Substring(tokens [9].Index, tokens [9].Message.Length),
"1 space");
Assert.AreEqual(tokens [10].Message,
message.Substring(tokens [10].Index, tokens [10].Message.Length),
"woke");
Assert.AreEqual(tokens [11].Message,
message.Substring(tokens [11].Index, tokens [11].Message.Length),
"1 space");
Assert.AreEqual(tokens [12].Message,
message.Substring(tokens [12].Index, tokens [12].Message.Length),
"from");
Assert.AreEqual(tokens [13].Message,
message.Substring(tokens [13].Index, tokens [13].Message.Length),
"1 space");
Assert.AreEqual(tokens [14].Message,
message.Substring(tokens [14].Index, tokens [14].Message.Length),
"troubled");
Assert.AreEqual(tokens [15].Message,
message.Substring(tokens [15].Index, tokens [15].Message.Length),
"1 space");
Assert.AreEqual(tokens [16].Message,
message.Substring(tokens [16].Index, tokens [16].Message.Length),
"dreams,");
Assert.AreEqual(tokens [17].Message,
message.Substring(tokens [17].Index, tokens [17].Message.Length),
"1 space");
Assert.AreEqual(tokens [18].Message,
message.Substring(tokens [18].Index, tokens [18].Message.Length),
"he");
Assert.AreEqual(tokens [19].Message,
message.Substring(tokens [19].Index, tokens [19].Message.Length),
"1 space");
Assert.AreEqual(tokens [20].Message,
message.Substring(tokens [20].Index, tokens [20].Message.Length),
"found");
Assert.AreEqual(tokens [21].Message,
message.Substring(tokens [21].Index, tokens [21].Message.Length),
"1 space");
Assert.AreEqual(tokens [22].Message,
message.Substring(tokens [22].Index, tokens [22].Message.Length),
"himself");
Assert.AreEqual(tokens [23].Message,
message.Substring(tokens [23].Index, tokens [23].Message.Length),
"1 space");
Assert.AreEqual(tokens [24].Message,
message.Substring(tokens [24].Index, tokens [24].Message.Length),
"transformed");
Assert.AreEqual(tokens [25].Message,
message.Substring(tokens [25].Index, tokens [25].Message.Length),
"1 space");
Assert.AreEqual(tokens [26].Message,
message.Substring(tokens [26].Index, tokens [26].Message.Length),
"in");
Assert.AreEqual(tokens [27].Message,
message.Substring(tokens [27].Index, tokens [27].Message.Length),
"1 space");
Assert.AreEqual(tokens [28].Message,
message.Substring(tokens [28].Index, tokens [28].Message.Length),
"his");
Assert.AreEqual(tokens [29].Message,
message.Substring(tokens [29].Index, tokens [29].Message.Length),
"1 space");
Assert.AreEqual(tokens [30].Message,
message.Substring(tokens [30].Index, tokens [30].Message.Length),
"bed");
Assert.AreEqual(tokens [31].Message,
message.Substring(tokens [31].Index, tokens [31].Message.Length),
"1 space");
Assert.AreEqual(tokens [32].Message,
message.Substring(tokens [32].Index, tokens [32].Message.Length),
"into");
Assert.AreEqual(tokens [33].Message,
message.Substring(tokens [33].Index, tokens [33].Message.Length),
"1 space");
Assert.AreEqual(tokens [34].Message,
message.Substring(tokens [34].Index, tokens [34].Message.Length),
"a");
Assert.AreEqual(tokens [35].Message,
message.Substring(tokens [35].Index, tokens [35].Message.Length),
"1 space");
Assert.AreEqual(tokens [36].Message,
message.Substring(tokens [36].Index, tokens [36].Message.Length),
"horrible");
Assert.AreEqual(tokens [37].Message,
message.Substring(tokens [37].Index, tokens [37].Message.Length),
"1 space");
Assert.AreEqual(tokens [38].Message,
message.Substring(tokens [38].Index, tokens [38].Message.Length),
"vermin.");
Assert.AreEqual(tokens [39].Message,
message.Substring(tokens [39].Index, tokens [39].Message.Length),
"3 spaces");
}
public WordTokenCollection Backwards(int index, int maxLength)
{
WordTokenCollection tokens = new WordTokenCollection();
int newIndex = 0;
int oldIndex = 0;
char separator = ' ';
bool wasSpace = false;
int spacesCounter = 1;
if (index <= 0)
{
return(tokens);
}
do
{
if (index == -1) //We have reached last found space
{
if (AddSpaces(tokens, newIndex + spacesCounter,
ref wasSpace, ref spacesCounter) == false)
{
if (tokens.Count == maxLength)
{
break;
}
tokens.Add(new WordToken(message.Substring(newIndex + 1, index - newIndex),
newIndex - 1));
}
break;
}
newIndex = message.LastIndexOf(separator, index, index + 1);
if (newIndex == -1)
{
AddSpaces(tokens, oldIndex + spacesCounter, ref wasSpace,
ref spacesCounter);
if (tokens.Count == maxLength)
{
break;
}
tokens.Add(new WordToken(message.Substring(0, index + 1), 0));
break;
}
if (oldIndex - newIndex == 1) //Previous index is also space
{
wasSpace = true;
oldIndex = newIndex;
index = newIndex - 1;
spacesCounter++;
continue;
}
else if (newIndex - index == 0) //First index is space
{
wasSpace = true;
oldIndex = newIndex;
index = newIndex - 1;
continue;
}
AddSpaces(tokens, oldIndex + spacesCounter, ref wasSpace,
ref spacesCounter);
if (tokens.Count == maxLength)
{
break;
}
tokens.Add(new WordToken(message.Substring(newIndex + 1, index - newIndex),
newIndex + 1));
if (tokens.Count == maxLength)
{
break;
}
oldIndex = newIndex;
index = newIndex - 1;
} while (true);
return(tokens);
}
public WordTokenCollection Forward(int index, int maxLength)
{
WordTokenCollection tokens = new WordTokenCollection();
int newIndex = 0;
int oldIndex = 0;
int spacesCounter = 1;
char separator = ' ';
bool wasSpace = false;
if (index >= message.Length)
{
return(tokens);
}
do
{
newIndex = message.IndexOf(separator, index);
if (newIndex == -1)
{
//We only add substring if there aren't spaces
AddSpaces(tokens, index, ref wasSpace, ref spacesCounter);
if (tokens.Count == maxLength)
{
break;
}
if (index != message.Length)
{
tokens.Add(new WordToken(message.Substring(index), index));
}
break;
}
if (newIndex - oldIndex == 1) //Previous index is also space
{
wasSpace = true;
oldIndex = newIndex;
index = newIndex + 1;
spacesCounter++;
continue;
}
else if (newIndex - index == 0) //First index is space
{
wasSpace = true;
oldIndex = newIndex;
index = newIndex + 1;
continue;
}
AddSpaces(tokens, index, ref wasSpace, ref spacesCounter);
if (tokens.Count == maxLength)
{
break;
}
tokens.Add(new WordToken(message.Substring(index, newIndex - index),
index));
if (tokens.Count == maxLength)
{
break;
}
oldIndex = newIndex;
index = newIndex + 1;
} while (true);
return(tokens);
}
public WordTokenCollection Backwards (int index, int maxLength) {
WordTokenCollection tokens = new WordTokenCollection ();
int newIndex = 0;
int oldIndex = 0;
char separator = ' ';
bool wasSpace = false;
int spacesCounter = 1;
if (index <= 0)
return tokens;
do {
if (index == -1) { //We have reached last found space
if (AddSpaces (tokens, newIndex + spacesCounter,
ref wasSpace, ref spacesCounter) == false) {
if (tokens.Count == maxLength)
break;
tokens.Add (new WordToken (message.Substring (newIndex + 1, index - newIndex),
newIndex - 1));
}
break;
}
newIndex = message.LastIndexOf (separator, index, index + 1);
if (newIndex == -1) {
AddSpaces (tokens, oldIndex + spacesCounter, ref wasSpace,
ref spacesCounter);
if (tokens.Count == maxLength)
break;
tokens.Add (new WordToken (message.Substring (0, index + 1), 0));
break;
}
if (oldIndex - newIndex == 1) { //Previous index is also space
wasSpace = true;
oldIndex = newIndex;
index = newIndex - 1;
spacesCounter++;
continue;
} else if (newIndex - index == 0) { //First index is space
wasSpace = true;
oldIndex = newIndex;
index = newIndex - 1;
continue;
}
AddSpaces (tokens, oldIndex + spacesCounter, ref wasSpace,
ref spacesCounter);
if (tokens.Count == maxLength)
break;
tokens.Add (new WordToken (message.Substring (newIndex + 1, index - newIndex),
newIndex + 1));
if (tokens.Count == maxLength)
break;
oldIndex = newIndex;
index = newIndex - 1;
} while (true);
return tokens;
}
private bool AddSpaces (WordTokenCollection tokens, int index,
ref bool wasSpace, ref int spaces) {
bool previousValue = wasSpace;
if (wasSpace) {
tokens.Add (new WordToken (new String (' ', spaces), index - spaces));
spaces = 1;
wasSpace = false;
} else if (tokens.Count > 0)
tokens.Add (new WordToken (" ", index - spaces));
return previousValue;
}
public WordTokenCollection Forward (int index, int maxLength) {
WordTokenCollection tokens = new WordTokenCollection ();
int newIndex = 0;
int oldIndex = 0;
int spacesCounter = 1;
char separator = ' ';
bool wasSpace = false;
if (index >= message.Length)
return tokens;
do {
newIndex = message.IndexOf (separator, index);
if (newIndex == -1) {
//We only add substring if there aren't spaces
AddSpaces (tokens, index, ref wasSpace, ref spacesCounter);
if (tokens.Count == maxLength)
break;
if (index != message.Length)
tokens.Add (new WordToken (message.Substring (index), index));
break;
}
if (newIndex - oldIndex == 1) { //Previous index is also space
wasSpace = true;
oldIndex = newIndex;
index = newIndex + 1;
spacesCounter++;
continue;
} else if (newIndex - index == 0) { //First index is space
wasSpace = true;
oldIndex = newIndex;
index = newIndex + 1;
continue;
}
AddSpaces (tokens, index, ref wasSpace, ref spacesCounter);
if (tokens.Count == maxLength)
break;
tokens.Add (new WordToken (message.Substring (index, newIndex - index),
index));
if (tokens.Count == maxLength)
break;
oldIndex = newIndex;
index = newIndex + 1;
} while (true);
return tokens;
}