public void TestNextDelimiterToken()
{
String str = " , 1 2 3 AHHHHH! 5.5 6 7 , 8\n";
String[] expected = { " ", ",", " ", "1", " ", "2", " ", "3", " ",
"AHHHHH!", " ", "5.5", " ", "6", " ", "7", " ", ",",
" ", "8\n" };
String delimiter = " ";
IBM.ICU.Util.StringTokenizer tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter, true,
true);
int currtoken = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected[currtoken]))
{
Errln("Error token mismatch, expected " + expected[currtoken]);
}
currtoken++;
}
if (currtoken != expected.Length)
{
Errln("Didn't get correct number of tokens");
}
tokenizer = new IBM.ICU.Util.StringTokenizer("", delimiter, true);
if (tokenizer.HasNext())
{
Errln("Empty string should not have any tokens");
}
try
{
tokenizer.NextToken();
Errln("Empty string should not have any tokens");
}
catch (Exception e)
{
Logln("PASS: Empty string failed as expected");
}
tokenizer = new IBM.ICU.Util.StringTokenizer(", ,", ", ", true, true);
if (!tokenizer.HasNext())
{
Errln("String with only delimiters should have tokens when delimiter is treated as tokens");
}
if (!tokenizer.NextToken().Equals(", ,"))
{
Errln("String with only delimiters should return itself when delimiter is treated as tokens");
}
tokenizer = new IBM.ICU.Util.StringTokenizer("q, ,", ", ", true, true);
if (!tokenizer.HasNext())
{
Errln("String should have some tokens");
}
if (!tokenizer.NextToken().Equals("q") ||
!tokenizer.NextToken().Equals(", ,"))
{
Errln("String tokens do not match expected results");
}
try
{
tokenizer = new IBM.ICU.Util.StringTokenizer(null, delimiter, true);
Errln("StringTokenizer constructed with null source should throw a nullpointerexception");
}
catch (Exception e_0)
{
Logln("PASS: StringTokenizer constructed with null source failed as expected");
}
tokenizer = new IBM.ICU.Util.StringTokenizer(str, "q", true);
if (!tokenizer.NextToken().Equals(str))
{
Errln("Should have recieved the same string when there are no delimiters");
}
}
public void TestSupplementary()
{
String str = "bmp string \ud800 with a unmatched surrogate character";
String delimiter = "\ud800\udc00";
String[] expected = { str };
IBM.ICU.Util.StringTokenizer tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
if (!tokenizer.NextToken().Equals(expected[0]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
delimiter = "\ud800";
String[] expected1 = { "bmp string ",
" with a unmatched surrogate character" };
tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
int i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected1[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
str = "string \ud800\udc00 with supplementary character";
delimiter = "\ud800";
String[] expected2 = { str };
tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
if (!tokenizer.NextToken().Equals(expected2[0]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
delimiter = "\ud800\udc00";
String[] expected3 = { "string ", " with supplementary character" };
tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected3[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
str = "\ud800 \ud800\udc00 \ud800 \ud800\udc00";
delimiter = "\ud800";
String[] expected4 = { " \ud800\udc00 ", " \ud800\udc00" };
i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected4[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
delimiter = "\ud800\udc00";
String[] expected5 = { "\ud800 ", " \ud800 " };
i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected5[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
}