public void TestCountTokensNoCoalesce()
{
// jitterbug 5207
String str = "\"\"";
String del = "\"";
IBM.ICU.Util.StringTokenizer st = new IBM.ICU.Util.StringTokenizer(str, del, true);
int count = 0;
while (st.HasMoreTokens())
{
String t = st.NextToken();
Logln("[" + count + "] '" + t + "'");
++count;
}
st = new IBM.ICU.Util.StringTokenizer(str, del, true);
int ncount = st.CountTokens();
int xcount = 0;
while (st.HasMoreTokens())
{
String t_0 = st.NextToken();
Logln("[" + xcount + "] '" + t_0 + "'");
++xcount;
}
if (count != ncount || count != xcount)
{
Errln("inconsistent counts " + count + ", " + ncount + ", "
+ xcount);
}
}
public void TestNoCoalesce()
{
String str = "This is a test\rto see if\nwhitespace is handled \n\r unusually\r\n by our tokenizer\n\n\n!!!plus some other odd ones like \ttab\ttab\ttab\nand form\ffeed\ffoo.\n";
String delims = " \t\n\r\f\ud800\udc00";
ILOG.J2CsMapping.Util.StringTokenizer jt = new ILOG.J2CsMapping.Util.StringTokenizer(str,
delims, true);
IBM.ICU.Util.StringTokenizer it = new IBM.ICU.Util.StringTokenizer(
str, delims, true);
int n = 0;
while (jt.HasMoreTokens() && it.HasMoreTokens())
{
AssertEquals("[" + n++.ToString() + "]", jt.NextToken(),
it.NextToken());
}
AssertFalse("java tokenizer has no more tokens", jt.HasMoreTokens());
AssertFalse("icu tokenizer has no more tokens", it.HasMoreTokens());
String sur = "Even\ud800\udc00 works.\n\n";
it = new IBM.ICU.Util.StringTokenizer(sur, delims, true); // no
// coalesce
AssertEquals("sur1", it.NextToken(), "Even");
AssertEquals("sur2", it.NextToken(), "\ud800\udc00");
AssertEquals("sur3", it.NextToken(), " ");
AssertEquals("sur4", it.NextToken(), "works.");
AssertEquals("sur5", it.NextToken(), "\n");
AssertEquals("sur6", it.NextToken(), "\n");
AssertFalse("sur7", it.HasMoreTokens());
}
public void TestNextNewDelimiters()
{
String str = "abc0def1ghi2jkl3mno4pqr0stu1vwx2yza3bcd4efg0hij1klm2nop3qrs4tuv";
String[] delimiter = { "0", "1", "2", "3", "4" };
String[][] expected = { new String[] { "abc", "pqr", "efg" },
new String[] { "def", "stu", "hij" },
new String[] { "ghi", "vwx", "klm" },
new String[] { "jkl", "yza", "nop" },
new String[] { "mno", "bcd", "qrs" } };
IBM.ICU.Util.StringTokenizer st = new IBM.ICU.Util.StringTokenizer(str);
int size = expected[0].Length;
for (int i = 0; i < size; i++)
{
for (int j = 0; j < delimiter.Length; j++)
{
if (!st.NextToken(delimiter[j]).Equals(expected[j][i]))
{
Errln("nextToken() with delimiters error " + i + " " + j);
}
if (st.CountTokens() != expected[j].Length - i)
{
Errln("countTokens() after nextToken() with delimiters error"
+ i + " " + j);
}
}
}
st = new IBM.ICU.Util.StringTokenizer(str);
String[] delimiter1 = { "0", "2", "4" };
String[] expected1 = { "abc", "def1ghi", "jkl3mno", "pqr", "stu1vwx",
"yza3bcd", "efg", "hij1klm", "nop3qrs", "tuv" };
for (int i_0 = 0; i_0 < expected1.Length; i_0++)
{
if (!st.NextToken(delimiter1[i_0 % delimiter1.Length]).Equals(
expected1[i_0]))
{
Errln("nextToken() with delimiters error " + i_0);
}
}
}
public void TestCountTokens()
{
String str = "this\tis\na\rstring\ftesting\tStringTokenizer\nconstructors!";
String delimiter = " \t\n\r\f";
String[] expected = { "this", "is", "a", "string", "testing",
"StringTokenizer", "constructors!" };
String[] expectedreturn = { "this", "\t", "is", "\n", "a", "\r",
"string", "\f", "testing", "\t", "StringTokenizer", "\n",
"constructors!" };
IBM.ICU.Util.StringTokenizer st = new IBM.ICU.Util.StringTokenizer(str, delimiter);
IBM.ICU.Util.StringTokenizer streturn = new IBM.ICU.Util.StringTokenizer(str, delimiter, true);
if (st.CountTokens() != expected.Length)
{
Errln("CountTokens failed for non-delimiter tokens");
}
if (streturn.CountTokens() != expectedreturn.Length)
{
Errln("CountTokens failed for delimiter tokens");
}
for (int i = 0; i < expected.Length; i++)
{
if (!st.NextToken().Equals(expected[i]) ||
st.CountTokens() != expected.Length - i - 1)
{
Errln("CountTokens default delimiter gives wrong results");
}
}
for (int i_0 = 0; i_0 < expectedreturn.Length; i_0++)
{
if (!streturn.NextToken().Equals(expectedreturn[i_0]) ||
streturn.CountTokens() != expectedreturn.Length - i_0 - 1)
{
Errln("CountTokens with default delimiter and delimiter tokens gives wrong results");
}
}
}
public void TestBug4423()
{
// bug 4423: a bad interaction between countTokens() and
// hasMoreTokens().
//
String s1 = "This is a test";
IBM.ICU.Util.StringTokenizer tzr = new IBM.ICU.Util.StringTokenizer(s1);
int tokenCount = 0;
int t = tzr.CountTokens();
if (t != 4)
{
Errln("tzr.countTokens() returned " + t + ". Expected 4");
}
while (tzr.HasMoreTokens())
{
String tok = tzr.NextToken();
if (tok.Length == 0)
{
Errln("token with length == 0");
}
tokenCount++;
}
if (tokenCount != 4)
{
Errln("Incorrect number of tokens found = " + tokenCount);
}
// Precomputed tokens arrays can grow. Check for edge cases around
// boundary where growth is forced. Array grows in increments of 100
// tokens.
String s2 = "";
for (int i = 1; i < 250; i++)
{
s2 = s2 + " " + i;
IBM.ICU.Util.StringTokenizer tzb = new IBM.ICU.Util.StringTokenizer(s2);
int t2 = tzb.CountTokens();
if (t2 != i)
{
Errln("tzb.countTokens() returned " + t + ". Expected " + i);
break;
}
int j = 0;
while (tzb.HasMoreTokens())
{
String tok_0 = tzb.NextToken();
j++;
if (tok_0.Equals(ILOG.J2CsMapping.Util.IlNumber.ToString(j)) == false)
{
Errln("Wrong token string. Expected \"" + j + "\", got \""
+ tok_0 + "\".");
break;
}
}
if (j != i)
{
Errln("Wrong number of tokens. Expected " + i + ". Got " + j
+ ".");
break;
}
}
}
public void TestConstructors()
{
String str = "this\tis\na\rstring\ftesting\tStringTokenizer\nconstructors!";
String delimiter = " \t\n\r\f";
String[] expected = { "this", "is", "a", "string", "testing",
"StringTokenizer", "constructors!" };
IBM.ICU.Util.StringTokenizer defaultst = new IBM.ICU.Util.StringTokenizer(str);
IBM.ICU.Util.StringTokenizer stdelimiter = new IBM.ICU.Util.StringTokenizer(str, delimiter);
IBM.ICU.Util.StringTokenizer stdelimiterreturn = new IBM.ICU.Util.StringTokenizer(str, delimiter,
false);
UnicodeSet delimiterset = new UnicodeSet("[" + delimiter + "]", false);
IBM.ICU.Util.StringTokenizer stdelimiterset = new IBM.ICU.Util.StringTokenizer(str, delimiterset);
IBM.ICU.Util.StringTokenizer stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str,
delimiterset, false);
for (int i = 0; i < expected.Length; i++)
{
if (!(defaultst.NextToken().Equals(expected[i]) &&
stdelimiter.NextToken().Equals(expected[i]) &&
stdelimiterreturn.NextToken().Equals(expected[i]) &&
stdelimiterset.NextToken().Equals(expected[i]) && stdelimitersetreturn
.NextToken().Equals(expected[i])))
{
Errln("Constructor with default delimiter gives wrong results");
}
}
String[] expected1 = { "this", "\t", "is", "\n", "a", "\r", "string",
"\f", "testing", "\t", "StringTokenizer", "\n", "constructors!" };
stdelimiterreturn = new IBM.ICU.Util.StringTokenizer(str, delimiter, true);
stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str, delimiterset, true);
for (int i_0 = 0; i_0 < expected1.Length; i_0++)
{
if (!(stdelimiterreturn.NextToken().Equals(expected1[i_0]) && stdelimitersetreturn
.NextToken().Equals(expected1[i_0])))
{
Errln("Constructor with default delimiter and delimiter tokens gives wrong results");
}
}
stdelimiter = new IBM.ICU.Util.StringTokenizer(str, (String)null);
stdelimiterreturn = new IBM.ICU.Util.StringTokenizer(str, (String)null, false);
delimiterset = null;
stdelimiterset = new IBM.ICU.Util.StringTokenizer(str, delimiterset);
stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str, delimiterset, false);
if (!(stdelimiter.NextToken().Equals(str) &&
stdelimiterreturn.NextToken().Equals(str) &&
stdelimiterset.NextToken().Equals(str) && stdelimitersetreturn
.NextToken().Equals(str)))
{
Errln("Constructor with null delimiter gives wrong results");
}
delimiter = "";
stdelimiter = new IBM.ICU.Util.StringTokenizer(str, delimiter);
stdelimiterreturn = new IBM.ICU.Util.StringTokenizer(str, delimiter, false);
delimiterset = new UnicodeSet();
stdelimiterset = new IBM.ICU.Util.StringTokenizer(str, delimiterset);
stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str, delimiterset, false);
if (!(stdelimiter.NextToken().Equals(str) &&
stdelimiterreturn.NextToken().Equals(str) &&
stdelimiterset.NextToken().Equals(str) && stdelimitersetreturn
.NextToken().Equals(str)))
{
Errln("Constructor with empty delimiter gives wrong results");
}
try
{
defaultst = new IBM.ICU.Util.StringTokenizer(null);
Errln("null string should throw an exception");
}
catch (Exception e)
{
Logln("PASS: Constructor with null string failed as expected");
}
try
{
stdelimiter = new IBM.ICU.Util.StringTokenizer(null, delimiter);
Errln("null string should throw an exception");
}
catch (Exception e_1)
{
Logln("PASS: Constructor with null string failed as expected");
}
try
{
stdelimiterreturn = new IBM.ICU.Util.StringTokenizer(null, delimiter, false);
Errln("null string should throw an exception");
}
catch (Exception e_2)
{
Logln("PASS: Constructor with null string failed as expected");
}
try
{
stdelimiterset = new IBM.ICU.Util.StringTokenizer(null, delimiterset);
Errln("null string should throw an exception");
}
catch (Exception e_3)
{
Logln("PASS: Constructor with null string failed as expected");
}
try
{
stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(null, delimiterset,
false);
Errln("null string should throw an exception");
}
catch (Exception e_4)
{
Logln("PASS: Constructor with null string failed as expected");
}
}
public void TestNextDelimiterToken()
{
String str = " , 1 2 3 AHHHHH! 5.5 6 7 , 8\n";
String[] expected = { " ", ",", " ", "1", " ", "2", " ", "3", " ",
"AHHHHH!", " ", "5.5", " ", "6", " ", "7", " ", ",",
" ", "8\n" };
String delimiter = " ";
IBM.ICU.Util.StringTokenizer tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter, true,
true);
int currtoken = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected[currtoken]))
{
Errln("Error token mismatch, expected " + expected[currtoken]);
}
currtoken++;
}
if (currtoken != expected.Length)
{
Errln("Didn't get correct number of tokens");
}
tokenizer = new IBM.ICU.Util.StringTokenizer("", delimiter, true);
if (tokenizer.HasNext())
{
Errln("Empty string should not have any tokens");
}
try
{
tokenizer.NextToken();
Errln("Empty string should not have any tokens");
}
catch (Exception e)
{
Logln("PASS: Empty string failed as expected");
}
tokenizer = new IBM.ICU.Util.StringTokenizer(", ,", ", ", true, true);
if (!tokenizer.HasNext())
{
Errln("String with only delimiters should have tokens when delimiter is treated as tokens");
}
if (!tokenizer.NextToken().Equals(", ,"))
{
Errln("String with only delimiters should return itself when delimiter is treated as tokens");
}
tokenizer = new IBM.ICU.Util.StringTokenizer("q, ,", ", ", true, true);
if (!tokenizer.HasNext())
{
Errln("String should have some tokens");
}
if (!tokenizer.NextToken().Equals("q") ||
!tokenizer.NextToken().Equals(", ,"))
{
Errln("String tokens do not match expected results");
}
try
{
tokenizer = new IBM.ICU.Util.StringTokenizer(null, delimiter, true);
Errln("StringTokenizer constructed with null source should throw a nullpointerexception");
}
catch (Exception e_0)
{
Logln("PASS: StringTokenizer constructed with null source failed as expected");
}
tokenizer = new IBM.ICU.Util.StringTokenizer(str, "q", true);
if (!tokenizer.NextToken().Equals(str))
{
Errln("Should have recieved the same string when there are no delimiters");
}
}
public void TestSupplementary()
{
String str = "bmp string \ud800 with a unmatched surrogate character";
String delimiter = "\ud800\udc00";
String[] expected = { str };
IBM.ICU.Util.StringTokenizer tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
if (!tokenizer.NextToken().Equals(expected[0]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
delimiter = "\ud800";
String[] expected1 = { "bmp string ",
" with a unmatched surrogate character" };
tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
int i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected1[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
str = "string \ud800\udc00 with supplementary character";
delimiter = "\ud800";
String[] expected2 = { str };
tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
if (!tokenizer.NextToken().Equals(expected2[0]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
delimiter = "\ud800\udc00";
String[] expected3 = { "string ", " with supplementary character" };
tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected3[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
str = "\ud800 \ud800\udc00 \ud800 \ud800\udc00";
delimiter = "\ud800";
String[] expected4 = { " \ud800\udc00 ", " \ud800\udc00" };
i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected4[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
delimiter = "\ud800\udc00";
String[] expected5 = { "\ud800 ", " \ud800 " };
i = 0;
while (tokenizer.HasNext())
{
if (!tokenizer.NextToken().Equals(expected5[i++]))
{
Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
}
}
if (tokenizer.HasNext())
{
Errln("Number of tokens exceeded expected");
}
}
