public void TestNoCoalesce()
{
String str = "This is a test\rto see if\nwhitespace is handled \n\r unusually\r\n by our tokenizer\n\n\n!!!plus some other odd ones like \ttab\ttab\ttab\nand form\ffeed\ffoo.\n";
String delims = " \t\n\r\f\ud800\udc00";
ILOG.J2CsMapping.Util.StringTokenizer jt = new ILOG.J2CsMapping.Util.StringTokenizer(str,
delims, true);
IBM.ICU.Util.StringTokenizer it = new IBM.ICU.Util.StringTokenizer(
str, delims, true);
int n = 0;
while (jt.HasMoreTokens() && it.HasMoreTokens())
{
AssertEquals("[" + n++.ToString() + "]", jt.NextToken(),
it.NextToken());
}
AssertFalse("java tokenizer has no more tokens", jt.HasMoreTokens());
AssertFalse("icu tokenizer has no more tokens", it.HasMoreTokens());
String sur = "Even\ud800\udc00 works.\n\n";
it = new IBM.ICU.Util.StringTokenizer(sur, delims, true); // no
// coalesce
AssertEquals("sur1", it.NextToken(), "Even");
AssertEquals("sur2", it.NextToken(), "\ud800\udc00");
AssertEquals("sur3", it.NextToken(), " ");
AssertEquals("sur4", it.NextToken(), "works.");
AssertEquals("sur5", it.NextToken(), "\n");
AssertEquals("sur6", it.NextToken(), "\n");
AssertFalse("sur7", it.HasMoreTokens());
}
public void TestCountTokensNoCoalesce()
{
// jitterbug 5207
String str = "\"\"";
String del = "\"";
IBM.ICU.Util.StringTokenizer st = new IBM.ICU.Util.StringTokenizer(str, del, true);
int count = 0;
while (st.HasMoreTokens())
{
String t = st.NextToken();
Logln("[" + count + "] '" + t + "'");
++count;
}
st = new IBM.ICU.Util.StringTokenizer(str, del, true);
int ncount = st.CountTokens();
int xcount = 0;
while (st.HasMoreTokens())
{
String t_0 = st.NextToken();
Logln("[" + xcount + "] '" + t_0 + "'");
++xcount;
}
if (count != ncount || count != xcount)
{
Errln("inconsistent counts " + count + ", " + ncount + ", "
+ xcount);
}
}
public void TestBug4423()
{
// bug 4423: a bad interaction between countTokens() and
// hasMoreTokens().
//
String s1 = "This is a test";
IBM.ICU.Util.StringTokenizer tzr = new IBM.ICU.Util.StringTokenizer(s1);
int tokenCount = 0;
int t = tzr.CountTokens();
if (t != 4)
{
Errln("tzr.countTokens() returned " + t + ". Expected 4");
}
while (tzr.HasMoreTokens())
{
String tok = tzr.NextToken();
if (tok.Length == 0)
{
Errln("token with length == 0");
}
tokenCount++;
}
if (tokenCount != 4)
{
Errln("Incorrect number of tokens found = " + tokenCount);
}
// Precomputed tokens arrays can grow. Check for edge cases around
// boundary where growth is forced. Array grows in increments of 100
// tokens.
String s2 = "";
for (int i = 1; i < 250; i++)
{
s2 = s2 + " " + i;
IBM.ICU.Util.StringTokenizer tzb = new IBM.ICU.Util.StringTokenizer(s2);
int t2 = tzb.CountTokens();
if (t2 != i)
{
Errln("tzb.countTokens() returned " + t + ". Expected " + i);
break;
}
int j = 0;
while (tzb.HasMoreTokens())
{
String tok_0 = tzb.NextToken();
j++;
if (tok_0.Equals(ILOG.J2CsMapping.Util.IlNumber.ToString(j)) == false)
{
Errln("Wrong token string. Expected \"" + j + "\", got \""
+ tok_0 + "\".");
break;
}
}
if (j != i)
{
Errln("Wrong number of tokens. Expected " + i + ". Got " + j
+ ".");
break;
}
}
}
