public void TestNoCoalesce() { String str = "This is a test\rto see if\nwhitespace is handled \n\r unusually\r\n by our tokenizer\n\n\n!!!plus some other odd ones like \ttab\ttab\ttab\nand form\ffeed\ffoo.\n"; String delims = " \t\n\r\f\ud800\udc00"; ILOG.J2CsMapping.Util.StringTokenizer jt = new ILOG.J2CsMapping.Util.StringTokenizer(str, delims, true); IBM.ICU.Util.StringTokenizer it = new IBM.ICU.Util.StringTokenizer( str, delims, true); int n = 0; while (jt.HasMoreTokens() && it.HasMoreTokens()) { AssertEquals("[" + n++.ToString() + "]", jt.NextToken(), it.NextToken()); } AssertFalse("java tokenizer has no more tokens", jt.HasMoreTokens()); AssertFalse("icu tokenizer has no more tokens", it.HasMoreTokens()); String sur = "Even\ud800\udc00 works.\n\n"; it = new IBM.ICU.Util.StringTokenizer(sur, delims, true); // no // coalesce AssertEquals("sur1", it.NextToken(), "Even"); AssertEquals("sur2", it.NextToken(), "\ud800\udc00"); AssertEquals("sur3", it.NextToken(), " "); AssertEquals("sur4", it.NextToken(), "works."); AssertEquals("sur5", it.NextToken(), "\n"); AssertEquals("sur6", it.NextToken(), "\n"); AssertFalse("sur7", it.HasMoreTokens()); }
public void TestCountTokensNoCoalesce() { // jitterbug 5207 String str = "\"\""; String del = "\""; IBM.ICU.Util.StringTokenizer st = new IBM.ICU.Util.StringTokenizer(str, del, true); int count = 0; while (st.HasMoreTokens()) { String t = st.NextToken(); Logln("[" + count + "] '" + t + "'"); ++count; } st = new IBM.ICU.Util.StringTokenizer(str, del, true); int ncount = st.CountTokens(); int xcount = 0; while (st.HasMoreTokens()) { String t_0 = st.NextToken(); Logln("[" + xcount + "] '" + t_0 + "'"); ++xcount; } if (count != ncount || count != xcount) { Errln("inconsistent counts " + count + ", " + ncount + ", " + xcount); } }
public void TestBug4423() { // bug 4423: a bad interaction between countTokens() and // hasMoreTokens(). // String s1 = "This is a test"; IBM.ICU.Util.StringTokenizer tzr = new IBM.ICU.Util.StringTokenizer(s1); int tokenCount = 0; int t = tzr.CountTokens(); if (t != 4) { Errln("tzr.countTokens() returned " + t + ". Expected 4"); } while (tzr.HasMoreTokens()) { String tok = tzr.NextToken(); if (tok.Length == 0) { Errln("token with length == 0"); } tokenCount++; } if (tokenCount != 4) { Errln("Incorrect number of tokens found = " + tokenCount); } // Precomputed tokens arrays can grow. Check for edge cases around // boundary where growth is forced. Array grows in increments of 100 // tokens. String s2 = ""; for (int i = 1; i < 250; i++) { s2 = s2 + " " + i; IBM.ICU.Util.StringTokenizer tzb = new IBM.ICU.Util.StringTokenizer(s2); int t2 = tzb.CountTokens(); if (t2 != i) { Errln("tzb.countTokens() returned " + t + ". Expected " + i); break; } int j = 0; while (tzb.HasMoreTokens()) { String tok_0 = tzb.NextToken(); j++; if (tok_0.Equals(ILOG.J2CsMapping.Util.IlNumber.ToString(j)) == false) { Errln("Wrong token string. Expected \"" + j + "\", got \"" + tok_0 + "\"."); break; } } if (j != i) { Errln("Wrong number of tokens. Expected " + i + ". Got " + j + "."); break; } } }