Exemplo n.º 1
0
        public void TestCountTokensNoCoalesce()
        {
            // jitterbug 5207
            String str = "\"\"";
            String del = "\"";

            IBM.ICU.Util.StringTokenizer st = new IBM.ICU.Util.StringTokenizer(str, del, true);
            int count = 0;

            while (st.HasMoreTokens())
            {
                String t = st.NextToken();
                Logln("[" + count + "] '" + t + "'");
                ++count;
            }
            st = new IBM.ICU.Util.StringTokenizer(str, del, true);
            int ncount = st.CountTokens();
            int xcount = 0;

            while (st.HasMoreTokens())
            {
                String t_0 = st.NextToken();
                Logln("[" + xcount + "] '" + t_0 + "'");
                ++xcount;
            }
            if (count != ncount || count != xcount)
            {
                Errln("inconsistent counts " + count + ", " + ncount + ", "
                      + xcount);
            }
        }
Exemplo n.º 2
0
        public void TestNoCoalesce()
        {
            String str    = "This is   a test\rto see if\nwhitespace is handled \n\r unusually\r\n by our tokenizer\n\n\n!!!plus some other odd ones like \ttab\ttab\ttab\nand form\ffeed\ffoo.\n";
            String delims = " \t\n\r\f\ud800\udc00";

            ILOG.J2CsMapping.Util.StringTokenizer jt = new ILOG.J2CsMapping.Util.StringTokenizer(str,
                                                                                                 delims, true);
            IBM.ICU.Util.StringTokenizer it = new IBM.ICU.Util.StringTokenizer(
                str, delims, true);
            int n = 0;

            while (jt.HasMoreTokens() && it.HasMoreTokens())
            {
                AssertEquals("[" + n++.ToString() + "]", jt.NextToken(),
                             it.NextToken());
            }
            AssertFalse("java tokenizer has no more tokens", jt.HasMoreTokens());
            AssertFalse("icu tokenizer has no more tokens", it.HasMoreTokens());

            String sur = "Even\ud800\udc00 works.\n\n";

            it = new IBM.ICU.Util.StringTokenizer(sur, delims, true); // no
            // coalesce
            AssertEquals("sur1", it.NextToken(), "Even");
            AssertEquals("sur2", it.NextToken(), "\ud800\udc00");
            AssertEquals("sur3", it.NextToken(), " ");
            AssertEquals("sur4", it.NextToken(), "works.");
            AssertEquals("sur5", it.NextToken(), "\n");
            AssertEquals("sur6", it.NextToken(), "\n");
            AssertFalse("sur7", it.HasMoreTokens());
        }
Exemplo n.º 3
0
        public void TestNextNewDelimiters()
        {
            String str = "abc0def1ghi2jkl3mno4pqr0stu1vwx2yza3bcd4efg0hij1klm2nop3qrs4tuv";

            String[]   delimiter = { "0", "1", "2", "3", "4" };
            String[][] expected  = { new String[] { "abc", "pqr", "efg" },
                                     new String[]  { "def", "stu", "hij" },
                                     new String[]  { "ghi", "vwx", "klm" },
                                     new String[]  { "jkl", "yza", "nop" },
                                     new String[]  { "mno", "bcd", "qrs" } };
            IBM.ICU.Util.StringTokenizer st = new IBM.ICU.Util.StringTokenizer(str);
            int size = expected[0].Length;

            for (int i = 0; i < size; i++)
            {
                for (int j = 0; j < delimiter.Length; j++)
                {
                    if (!st.NextToken(delimiter[j]).Equals(expected[j][i]))
                    {
                        Errln("nextToken() with delimiters error " + i + " " + j);
                    }
                    if (st.CountTokens() != expected[j].Length - i)
                    {
                        Errln("countTokens() after nextToken() with delimiters error"
                              + i + " " + j);
                    }
                }
            }
            st = new IBM.ICU.Util.StringTokenizer(str);
            String[] delimiter1 = { "0", "2", "4" };
            String[] expected1  = { "abc",    "def1ghi", "jkl3mno", "pqr",     "stu1vwx",
                                    "yza3bcd", "efg",     "hij1klm", "nop3qrs", "tuv" };
            for (int i_0 = 0; i_0 < expected1.Length; i_0++)
            {
                if (!st.NextToken(delimiter1[i_0 % delimiter1.Length]).Equals(
                        expected1[i_0]))
                {
                    Errln("nextToken() with delimiters error " + i_0);
                }
            }
        }
Exemplo n.º 4
0
        public void TestCountTokens()
        {
            String str       = "this\tis\na\rstring\ftesting\tStringTokenizer\nconstructors!";
            String delimiter = " \t\n\r\f";

            String[] expected = { "this",            "is", "a", "string", "testing",
                                  "StringTokenizer", "constructors!" };
            String[] expectedreturn = { "this",   "\t", "is",      "\n", "a",               "\r",
                                        "string", "\f", "testing", "\t", "StringTokenizer", "\n",
                                        "constructors!" };
            IBM.ICU.Util.StringTokenizer st       = new IBM.ICU.Util.StringTokenizer(str, delimiter);
            IBM.ICU.Util.StringTokenizer streturn = new IBM.ICU.Util.StringTokenizer(str, delimiter, true);
            if (st.CountTokens() != expected.Length)
            {
                Errln("CountTokens failed for non-delimiter tokens");
            }
            if (streturn.CountTokens() != expectedreturn.Length)
            {
                Errln("CountTokens failed for delimiter tokens");
            }
            for (int i = 0; i < expected.Length; i++)
            {
                if (!st.NextToken().Equals(expected[i]) ||
                    st.CountTokens() != expected.Length - i - 1)
                {
                    Errln("CountTokens default delimiter gives wrong results");
                }
            }
            for (int i_0 = 0; i_0 < expectedreturn.Length; i_0++)
            {
                if (!streturn.NextToken().Equals(expectedreturn[i_0]) ||
                    streturn.CountTokens() != expectedreturn.Length - i_0 - 1)
                {
                    Errln("CountTokens with default delimiter and delimiter tokens gives wrong results");
                }
            }
        }
Exemplo n.º 5
0
        public void TestBug4423()
        {
            // bug 4423: a bad interaction between countTokens() and
            // hasMoreTokens().
            //
            String s1 = "This is a test";

            IBM.ICU.Util.StringTokenizer tzr = new IBM.ICU.Util.StringTokenizer(s1);
            int tokenCount = 0;

            int t = tzr.CountTokens();

            if (t != 4)
            {
                Errln("tzr.countTokens() returned " + t + ".  Expected 4");
            }
            while (tzr.HasMoreTokens())
            {
                String tok = tzr.NextToken();
                if (tok.Length == 0)
                {
                    Errln("token with length == 0");
                }
                tokenCount++;
            }
            if (tokenCount != 4)
            {
                Errln("Incorrect number of tokens found = " + tokenCount);
            }

            // Precomputed tokens arrays can grow. Check for edge cases around
            // boundary where growth is forced. Array grows in increments of 100
            // tokens.
            String s2 = "";

            for (int i = 1; i < 250; i++)
            {
                s2 = s2 + " " + i;
                IBM.ICU.Util.StringTokenizer tzb = new IBM.ICU.Util.StringTokenizer(s2);
                int t2 = tzb.CountTokens();
                if (t2 != i)
                {
                    Errln("tzb.countTokens() returned " + t + ".  Expected " + i);
                    break;
                }
                int j = 0;
                while (tzb.HasMoreTokens())
                {
                    String tok_0 = tzb.NextToken();
                    j++;
                    if (tok_0.Equals(ILOG.J2CsMapping.Util.IlNumber.ToString(j)) == false)
                    {
                        Errln("Wrong token string.  Expected \"" + j + "\", got \""
                              + tok_0 + "\".");
                        break;
                    }
                }
                if (j != i)
                {
                    Errln("Wrong number of tokens.  Expected " + i + ".  Got " + j
                          + ".");
                    break;
                }
            }
        }
Exemplo n.º 6
0
        public void TestConstructors()
        {
            String str       = "this\tis\na\rstring\ftesting\tStringTokenizer\nconstructors!";
            String delimiter = " \t\n\r\f";

            String[] expected = { "this",            "is", "a", "string", "testing",
                                  "StringTokenizer", "constructors!" };
            IBM.ICU.Util.StringTokenizer defaultst         = new IBM.ICU.Util.StringTokenizer(str);
            IBM.ICU.Util.StringTokenizer stdelimiter       = new IBM.ICU.Util.StringTokenizer(str, delimiter);
            IBM.ICU.Util.StringTokenizer stdelimiterreturn = new IBM.ICU.Util.StringTokenizer(str, delimiter,
                                                                                              false);
            UnicodeSet delimiterset = new UnicodeSet("[" + delimiter + "]", false);

            IBM.ICU.Util.StringTokenizer stdelimiterset       = new IBM.ICU.Util.StringTokenizer(str, delimiterset);
            IBM.ICU.Util.StringTokenizer stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str,
                                                                                                 delimiterset, false);
            for (int i = 0; i < expected.Length; i++)
            {
                if (!(defaultst.NextToken().Equals(expected[i]) &&
                      stdelimiter.NextToken().Equals(expected[i]) &&
                      stdelimiterreturn.NextToken().Equals(expected[i]) &&
                      stdelimiterset.NextToken().Equals(expected[i]) && stdelimitersetreturn
                      .NextToken().Equals(expected[i])))
                {
                    Errln("Constructor with default delimiter gives wrong results");
                }
            }

            String[] expected1 = { "this", "\t",      "is", "\n",              "a",  "\r", "string",
                                   "\f",   "testing", "\t", "StringTokenizer", "\n", "constructors!" };
            stdelimiterreturn    = new IBM.ICU.Util.StringTokenizer(str, delimiter, true);
            stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str, delimiterset, true);
            for (int i_0 = 0; i_0 < expected1.Length; i_0++)
            {
                if (!(stdelimiterreturn.NextToken().Equals(expected1[i_0]) && stdelimitersetreturn
                      .NextToken().Equals(expected1[i_0])))
                {
                    Errln("Constructor with default delimiter and delimiter tokens gives wrong results");
                }
            }

            stdelimiter          = new IBM.ICU.Util.StringTokenizer(str, (String)null);
            stdelimiterreturn    = new IBM.ICU.Util.StringTokenizer(str, (String)null, false);
            delimiterset         = null;
            stdelimiterset       = new IBM.ICU.Util.StringTokenizer(str, delimiterset);
            stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str, delimiterset, false);

            if (!(stdelimiter.NextToken().Equals(str) &&
                  stdelimiterreturn.NextToken().Equals(str) &&
                  stdelimiterset.NextToken().Equals(str) && stdelimitersetreturn
                  .NextToken().Equals(str)))
            {
                Errln("Constructor with null delimiter gives wrong results");
            }

            delimiter            = "";
            stdelimiter          = new IBM.ICU.Util.StringTokenizer(str, delimiter);
            stdelimiterreturn    = new IBM.ICU.Util.StringTokenizer(str, delimiter, false);
            delimiterset         = new UnicodeSet();
            stdelimiterset       = new IBM.ICU.Util.StringTokenizer(str, delimiterset);
            stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(str, delimiterset, false);

            if (!(stdelimiter.NextToken().Equals(str) &&
                  stdelimiterreturn.NextToken().Equals(str) &&
                  stdelimiterset.NextToken().Equals(str) && stdelimitersetreturn
                  .NextToken().Equals(str)))
            {
                Errln("Constructor with empty delimiter gives wrong results");
            }

            try
            {
                defaultst = new IBM.ICU.Util.StringTokenizer(null);
                Errln("null string should throw an exception");
            }
            catch (Exception e)
            {
                Logln("PASS: Constructor with null string failed as expected");
            }
            try
            {
                stdelimiter = new IBM.ICU.Util.StringTokenizer(null, delimiter);
                Errln("null string should throw an exception");
            }
            catch (Exception e_1)
            {
                Logln("PASS: Constructor with null string failed as expected");
            }
            try
            {
                stdelimiterreturn = new IBM.ICU.Util.StringTokenizer(null, delimiter, false);
                Errln("null string should throw an exception");
            }
            catch (Exception e_2)
            {
                Logln("PASS: Constructor with null string failed as expected");
            }
            try
            {
                stdelimiterset = new IBM.ICU.Util.StringTokenizer(null, delimiterset);
                Errln("null string should throw an exception");
            }
            catch (Exception e_3)
            {
                Logln("PASS: Constructor with null string failed as expected");
            }
            try
            {
                stdelimitersetreturn = new IBM.ICU.Util.StringTokenizer(null, delimiterset,
                                                                        false);
                Errln("null string should throw an exception");
            }
            catch (Exception e_4)
            {
                Logln("PASS: Constructor with null string failed as expected");
            }
        }
Exemplo n.º 7
0
        public void TestNextDelimiterToken()
        {
            String str = "  ,  1 2 3  AHHHHH! 5.5 6 7    ,        8\n";

            String[] expected = { "  ",       ",", "  ",  "1", " ", "2", " ", "3",    "  ",
                                  "AHHHHH!",  " ", "5.5", " ", "6", " ", "7", "    ", ",",
                                  "        ", "8\n" };
            String   delimiter = " ";

            IBM.ICU.Util.StringTokenizer tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter, true,
                                                                                      true);

            int currtoken = 0;

            while (tokenizer.HasNext())
            {
                if (!tokenizer.NextToken().Equals(expected[currtoken]))
                {
                    Errln("Error token mismatch, expected " + expected[currtoken]);
                }
                currtoken++;
            }

            if (currtoken != expected.Length)
            {
                Errln("Didn't get correct number of tokens");
            }

            tokenizer = new IBM.ICU.Util.StringTokenizer("", delimiter, true);
            if (tokenizer.HasNext())
            {
                Errln("Empty string should not have any tokens");
            }
            try
            {
                tokenizer.NextToken();
                Errln("Empty string should not have any tokens");
            }
            catch (Exception e)
            {
                Logln("PASS: Empty string failed as expected");
            }

            tokenizer = new IBM.ICU.Util.StringTokenizer(", ,", ", ", true, true);
            if (!tokenizer.HasNext())
            {
                Errln("String with only delimiters should have tokens when delimiter is treated as tokens");
            }
            if (!tokenizer.NextToken().Equals(", ,"))
            {
                Errln("String with only delimiters should return itself when delimiter is treated as tokens");
            }

            tokenizer = new IBM.ICU.Util.StringTokenizer("q, ,", ", ", true, true);

            if (!tokenizer.HasNext())
            {
                Errln("String should have some tokens");
            }
            if (!tokenizer.NextToken().Equals("q") ||
                !tokenizer.NextToken().Equals(", ,"))
            {
                Errln("String tokens do not match expected results");
            }

            try
            {
                tokenizer = new IBM.ICU.Util.StringTokenizer(null, delimiter, true);
                Errln("StringTokenizer constructed with null source should throw a nullpointerexception");
            }
            catch (Exception e_0)
            {
                Logln("PASS: StringTokenizer constructed with null source failed as expected");
            }

            tokenizer = new IBM.ICU.Util.StringTokenizer(str, "q", true);
            if (!tokenizer.NextToken().Equals(str))
            {
                Errln("Should have recieved the same string when there are no delimiters");
            }
        }
Exemplo n.º 8
0
        public void TestSupplementary()
        {
            String str       = "bmp string \ud800 with a unmatched surrogate character";
            String delimiter = "\ud800\udc00";

            String[] expected = { str };

            IBM.ICU.Util.StringTokenizer tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
            if (!tokenizer.NextToken().Equals(expected[0]))
            {
                Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
            }
            if (tokenizer.HasNext())
            {
                Errln("Number of tokens exceeded expected");
            }
            delimiter = "\ud800";
            String[] expected1 = { "bmp string ",
                                   " with a unmatched surrogate character" };
            tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
            int i = 0;

            while (tokenizer.HasNext())
            {
                if (!tokenizer.NextToken().Equals(expected1[i++]))
                {
                    Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
                }
            }
            if (tokenizer.HasNext())
            {
                Errln("Number of tokens exceeded expected");
            }

            str       = "string \ud800\udc00 with supplementary character";
            delimiter = "\ud800";
            String[] expected2 = { str };
            tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
            if (!tokenizer.NextToken().Equals(expected2[0]))
            {
                Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
            }
            if (tokenizer.HasNext())
            {
                Errln("Number of tokens exceeded expected");
            }

            delimiter = "\ud800\udc00";
            String[] expected3 = { "string ", " with supplementary character" };
            tokenizer = new IBM.ICU.Util.StringTokenizer(str, delimiter);
            i         = 0;
            while (tokenizer.HasNext())
            {
                if (!tokenizer.NextToken().Equals(expected3[i++]))
                {
                    Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
                }
            }
            if (tokenizer.HasNext())
            {
                Errln("Number of tokens exceeded expected");
            }

            str       = "\ud800 \ud800\udc00 \ud800 \ud800\udc00";
            delimiter = "\ud800";
            String[] expected4 = { " \ud800\udc00 ", " \ud800\udc00" };
            i = 0;
            while (tokenizer.HasNext())
            {
                if (!tokenizer.NextToken().Equals(expected4[i++]))
                {
                    Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
                }
            }
            if (tokenizer.HasNext())
            {
                Errln("Number of tokens exceeded expected");
            }

            delimiter = "\ud800\udc00";
            String[] expected5 = { "\ud800 ", " \ud800 " };
            i = 0;
            while (tokenizer.HasNext())
            {
                if (!tokenizer.NextToken().Equals(expected5[i++]))
                {
                    Errln("Error parsing \"" + IBM.ICU.Impl.Utility.Hex(str) + "\"");
                }
            }
            if (tokenizer.HasNext())
            {
                Errln("Number of tokens exceeded expected");
            }
        }