Tokenizer for path-like hierarchies.

Take something like:

 /something/something/else 
and make:
 /something /something/something /something/something/else 

Inheritance: Tokenizer
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testOnlyDelimiters() throws Exception
        public virtual void testOnlyDelimiters()
        {
            string path = "//";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path));

            assertTokenStreamContents(t, new string[] { "/", "//" }, new int[] { 0, 0 }, new int[] { 1, 2 }, new int[] { 1, 0 }, path.Length);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testEndOfDelimiter() throws Exception
        public virtual void testEndOfDelimiter()
        {
            string path = "/a/b/c/";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path));

            assertTokenStreamContents(t, new string[] { "/a", "/a/b", "/a/b/c", "/a/b/c/" }, new int[] { 0, 0, 0, 0 }, new int[] { 2, 4, 6, 7 }, new int[] { 1, 0, 0, 0 }, path.Length);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testStartOfCharEndOfDelimiter() throws Exception
        public virtual void testStartOfCharEndOfDelimiter()
        {
            string path = "a/b/c/";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path));

            assertTokenStreamContents(t, new string[] { "a", "a/b", "a/b/c", "a/b/c/" }, new int[] { 0, 0, 0, 0 }, new int[] { 1, 3, 5, 6 }, new int[] { 1, 0, 0, 0 }, path.Length);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testBasicSkip() throws Exception
        public virtual void testBasicSkip()
        {
            string path = "/a/b/c";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1);

            assertTokenStreamContents(t, new string[] { "/b", "/b/c" }, new int[] { 2, 2 }, new int[] { 4, 6 }, new int[] { 1, 0 }, path.Length);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testOnlyDelimiterSkip() throws Exception
        public virtual void testOnlyDelimiterSkip()
        {
            string path = "/";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1);

            assertTokenStreamContents(t, new string[] {}, new int[] {}, new int[] {}, new int[] {}, path.Length);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testStartOfCharSkip() throws Exception
        public virtual void testStartOfCharSkip()
        {
            string path = "a/b/c";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1);

            assertTokenStreamContents(t, new string[] { "/b", "/b/c" }, new int[] { 1, 1 }, new int[] { 3, 5 }, new int[] { 1, 0 }, path.Length);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testReplace() throws Exception
        public virtual void testReplace()
        {
            string path = "/a/b/c";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), '/', '\\');

            assertTokenStreamContents(t, new string[] { "\\a", "\\a\\b", "\\a\\b\\c" }, new int[] { 0, 0, 0 }, new int[] { 2, 4, 6 }, new int[] { 1, 0, 0 }, path.Length);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testWindowsPath() throws Exception
        public virtual void testWindowsPath()
        {
            string path = "c:\\a\\b\\c";
            PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), '\\', '\\');

            assertTokenStreamContents(t, new string[] { "c:", "c:\\a", "c:\\a\\b", "c:\\a\\b\\c" }, new int[] { 0, 0, 0, 0 }, new int[] { 2, 4, 6, 8 }, new int[] { 1, 0, 0, 0 }, path.Length);
        }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testNormalizeWinDelimToLinuxDelim() throws Exception
 public virtual void testNormalizeWinDelimToLinuxDelim()
 {
     NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
     builder.add("\\", "/");
     NormalizeCharMap normMap = builder.build();
     string path = "c:\\a\\b\\c";
     Reader cs = new MappingCharFilter(normMap, new StringReader(path));
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(cs);
     assertTokenStreamContents(t, new string[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"}, new int[]{0, 0, 0, 0}, new int[]{2, 4, 6, 8}, new int[]{1, 0, 0, 0}, path.Length);
 }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testNormalizeWinDelimToLinuxDelim() throws Exception
        public virtual void testNormalizeWinDelimToLinuxDelim()
        {
            NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
            builder.add("\\", "/");
            NormalizeCharMap       normMap = builder.build();
            string                 path    = "c:\\a\\b\\c";
            Reader                 cs      = new MappingCharFilter(normMap, new StringReader(path));
            PathHierarchyTokenizer t       = new PathHierarchyTokenizer(cs);

            assertTokenStreamContents(t, new string[] { "c:", "c:/a", "c:/a/b", "c:/a/b/c" }, new int[] { 0, 0, 0, 0 }, new int[] { 2, 4, 6, 8 }, new int[] { 1, 0, 0, 0 }, path.Length);
        }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testEndOfDelimiterSkip() throws Exception
 public virtual void testEndOfDelimiterSkip()
 {
     string path = "/a/b/c/";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1);
     assertTokenStreamContents(t, new string[]{"/b", "/b/c", "/b/c/"}, new int[]{2, 2, 2}, new int[]{4, 6, 7}, new int[]{1, 0, 0}, path.Length);
 }
 protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
 {
     Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
     return new TokenStreamComponents(tokenizer, tokenizer);
 }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testBasic() throws Exception
 public virtual void testBasic()
 {
     string path = "/a/b/c";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path));
     assertTokenStreamContents(t, new string[]{"/a", "/a/b", "/a/b/c"}, new int[]{0, 0, 0}, new int[]{2, 4, 6}, new int[]{1, 0, 0}, path.Length);
 }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testWindowsPath() throws Exception
 public virtual void testWindowsPath()
 {
     string path = "c:\\a\\b\\c";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), '\\', '\\');
     assertTokenStreamContents(t, new string[]{"c:", "c:\\a", "c:\\a\\b", "c:\\a\\b\\c"}, new int[]{0, 0, 0, 0}, new int[]{2, 4, 6, 8}, new int[]{1, 0, 0, 0}, path.Length);
 }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testStartOfCharSkip() throws Exception
 public virtual void testStartOfCharSkip()
 {
     string path = "a/b/c";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1);
     assertTokenStreamContents(t, new string[]{"/b", "/b/c"}, new int[]{1, 1}, new int[]{3, 5}, new int[]{1, 0}, path.Length);
 }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testStartOfCharEndOfDelimiter() throws Exception
 public virtual void testStartOfCharEndOfDelimiter()
 {
     string path = "a/b/c/";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path));
     assertTokenStreamContents(t, new string[]{"a", "a/b", "a/b/c", "a/b/c/"}, new int[]{0, 0, 0, 0}, new int[]{1, 3, 5, 6}, new int[]{1, 0, 0, 0}, path.Length);
 }
            protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
            {
                Tokenizer tokenizer = new PathHierarchyTokenizer(reader);

                return(new TokenStreamComponents(tokenizer, tokenizer));
            }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testOnlyDelimiters() throws Exception
 public virtual void testOnlyDelimiters()
 {
     string path = "//";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path));
     assertTokenStreamContents(t, new string[]{"/", "//"}, new int[]{0, 0}, new int[]{1, 2}, new int[]{1, 0}, path.Length);
 }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testOnlyDelimiterSkip() throws Exception
 public virtual void testOnlyDelimiterSkip()
 {
     string path = "/";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1);
     assertTokenStreamContents(t, new string[]{}, new int[]{}, new int[]{}, new int[]{}, path.Length);
 }
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: public void testReplace() throws Exception
 public virtual void testReplace()
 {
     string path = "/a/b/c";
     PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), '/', '\\');
     assertTokenStreamContents(t, new string[]{"\\a", "\\a\\b", "\\a\\b\\c"}, new int[]{0, 0, 0}, new int[]{2, 4, 6}, new int[]{1, 0, 0}, path.Length);
 }