public virtual void TestStartOfCharEndOfDelimiter() { string path = "a/b/c/"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path)); AssertTokenStreamContents(t, new string[] { "a", "a/b", "a/b/c", "a/b/c/" }, new int[] { 0, 0, 0, 0 }, new int[] { 1, 3, 5, 6 }, new int[] { 1, 0, 0, 0 }, path.Length); }
public virtual void TestReplace() { string path = "/a/b/c"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), '/', '\\'); AssertTokenStreamContents(t, new string[] { "\\a", "\\a\\b", "\\a\\b\\c" }, new int[] { 0, 0, 0 }, new int[] { 2, 4, 6 }, new int[] { 1, 0, 0 }, path.Length); }
public virtual void TestEndOfDelimiter() { string path = "/a/b/c/"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path)); AssertTokenStreamContents(t, new string[] { "/a", "/a/b", "/a/b/c", "/a/b/c/" }, new int[] { 0, 0, 0, 0 }, new int[] { 2, 4, 6, 7 }, new int[] { 1, 0, 0, 0 }, path.Length); }
public virtual void TestOnlyDelimiterSkip() { string path = "/"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1); AssertTokenStreamContents(t, new string[] { }, new int[] { }, new int[] { }, new int[] { }, path.Length); }
public virtual void TestStartOfCharSkip() { string path = "a/b/c"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1); AssertTokenStreamContents(t, new string[] { "/b", "/b/c" }, new int[] { 1, 1 }, new int[] { 3, 5 }, new int[] { 1, 0 }, path.Length); }
public virtual void TestBasicSkip() { string path = "/a/b/c"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), 1); AssertTokenStreamContents(t, new string[] { "/b", "/b/c" }, new int[] { 2, 2 }, new int[] { 4, 6 }, new int[] { 1, 0 }, path.Length); }
public virtual void TestOnlyDelimiters() { string path = "//"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path)); AssertTokenStreamContents(t, new string[] { "/", "//" }, new int[] { 0, 0 }, new int[] { 1, 2 }, new int[] { 1, 0 }, path.Length); }
public virtual void TestWindowsPath() { string path = "c:\\a\\b\\c"; PathHierarchyTokenizer t = new PathHierarchyTokenizer(new StringReader(path), '\\', '\\'); AssertTokenStreamContents(t, new string[] { "c:", "c:\\a", "c:\\a\\b", "c:\\a\\b\\c" }, new int[] { 0, 0, 0, 0 }, new int[] { 2, 4, 6, 8 }, new int[] { 1, 0, 0, 0 }, path.Length); }
public virtual void TestRandomStrings() { Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer tokenizer = new PathHierarchyTokenizer(reader); return(new TokenStreamComponents(tokenizer, tokenizer)); }); CheckRandomData(Random, a, 1000 * RandomMultiplier); }
public virtual void TestNormalizeWinDelimToLinuxDelim() { NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder(); builder.Add("\\", "/"); NormalizeCharMap normMap = builder.Build(); string path = "c:\\a\\b\\c"; Reader cs = new MappingCharFilter(normMap, new StringReader(path)); PathHierarchyTokenizer t = new PathHierarchyTokenizer(cs); AssertTokenStreamContents(t, new string[] { "c:", "c:/a", "c:/a/b", "c:/a/b/c" }, new int[] { 0, 0, 0, 0 }, new int[] { 2, 4, 6, 8 }, new int[] { 1, 0, 0, 0 }, path.Length); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new PathHierarchyTokenizer(reader); return new TokenStreamComponents(tokenizer, tokenizer); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new PathHierarchyTokenizer(reader); return(new TokenStreamComponents(tokenizer, tokenizer)); }