public virtual void TestStopPositons() { StringBuilder sb = new StringBuilder(); for (int i = 10; i < 20; i++) { if (i % 3 != 0) { sb.Append(i).Append(" "); } else { string w = English.IntToEnglish(i).Trim(); sb.Append(w).Append(" "); } } log(sb.ToString()); string[] stopTypes = new string[] { "<NUM>" }; ISet <string> stopSet = AsSet(stopTypes); // with increments StringReader reader = new StringReader(sb.ToString()); TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet); TestPositons(typeTokenFilter); // without increments reader = new StringReader(sb.ToString()); typeTokenFilter = #pragma warning disable 612, 618 new TypeTokenFilter(LuceneVersion.LUCENE_43, #pragma warning restore 612, 618 false, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet); TestPositons(typeTokenFilter); }
public override TokenStream Create(TokenStream input) { #pragma warning disable 612, 618 TokenStream filter = new TypeTokenFilter(m_luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist); #pragma warning restore 612, 618 return(filter); }
public virtual void TestStopPositons() { StringBuilder sb = new StringBuilder(); for (int i = 10; i < 20; i++) { if (i % 3 != 0) { sb.Append(i).Append(" "); } else { string w = English.IntToEnglish(i).Trim(); sb.Append(w).Append(" "); } } log(sb.ToString()); string[] stopTypes = new string[] { "<NUM>" }; ISet<string> stopSet = AsSet(stopTypes); // with increments StringReader reader = new StringReader(sb.ToString()); TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet); TestPositons(typeTokenFilter); // without increments reader = new StringReader(sb.ToString()); typeTokenFilter = #pragma warning disable 612, 618 new TypeTokenFilter(LuceneVersion.LUCENE_43, #pragma warning restore 612, 618 false, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet); TestPositons(typeTokenFilter); }
public virtual void TestTypeFilterWhitelist() { StringReader reader = new StringReader("121 is palindrome, while 123 is not"); ISet <string> stopTypes = Collections.Singleton("<NUM>"); TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true); AssertTokenStreamContents(stream, new string[] { "121", "123" }); }
public virtual void TestTypeFilter() { StringReader reader = new StringReader("121 is palindrome, while 123 is not"); ISet <string> stopTypes = AsSet("<NUM>"); TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes); AssertTokenStreamContents(stream, new string[] { "is", "palindrome", "while", "is", "not" }); }
public virtual void TestTypeFilter() { StringReader reader = new StringReader("121 is palindrome, while 123 is not"); ISet<string> stopTypes = AsSet("<NUM>"); TokenStream stream = #pragma warning disable 612, 618 new TypeTokenFilter( #pragma warning restore 612, 618 TEST_VERSION_CURRENT, true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes); AssertTokenStreamContents(stream, new string[] { "is", "palindrome", "while", "is", "not" }); }
private void TestPositons(TypeTokenFilter stpf) { ITypeAttribute typeAtt = stpf.GetAttribute<ITypeAttribute>(); ICharTermAttribute termAttribute = stpf.GetAttribute<ICharTermAttribute>(); IPositionIncrementAttribute posIncrAtt = stpf.GetAttribute<IPositionIncrementAttribute>(); stpf.Reset(); bool enablePositionIncrements = stpf.EnablePositionIncrements; while (stpf.IncrementToken()) { log("Token: " + termAttribute.ToString() + ": " + typeAtt.Type + " - " + posIncrAtt.PositionIncrement); assertEquals("if position increment is enabled the positionIncrementAttribute value should be 3, otherwise 1", posIncrAtt.PositionIncrement, enablePositionIncrements ? 3 : 1); } stpf.End(); stpf.Dispose(); }
private void TestPositons(TypeTokenFilter stpf) { ITypeAttribute typeAtt = stpf.GetAttribute <ITypeAttribute>(); ICharTermAttribute termAttribute = stpf.GetAttribute <ICharTermAttribute>(); IPositionIncrementAttribute posIncrAtt = stpf.GetAttribute <IPositionIncrementAttribute>(); stpf.Reset(); bool enablePositionIncrements = stpf.EnablePositionIncrements; while (stpf.IncrementToken()) { log("Token: " + termAttribute.ToString() + ": " + typeAtt.Type + " - " + posIncrAtt.PositionIncrement); assertEquals("if position increment is enabled the positionIncrementAttribute value should be 3, otherwise 1", posIncrAtt.PositionIncrement, enablePositionIncrements ? 3 : 1); } stpf.End(); stpf.Dispose(); }
public override TokenStream Create(TokenStream input) { #pragma warning disable 612, 618 TokenStream filter = new TypeTokenFilter(luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist); #pragma warning restore 612, 618 return filter; }
public override TokenStream Create(TokenStream input) { TokenStream filter = new TypeTokenFilter(luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist); return(filter); }
public virtual void TestTypeFilterWhitelist() { StringReader reader = new StringReader("121 is palindrome, while 123 is not"); ISet<string> stopTypes = Collections.Singleton("<NUM>"); TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true); AssertTokenStreamContents(stream, new string[] { "121", "123" }); }
public override TokenStream Create(TokenStream input) { TokenStream filter = new TypeTokenFilter(luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist); return filter; }