public virtual void TestStopPositons()
{
StringBuilder sb = new StringBuilder();
for (int i = 10; i < 20; i++)
{
if (i % 3 != 0)
{
sb.Append(i).Append(" ");
}
else
{
string w = English.IntToEnglish(i).Trim();
sb.Append(w).Append(" ");
}
}
log(sb.ToString());
string[] stopTypes = new string[] { "<NUM>" };
ISet <string> stopSet = AsSet(stopTypes);
// with increments
StringReader reader = new StringReader(sb.ToString());
TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
TestPositons(typeTokenFilter);
// without increments
reader = new StringReader(sb.ToString());
typeTokenFilter =
#pragma warning disable 612, 618
new TypeTokenFilter(LuceneVersion.LUCENE_43,
#pragma warning restore 612, 618
false, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
TestPositons(typeTokenFilter);
}
public override TokenStream Create(TokenStream input)
{
#pragma warning disable 612, 618
TokenStream filter = new TypeTokenFilter(m_luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist);
#pragma warning restore 612, 618
return(filter);
}
public virtual void TestStopPositons()
{
StringBuilder sb = new StringBuilder();
for (int i = 10; i < 20; i++)
{
if (i % 3 != 0)
{
sb.Append(i).Append(" ");
}
else
{
string w = English.IntToEnglish(i).Trim();
sb.Append(w).Append(" ");
}
}
log(sb.ToString());
string[] stopTypes = new string[] { "<NUM>" };
ISet<string> stopSet = AsSet(stopTypes);
// with increments
StringReader reader = new StringReader(sb.ToString());
TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
TestPositons(typeTokenFilter);
// without increments
reader = new StringReader(sb.ToString());
typeTokenFilter =
#pragma warning disable 612, 618
new TypeTokenFilter(LuceneVersion.LUCENE_43,
#pragma warning restore 612, 618
false, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
TestPositons(typeTokenFilter);
}
public virtual void TestTypeFilterWhitelist()
{
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
ISet <string> stopTypes = Collections.Singleton("<NUM>");
TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
AssertTokenStreamContents(stream, new string[] { "121", "123" });
}
public virtual void TestTypeFilter()
{
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
ISet <string> stopTypes = AsSet("<NUM>");
TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
AssertTokenStreamContents(stream, new string[] { "is", "palindrome", "while", "is", "not" });
}
public virtual void TestTypeFilter()
{
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
ISet<string> stopTypes = AsSet("<NUM>");
TokenStream stream =
#pragma warning disable 612, 618
new TypeTokenFilter(
#pragma warning restore 612, 618
TEST_VERSION_CURRENT, true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
AssertTokenStreamContents(stream, new string[] { "is", "palindrome", "while", "is", "not" });
}
private void TestPositons(TypeTokenFilter stpf)
{
ITypeAttribute typeAtt = stpf.GetAttribute<ITypeAttribute>();
ICharTermAttribute termAttribute = stpf.GetAttribute<ICharTermAttribute>();
IPositionIncrementAttribute posIncrAtt = stpf.GetAttribute<IPositionIncrementAttribute>();
stpf.Reset();
bool enablePositionIncrements = stpf.EnablePositionIncrements;
while (stpf.IncrementToken())
{
log("Token: " + termAttribute.ToString() + ": " + typeAtt.Type + " - " + posIncrAtt.PositionIncrement);
assertEquals("if position increment is enabled the positionIncrementAttribute value should be 3, otherwise 1", posIncrAtt.PositionIncrement, enablePositionIncrements ? 3 : 1);
}
stpf.End();
stpf.Dispose();
}
private void TestPositons(TypeTokenFilter stpf)
{
ITypeAttribute typeAtt = stpf.GetAttribute <ITypeAttribute>();
ICharTermAttribute termAttribute = stpf.GetAttribute <ICharTermAttribute>();
IPositionIncrementAttribute posIncrAtt = stpf.GetAttribute <IPositionIncrementAttribute>();
stpf.Reset();
bool enablePositionIncrements = stpf.EnablePositionIncrements;
while (stpf.IncrementToken())
{
log("Token: " + termAttribute.ToString() + ": " + typeAtt.Type + " - " + posIncrAtt.PositionIncrement);
assertEquals("if position increment is enabled the positionIncrementAttribute value should be 3, otherwise 1", posIncrAtt.PositionIncrement, enablePositionIncrements ? 3 : 1);
}
stpf.End();
stpf.Dispose();
}
public override TokenStream Create(TokenStream input)
{
#pragma warning disable 612, 618
TokenStream filter = new TypeTokenFilter(luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist);
#pragma warning restore 612, 618
return filter;
}
public override TokenStream Create(TokenStream input)
{
TokenStream filter = new TypeTokenFilter(luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist);
return(filter);
}
public virtual void TestTypeFilterWhitelist()
{
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
ISet<string> stopTypes = Collections.Singleton("<NUM>");
TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
AssertTokenStreamContents(stream, new string[] { "121", "123" });
}
public override TokenStream Create(TokenStream input)
{
TokenStream filter = new TypeTokenFilter(luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist);
return filter;
}
