public virtual void TestStopList()
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
// {{Aroush how can we copy 'stopWordsSet' to 'System.String[]'?
System.String[] arrStopWordsSet = new System.String[3];
arrStopWordsSet[0] = "good";
arrStopWordsSet[1] = "test";
arrStopWordsSet[2] = "analyzer";
// Aroush}}
StopAnalyzer newStop = new StopAnalyzer(arrStopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsTrue(stream != null);
Token token = null;
try
{
while ((token = stream.Next()) != null)
{
System.String text = token.TermText();
Assert.IsTrue(stopWordsSet.Contains(text) == false);
}
}
catch (System.IO.IOException e)
{
Assert.IsTrue(false);
}
}
public virtual void TestStopListPositions()
{
bool defaultEnable = StopFilter.GetEnablePositionIncrementsDefault();
StopFilter.SetEnablePositionIncrementsDefault(true);
try
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
StopAnalyzer newStop = new StopAnalyzer(stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer with positions");
int[] expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 };
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
int i = 0;
TermAttribute termAtt = (TermAttribute)stream.GetAttribute(typeof(TermAttribute));
PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute)stream.AddAttribute(typeof(PositionIncrementAttribute));
while (stream.IncrementToken())
{
System.String text = termAtt.Term();
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(expectedIncr[i++], posIncrAtt.GetPositionIncrement());
}
}
finally
{
StopFilter.SetEnablePositionIncrementsDefault(defaultEnable);
}
}
public virtual void TestStopListPositions()
{
var stopWordsSet = Support.Compatibility.SetFactory.CreateHashSet <string>();
stopWordsSet.Add("good");
stopWordsSet.Add("test");
stopWordsSet.Add("analyzer");
var newStop = new StopAnalyzer(Version.LUCENE_CURRENT, stopWordsSet);
var reader = new System.IO.StringReader("This is a good test of the english stop analyzer with positions");
int[] expectedIncr = { 1, 1, 1, 3, 1, 1, 1, 2, 1 };
TokenStream stream = newStop.TokenStream("test", reader);
Assert.NotNull(stream);
int i = 0;
ITermAttribute termAtt = stream.GetAttribute <ITermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute <IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
string text = termAtt.Term;
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(expectedIncr[i++], posIncrAtt.PositionIncrement);
}
}
public virtual void TestStopList()
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
// {{Aroush how can we copy 'stopWordsSet' to 'System.String[]'?
System.String[] arrStopWordsSet = new System.String[3];
arrStopWordsSet[0] = "good";
arrStopWordsSet[1] = "test";
arrStopWordsSet[2] = "analyzer";
// Aroush}}
StopAnalyzer newStop = new StopAnalyzer(arrStopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsTrue(stream != null);
Token token = null;
try
{
while ((token = stream.Next()) != null)
{
System.String text = token.TermText();
Assert.IsTrue(stopWordsSet.Contains(text) == false);
}
}
catch (System.IO.IOException e)
{
Assert.IsTrue(false);
}
}
public virtual void TestDefaults()
{
Assert.IsTrue(stop != null);
System.IO.StringReader reader = new System.IO.StringReader("This is a test of the english stop analyzer");
TokenStream stream = stop.TokenStream("test", reader);
Assert.IsTrue(stream != null);
TermAttribute termAtt = (TermAttribute)stream.GetAttribute(typeof(TermAttribute));
while (stream.IncrementToken())
{
Assert.IsFalse(inValidTokens.Contains(termAtt.Term()));
}
}
public virtual void TestStopList()
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
StopAnalyzer newStop = new StopAnalyzer(stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
Token token = null;
while ((token = stream.Next()) != null)
{
System.String text = token.TermText();
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(1, token.GetPositionIncrement()); // by default stop tokenizer does not apply increments.
}
}
public virtual void TestStopList()
{
var stopWordsSet = Support.Compatibility.SetFactory.GetSet<string>();
stopWordsSet.Add("good");
stopWordsSet.Add("test");
stopWordsSet.Add("analyzer");
StopAnalyzer newStop = new StopAnalyzer(Version.LUCENE_24, stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
ITermAttribute termAtt = stream.GetAttribute<ITermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute<IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
System.String text = termAtt.Term;
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(1, posIncrAtt.PositionIncrement); // in 2.4 stop tokenizer does not apply increments.
}
}
public virtual void TestStopList()
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
StopAnalyzer newStop = new StopAnalyzer(stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
TermAttribute termAtt = (TermAttribute) stream.GetAttribute(typeof(TermAttribute));
PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.AddAttribute(typeof(PositionIncrementAttribute));
while (stream.IncrementToken())
{
System.String text = termAtt.Term();
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(1, posIncrAtt.GetPositionIncrement()); // by default stop tokenizer does not apply increments.
}
}
public virtual void TestDefaults()
{
Assert.IsTrue(stop != null);
System.IO.StringReader reader = new System.IO.StringReader("This is a test of the english stop analyzer");
TokenStream stream = stop.TokenStream("test", reader);
Assert.IsTrue(stream != null);
Token token = null;
try
{
while ((token = stream.Next()) != null)
{
Assert.IsTrue(inValidTokens.Contains(token.TermText()) == false);
}
}
catch (System.IO.IOException e)
{
Assert.IsTrue(false);
}
}
public virtual void TestStopList()
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
StopAnalyzer newStop = new StopAnalyzer(stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
TermAttribute termAtt = (TermAttribute)stream.GetAttribute(typeof(TermAttribute));
PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute)stream.AddAttribute(typeof(PositionIncrementAttribute));
while (stream.IncrementToken())
{
System.String text = termAtt.Term();
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(1, posIncrAtt.GetPositionIncrement()); // by default stop tokenizer does not apply increments.
}
}
public virtual void TestStopList()
{
var stopWordsSet = Support.Compatibility.SetFactory.CreateHashSet <string>();
stopWordsSet.Add("good");
stopWordsSet.Add("test");
stopWordsSet.Add("analyzer");
StopAnalyzer newStop = new StopAnalyzer(Version.LUCENE_24, stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
ITermAttribute termAtt = stream.GetAttribute <ITermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute <IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
System.String text = termAtt.Term;
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(1, posIncrAtt.PositionIncrement); // in 2.4 stop tokenizer does not apply increments.
}
}
public virtual void TestStopListPositions()
{
bool defaultEnable = StopFilter.GetEnablePositionIncrementsDefault();
StopFilter.SetEnablePositionIncrementsDefault(true);
try
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
StopAnalyzer newStop = new StopAnalyzer(stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer with positions");
int[] expectedIncr = new int[]{1, 1, 1, 3, 1, 1, 1, 2, 1};
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
Token token = null;
int i = 0;
while ((token = stream.Next()) != null)
{
System.String text = token.TermText();
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(expectedIncr[i++], token.GetPositionIncrement());
}
}
finally
{
StopFilter.SetEnablePositionIncrementsDefault(defaultEnable);
}
}
public virtual void TestStopListPositions()
{
var stopWordsSet = Support.Compatibility.SetFactory.GetSet<string>();
stopWordsSet.Add("good");
stopWordsSet.Add("test");
stopWordsSet.Add("analyzer");
var newStop = new StopAnalyzer(Version.LUCENE_CURRENT, stopWordsSet);
var reader = new System.IO.StringReader("This is a good test of the english stop analyzer with positions");
int[] expectedIncr = { 1, 1, 1, 3, 1, 1, 1, 2, 1};
TokenStream stream = newStop.TokenStream("test", reader);
Assert.NotNull(stream);
int i = 0;
ITermAttribute termAtt = stream.GetAttribute<ITermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute<IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
string text = termAtt.Term;
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(expectedIncr[i++], posIncrAtt.PositionIncrement);
}
}