public virtual void Test()
{
TokenRangeSinkFilter sinkFilter = new TokenRangeSinkFilter(2, 4);
string test = "The quick red fox jumped over the lazy brown dogs";
TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
TeeSinkTokenFilter.SinkTokenStream rangeToks = tee.NewSinkTokenStream(sinkFilter);
int count = 0;
tee.Reset();
while (tee.IncrementToken())
{
count++;
}
int sinkCount = 0;
rangeToks.Reset();
while (rangeToks.IncrementToken())
{
sinkCount++;
}
assertTrue(count + " does not equal: " + 10, count == 10);
assertTrue("rangeToks Size: " + sinkCount + " is not: " + 2, sinkCount == 2);
}
public virtual void TestLooseDateFormat()
{
DateRecognizerSinkFilter sinkFilter = new DateRecognizerSinkFilter(CultureInfo.InvariantCulture);
string test = "The quick red fox jumped over the lazy brown dogs on 7/11/2006 The dogs finally reacted on 7/2/2006";
TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
TeeSinkTokenFilter.SinkTokenStream sink = tee.NewSinkTokenStream(sinkFilter);
int count = 0;
tee.Reset();
while (tee.IncrementToken())
{
count++;
}
assertTrue(count + " does not equal: " + 18, count == 18);
int sinkCount = 0;
sink.Reset();
while (sink.IncrementToken())
{
sinkCount++;
}
assertTrue("sink Size: " + sinkCount + " is not: " + 2, sinkCount == 2);
}
public virtual void Test()
{
TokenTypeSinkFilter sinkFilter = new TokenTypeSinkFilter("D");
string test = "The quick red fox jumped over the lazy brown dogs";
TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
TeeSinkTokenFilter.SinkTokenStream sink = ttf.NewSinkTokenStream(sinkFilter);
bool seenDogs = false;
ICharTermAttribute termAtt = ttf.AddAttribute <ICharTermAttribute>();
ITypeAttribute typeAtt = ttf.AddAttribute <ITypeAttribute>();
ttf.Reset();
while (ttf.IncrementToken())
{
if (termAtt.ToString().Equals("dogs", StringComparison.Ordinal))
{
seenDogs = true;
assertTrue(typeAtt.Type + " is not equal to " + "D", typeAtt.Type.Equals("D", StringComparison.Ordinal) == true);
}
else
{
assertTrue(typeAtt.Type + " is not null and it should be", typeAtt.Type.Equals("word", StringComparison.Ordinal));
}
}
assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
int sinkCount = 0;
sink.Reset();
while (sink.IncrementToken())
{
sinkCount++;
}
assertTrue("sink Size: " + sinkCount + " is not: " + 1, sinkCount == 1);
}
public virtual void Test()
{
TokenTypeSinkFilter sinkFilter = new TokenTypeSinkFilter("D");
string test = "The quick red fox jumped over the lazy brown dogs";
TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
TeeSinkTokenFilter.SinkTokenStream sink = ttf.NewSinkTokenStream(sinkFilter);
bool seenDogs = false;
ICharTermAttribute termAtt = ttf.AddAttribute<ICharTermAttribute>();
ITypeAttribute typeAtt = ttf.AddAttribute<ITypeAttribute>();
ttf.Reset();
while (ttf.IncrementToken())
{
if (termAtt.ToString().Equals("dogs"))
{
seenDogs = true;
assertTrue(typeAtt.Type + " is not equal to " + "D", typeAtt.Type.Equals("D") == true);
}
else
{
assertTrue(typeAtt.Type + " is not null and it should be", typeAtt.Type.Equals("word"));
}
}
assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
int sinkCount = 0;
sink.Reset();
while (sink.IncrementToken())
{
sinkCount++;
}
assertTrue("sink Size: " + sinkCount + " is not: " + 1, sinkCount == 1);
}
public virtual void TestLooseDateFormat()
{
DateRecognizerSinkFilter sinkFilter = new DateRecognizerSinkFilter(CultureInfo.InvariantCulture);
string test = "The quick red fox jumped over the lazy brown dogs on 7/11/2006 The dogs finally reacted on 7/2/2006";
TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
TeeSinkTokenFilter.SinkTokenStream sink = tee.NewSinkTokenStream(sinkFilter);
int count = 0;
tee.Reset();
while (tee.IncrementToken())
{
count++;
}
assertTrue(count + " does not equal: " + 18, count == 18);
int sinkCount = 0;
sink.Reset();
while (sink.IncrementToken())
{
sinkCount++;
}
assertTrue("sink Size: " + sinkCount + " is not: " + 2, sinkCount == 2);
}
public virtual void Test()
{
TokenRangeSinkFilter sinkFilter = new TokenRangeSinkFilter(2, 4);
string test = "The quick red fox jumped over the lazy brown dogs";
TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
TeeSinkTokenFilter.SinkTokenStream rangeToks = tee.NewSinkTokenStream(sinkFilter);
int count = 0;
tee.Reset();
while (tee.IncrementToken())
{
count++;
}
int sinkCount = 0;
rangeToks.Reset();
while (rangeToks.IncrementToken())
{
sinkCount++;
}
assertTrue(count + " does not equal: " + 10, count == 10);
assertTrue("rangeToks Size: " + sinkCount + " is not: " + 2, sinkCount == 2);
}
/// <summary>
/// Not an explicit test, just useful to print out some info on performance
/// </summary>
public virtual void Performance()
{
int[] tokCount = new int[] { 100, 500, 1000, 2000, 5000, 10000 };
int[] modCounts = new int[] { 1, 2, 5, 10, 20, 50, 100, 200, 500 };
for (int k = 0; k < tokCount.Length; k++)
{
StringBuilder buffer = new StringBuilder();
Console.WriteLine("-----Tokens: " + tokCount[k] + "-----");
for (int i = 0; i < tokCount[k]; i++)
{
//buffer.Append(English.intToEnglish(i).toUpperCase(Locale.ROOT)).Append(' ');
buffer.Append(i.ToString(CultureInfo.InvariantCulture)).Append(' ');
}
//make sure we produce the same tokens
TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))));
TokenStream sink = teeStream.NewSinkTokenStream(new ModuloSinkFilter(this, 100));
teeStream.ConsumeAllTokens();
TokenStream stream = new ModuloTokenFilter(this, new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))), 100);
ICharTermAttribute tfTok = stream.AddAttribute <ICharTermAttribute>();
ICharTermAttribute sinkTok = sink.AddAttribute <ICharTermAttribute>();
for (int i = 0; stream.IncrementToken(); i++)
{
assertTrue(sink.IncrementToken());
assertTrue(tfTok + " is not equal to " + sinkTok + " at token: " + i, tfTok.Equals(sinkTok) == true);
}
//simulate two fields, each being analyzed once, for 20 documents
for (int j = 0; j < modCounts.Length; j++)
{
int tfPos = 0;
//long start = DateTimeHelperClass.CurrentUnixTimeMillis();
long start = Environment.TickCount;
for (int i = 0; i < 20; i++)
{
stream = new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString())));
IPositionIncrementAttribute posIncrAtt = stream.GetAttribute <IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
tfPos += posIncrAtt.PositionIncrement;
}
stream = new ModuloTokenFilter(this, new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))), modCounts[j]);
posIncrAtt = stream.GetAttribute <IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
tfPos += posIncrAtt.PositionIncrement;
}
}
//long finish = DateTimeHelperClass.CurrentUnixTimeMillis();
long finish = Environment.TickCount;
Console.WriteLine("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
int sinkPos = 0;
//simulate one field with one sink
//start = DateTimeHelperClass.CurrentUnixTimeMillis();
start = Environment.TickCount;
for (int i = 0; i < 20; i++)
{
teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))));
sink = teeStream.NewSinkTokenStream(new ModuloSinkFilter(this, modCounts[j]));
IPositionIncrementAttribute posIncrAtt = teeStream.GetAttribute <IPositionIncrementAttribute>();
while (teeStream.IncrementToken())
{
sinkPos += posIncrAtt.PositionIncrement;
}
//System.out.println("Modulo--------");
posIncrAtt = sink.GetAttribute <IPositionIncrementAttribute>();
while (sink.IncrementToken())
{
sinkPos += posIncrAtt.PositionIncrement;
}
}
//finish = DateTimeHelperClass.CurrentUnixTimeMillis();
finish = Environment.TickCount;
Console.WriteLine("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
assertTrue(sinkPos + " does not equal: " + tfPos, sinkPos == tfPos);
}
Console.WriteLine("- End Tokens: " + tokCount[k] + "-----");
}
}
