public void TestMethod1()
{
Lucene.Net.Analysis.Standard.StandardAnalyzer a = new Lucene.Net.Analysis.Standard.StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
string s = "我日中华人民共和国";
System.IO.StringReader reader = new System.IO.StringReader(s);
Lucene.Net.Analysis.TokenStream ts = a.TokenStream(s, reader);
bool hasnext = ts.IncrementToken();
Lucene.Net.Analysis.Tokenattributes.ITermAttribute ita;
while (hasnext)
{
ita = ts.GetAttribute <Lucene.Net.Analysis.Tokenattributes.ITermAttribute>();
Console.WriteLine(ita.Term);
hasnext = ts.IncrementToken();
}
Console.WriteLine("over");
ts.CloneAttributes();
reader.Close();
a.Close();
}
public void v()
{
//Analyzer analyzer = new CJKAnalyzer();
//TokenStream tokenStream = analyzer.TokenStream("", new StringReader("我爱你中国China中华人名共和国"));
//Lucene.Net.Analysis.Token token = null;
//while ((token = tokenStream.Next()) != null)
//{
// Response.Write(token.TermText() + "<br/>");
//}
Lucene.Net.Analysis.Standard.StandardAnalyzer a = new Lucene.Net.Analysis.Standard.StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
string s = "我日中华人民共和国";
System.IO.StringReader reader = new System.IO.StringReader(s);
Lucene.Net.Analysis.TokenStream ts = a.TokenStream(s, reader);
bool hasnext = ts.IncrementToken();
Lucene.Net.Analysis.Tokenattributes.ITermAttribute ita;
while (hasnext)
{
ita = ts.GetAttribute <Lucene.Net.Analysis.Tokenattributes.ITermAttribute>();
Console.WriteLine(ita.Term);
hasnext = ts.IncrementToken();
}
ts.CloneAttributes();
reader.Close();
a.Close();
Console.ReadKey();
}
public static Token NextToken(TokenStream input, Token reusableToken)
{
if (input == null)
return null;
if (!input.IncrementToken())
return null;
ITermAttribute termAtt = input.GetAttribute<ITermAttribute>();
IOffsetAttribute offsetAtt = input.GetAttribute<IOffsetAttribute>();
ITypeAttribute typeAtt = input.GetAttribute<ITypeAttribute>();
if (reusableToken == null)
{
reusableToken = new Token();
}
reusableToken.Clear();
if (termAtt != null)
reusableToken.SetTermBuffer(termAtt.TermBuffer(), 0, termAtt.TermLength());
if (offsetAtt != null)
{
reusableToken.StartOffset = offsetAtt.StartOffset;
reusableToken.EndOffset = offsetAtt.EndOffset;
}
if (typeAtt != null)
reusableToken.Type = typeAtt.Type;
return reusableToken;
}
/// <summary> Adds term frequencies found by tokenizing text from reader into the Map words</summary>
/// <param name="r">a source of text to be tokenized
/// </param>
/// <param name="termFreqMap">a Map of terms and their frequencies
/// </param>
/// <param name="fieldName">Used by analyzer for any special per-field analysis
/// </param>
protected void AddTermFrequencies(System.IO.TextReader r, System.Collections.IDictionary termFreqMap, System.String fieldName)
{
TokenStream ts = analyzer.TokenStream(fieldName, r);
var termAtt = ts.AddAttribute <ITermAttribute>();
int tokenCount = 0;
while (ts.IncrementToken())
{
// for every token
System.String word = termAtt.Term;
tokenCount++;
if (tokenCount > maxNumTokensParsed)
{
break;
}
if (IsNoiseWord(word))
{
continue;
}
// increment frequency
Int cnt = (Int)termFreqMap[word];
if (cnt == null)
{
termFreqMap[word] = new Int();
}
else
{
cnt.x++;
}
}
}
public string GetTokenView(TokenStream tokenStream, out int numberOfTokens)
{
var sb = new StringBuilder();
numberOfTokens = 0;
var termAttr = tokenStream.GetAttribute<ITermAttribute>();
var startOffset = tokenStream.GetAttribute<Lucene.Net.Analysis.Tokenattributes.IOffsetAttribute>();
while (tokenStream.IncrementToken())
{
sb.Append(termAttr.Term + " Start: " + startOffset.StartOffset.ToString().PadLeft(5) + " End: " + startOffset.EndOffset.ToString().PadLeft(5) + "\r\n");
//var view = "[" + termAttr.Term + "] ";
//sb.Append(view);
numberOfTokens++;
}
return sb.ToString();
//StringBuilder sb = new StringBuilder();
//Token token = tokenStream.Next();
//numberOfTokens = 0;
//while (token != null)
//{
// numberOfTokens++;
// sb.Append(token.TermText() + " Start: " + token.StartOffset().ToString().PadLeft(5) + " End: " + token.EndOffset().ToString().PadLeft(5) + "\r\n");
// token = tokenStream.Next();
//}
//return sb.ToString();
}
private static void DisplayTokens(TokenStream stream)
{
TermAttribute term = (TermAttribute) stream.AddAttribute(typeof(TermAttribute));
while (stream.IncrementToken())
{
Console.WriteLine("[{0}] ", term.Term());
}
}
/// <summary>
/// 分词测试
/// </summary>
/// <param name="keyword"></param>
/// <returns></returns>
public string Token(string keyword)
{
string ret = "";
System.IO.StringReader reader = new System.IO.StringReader(keyword);
Lucene.Net.Analysis.TokenStream ts = analyzer.TokenStream(keyword, reader);
bool hasNext = ts.IncrementToken();
Lucene.Net.Analysis.Tokenattributes.ITermAttribute ita;
while (hasNext)
{
ita = ts.GetAttribute <Lucene.Net.Analysis.Tokenattributes.ITermAttribute>();
ret += ita.Term + "|";
hasNext = ts.IncrementToken();
}
ts.CloneAttributes();
reader.Close();
analyzer.Close();
return(ret);
}
public static void DisplayTokens(TokenStream stream)
{
// error in Lucene.Net? should work, look in source code why not
// source: Lucene in Action, page ??
var term = stream.AddAttribute<TermAttribute>();
while (stream.IncrementToken()) {
#if LuceneV303
Trace.WriteLine("[" + term.Term + "] ");
#endif
}
}
/// <summary>
/// 分词方法
/// </summary>
/// <param name="words">待分词内容</param>
/// <param name="analyzer"></param>
/// <returns></returns>
private string cutWords(string words, Analyzer analyzer)
{
string resultStr = "";
System.IO.StringReader reader = new System.IO.StringReader(words);
Lucene.Net.Analysis.TokenStream ts = analyzer.TokenStream(words, reader);
bool hasNext = ts.IncrementToken();
Lucene.Net.Analysis.Tokenattributes.ITermAttribute ita;
while (hasNext)
{
ita = ts.GetAttribute <Lucene.Net.Analysis.Tokenattributes.ITermAttribute>();
resultStr += ita.Term + "|";
hasNext = ts.IncrementToken();
}
ts.CloneAttributes();
reader.Close();
analyzer.Close();
return(resultStr);
}
internal virtual void VerifyPayload(TokenStream ts)
{
IPayloadAttribute payloadAtt = ts.GetAttribute<IPayloadAttribute>();
for (byte b = 1; ; b++)
{
bool hasNext = ts.IncrementToken();
if (!hasNext)
break;
// System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
// System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
Assert.AreEqual(b, payloadAtt.Payload.ToByteArray()[0]);
}
}
public virtual void TestDefaults()
{
Assert.IsTrue(stop != null);
System.IO.StringReader reader = new System.IO.StringReader("This is a test of the english stop analyzer");
TokenStream stream = stop.TokenStream("test", reader);
Assert.IsTrue(stream != null);
TermAttribute termAtt = (TermAttribute)stream.GetAttribute(typeof(TermAttribute));
while (stream.IncrementToken())
{
Assert.IsFalse(inValidTokens.Contains(termAtt.Term()));
}
}
public override void Run()
{
try
{
foreach (var mapping in this.map)
{
string term = mapping.Key;
BytesRef expected = mapping.Value;
Exception priorException = null; // LUCENENET: No need to cast to IOExcpetion
TokenStream ts = this.analyzer.GetTokenStream("fake", new StringReader(term));
try
{
ITermToBytesRefAttribute termAtt = ts.AddAttribute <ITermToBytesRefAttribute>();
BytesRef bytes = termAtt.BytesRef;
ts.Reset();
Assert.IsTrue(ts.IncrementToken());
termAtt.FillBytesRef();
Assert.AreEqual(expected, bytes);
Assert.IsFalse(ts.IncrementToken());
ts.End();
}
catch (Exception e) when(e.IsIOException())
{
priorException = e;
}
finally
{
IOUtils.DisposeWhileHandlingException(priorException, ts);
}
}
}
catch (Exception e) when(e.IsIOException())
{
throw RuntimeException.Create(e);
}
}
public override void Run()
{
try
{
foreach (KeyValuePair <string, BytesRef> mapping in Map)
{
string term = mapping.Key;
BytesRef expected = mapping.Value;
IOException priorException = null;
TokenStream ts = Analyzer.TokenStream("fake", new StreamReader(term));
try
{
ITermToBytesRefAttribute termAtt = ts.AddAttribute <ITermToBytesRefAttribute>();
BytesRef bytes = termAtt.BytesRef;
ts.Reset();
Assert.IsTrue(ts.IncrementToken());
termAtt.FillBytesRef();
Assert.AreEqual(expected, bytes);
Assert.IsFalse(ts.IncrementToken());
ts.End();
}
catch (IOException e)
{
priorException = e;
}
finally
{
IOUtils.CloseWhileHandlingException(priorException, ts);
}
}
}
catch (IOException e)
{
throw (Exception)e;
}
}
private void CheckTokens(TokenStream stream)
{
int count = 0;
ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>();
while (stream.IncrementToken())
{
Assert.IsTrue(count < Tokens.Length);
Assert.AreEqual(Tokens[count], termAtt.ToString());
count++;
}
Assert.AreEqual(Tokens.Length, count);
}
private void checkTokens(TokenStream stream)
{
int count = 0;
TermAttribute termAtt = (TermAttribute)stream.GetAttribute(typeof(TermAttribute));
Assert.IsNotNull(termAtt);
while (stream.IncrementToken())
{
Assert.IsTrue(count < tokens.Length);
Assert.AreEqual(tokens[count], termAtt.Term());
count++;
}
Assert.AreEqual(tokens.Length, count);
}
internal virtual void VerifyPayload(TokenStream ts)
{
IPayloadAttribute payloadAtt = ts.GetAttribute <IPayloadAttribute>();
for (byte b = 1; ; b++)
{
bool hasNext = ts.IncrementToken();
if (!hasNext)
{
break;
}
// System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
// System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
Assert.AreEqual(b, payloadAtt.Payload.ToByteArray()[0]);
}
}
public void TestLUCENE_3042()
{
String testString = "t";
Analyzer analyzer = new MockAnalyzer(Random);
using (TokenStream stream = analyzer.GetTokenStream("dummy", testString))
{
stream.Reset();
while (stream.IncrementToken())
{
// consume
}
stream.End();
}
AssertAnalyzesTo(analyzer, testString, new String[] { "t" });
}
public void TestForwardOffsets()
{
int num = AtLeast(1000);
for (int i = 0; i < num; i++)
{
String s = TestUtil.RandomHtmlishString(Random, 20);
StringReader reader = new StringReader(s);
MockCharFilter charfilter = new MockCharFilter(reader, 2);
MockAnalyzer analyzer = new MockAnalyzer(Random);
using TokenStream ts = analyzer.GetTokenStream("bogus", charfilter);
ts.Reset();
while (ts.IncrementToken())
{
;
}
ts.End();
}
}
public static List <string> SplitWords(string content)
{
List <string> strList = new List <string>();
using (Analyzer analyzer = new PanGuAnalyzer())//指定使用盘古 PanGuAnalyzer 分词算法
{
using (System.IO.StringReader reader = new System.IO.StringReader(content))
{
Lucene.Net.Analysis.TokenStream ts = analyzer.TokenStream(content, reader);
while (ts.IncrementToken())
{
var ita = ts.GetAttribute <Lucene.Net.Analysis.Tokenattributes.ITermAttribute>();
strList.Add(ita.Term);
}
ts.CloneAttributes();
}
}
return(strList);
}
public virtual void TestStopList()
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
stopWordsSet.Add("good", "good");
stopWordsSet.Add("test", "test");
stopWordsSet.Add("analyzer", "analyzer");
StopAnalyzer newStop = new StopAnalyzer(stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
TermAttribute termAtt = (TermAttribute)stream.GetAttribute(typeof(TermAttribute));
PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute)stream.AddAttribute(typeof(PositionIncrementAttribute));
while (stream.IncrementToken())
{
System.String text = termAtt.Term();
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(1, posIncrAtt.GetPositionIncrement()); // by default stop tokenizer does not apply increments.
}
}
public virtual void TestStopList()
{
var stopWordsSet = Support.Compatibility.SetFactory.CreateHashSet <string>();
stopWordsSet.Add("good");
stopWordsSet.Add("test");
stopWordsSet.Add("analyzer");
StopAnalyzer newStop = new StopAnalyzer(Version.LUCENE_24, stopWordsSet);
System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.TokenStream("test", reader);
Assert.IsNotNull(stream);
ITermAttribute termAtt = stream.GetAttribute <ITermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute <IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
System.String text = termAtt.Term;
Assert.IsFalse(stopWordsSet.Contains(text));
Assert.AreEqual(1, posIncrAtt.PositionIncrement); // in 2.4 stop tokenizer does not apply increments.
}
}
private static void ConsumeStreamNewAPI(TokenStream stream)
{
stream.Reset();
PayloadAttribute payloadAtt = (PayloadAttribute)stream.AddAttribute(typeof(PayloadAttribute));
TermAttribute termAtt = (TermAttribute)stream.AddAttribute(typeof(TermAttribute));
int i = 0;
while (stream.IncrementToken())
{
System.String term = termAtt.Term();
Payload p = payloadAtt.GetPayload();
if (p != null && p.GetData().Length == 1 && p.GetData()[0] == PartOfSpeechAnnotatingFilter.PROPER_NOUN_ANNOTATION)
{
Assert.IsTrue("tokenstream".Equals(term), "only TokenStream is a proper noun");
}
else
{
Assert.IsFalse("tokenstream".Equals(term), "all other tokens (if this test fails, the special POSToken subclass is not correctly passed through the chain)");
}
Assert.AreEqual(results[i], term);
i++;
}
}
internal virtual void AssertTermEquals(System.String expected, TokenStream stream, TermAttribute termAtt)
{
Assert.IsTrue(stream.IncrementToken());
Assert.AreEqual(expected, termAtt.Term());
}
public string GetView(TokenStream tokenStream, out int numberOfTokens)
{
var sb = new StringBuilder();
var termDictionary = new Dictionary<string, int>();
var termAttr = tokenStream.GetAttribute<ITermAttribute>();
while (tokenStream.IncrementToken())
{
if (termDictionary.Keys.Contains(termAttr.Term))
termDictionary[termAttr.Term] = termDictionary[termAttr.Term] + 1;
else
termDictionary.Add(termAttr.Term, 1);
}
foreach (var item in termDictionary.OrderBy(x => x.Key))
{
sb.Append(item.Key + " [" + item.Value + "] ");
}
numberOfTokens = termDictionary.Count;
return sb.ToString();
}
public static void AssertTokenStreamContents(TokenStream ts, System.String[] output, int[] startOffsets, int[] endOffsets, System.String[] types, int[] posIncrements, int? finalOffset)
{
Assert.IsNotNull(output);
ICheckClearAttributesAttribute checkClearAtt = ts.AddAttribute<ICheckClearAttributesAttribute>();
Assert.IsTrue(ts.HasAttribute<ITermAttribute>(), "has no TermAttribute");
ITermAttribute termAtt = ts.GetAttribute<ITermAttribute>();
IOffsetAttribute offsetAtt = null;
if (startOffsets != null || endOffsets != null || finalOffset != null)
{
Assert.IsTrue(ts.HasAttribute<IOffsetAttribute>(), "has no OffsetAttribute");
offsetAtt = ts.GetAttribute<IOffsetAttribute>();
}
ITypeAttribute typeAtt = null;
if (types != null)
{
Assert.IsTrue(ts.HasAttribute<ITypeAttribute>(), "has no TypeAttribute");
typeAtt = ts.GetAttribute<ITypeAttribute>();
}
IPositionIncrementAttribute posIncrAtt = null;
if (posIncrements != null)
{
Assert.IsTrue(ts.HasAttribute<IPositionIncrementAttribute>(), "has no PositionIncrementAttribute");
posIncrAtt = ts.GetAttribute<IPositionIncrementAttribute>();
}
ts.Reset();
for (int i = 0; i < output.Length; i++)
{
// extra safety to enforce, that the state is not preserved and also assign bogus values
ts.ClearAttributes();
termAtt.SetTermBuffer("bogusTerm");
if (offsetAtt != null) offsetAtt.SetOffset(14584724, 24683243);
if (typeAtt != null) typeAtt.Type = "bogusType";
if (posIncrAtt != null) posIncrAtt.PositionIncrement = 45987657;
checkClearAtt.GetAndResetClearCalled(); // reset it, because we called clearAttribute() before
Assert.IsTrue(ts.IncrementToken(), "token " + i + " does not exist");
Assert.IsTrue(checkClearAtt.GetAndResetClearCalled(), "clearAttributes() was not called correctly in TokenStream chain");
Assert.AreEqual(output[i], termAtt.Term, "term " + i);
if (startOffsets != null)
Assert.AreEqual(startOffsets[i], offsetAtt.StartOffset, "startOffset " + i);
if (endOffsets != null)
Assert.AreEqual(endOffsets[i], offsetAtt.EndOffset, "endOffset " + i);
if (types != null)
Assert.AreEqual(types[i], typeAtt.Type, "type " + i);
if (posIncrements != null)
Assert.AreEqual(posIncrements[i], posIncrAtt.PositionIncrement, "posIncrement " + i);
}
Assert.IsFalse(ts.IncrementToken(), "end of stream");
ts.End();
if (finalOffset.HasValue)
Assert.AreEqual(finalOffset, offsetAtt.EndOffset, "finalOffset ");
ts.Close();
}
public virtual void ToDot()
{
@in.Reset();
WriteHeader();
// TODO: is there some way to tell dot that it should
// make the "main path" a straight line and have the
// non-sausage arcs not affect node placement...
int pos = -1;
int lastEndPos = -1;
while (@in.IncrementToken())
{
bool isFirst = pos == -1;
int posInc = PosIncAtt.PositionIncrement;
if (isFirst && posInc == 0)
{
// TODO: hmm are TS's still allowed to do this...?
Console.Error.WriteLine("WARNING: first posInc was 0; correcting to 1");
posInc = 1;
}
if (posInc > 0)
{
// New node:
pos += posInc;
WriteNode(pos, Convert.ToString(pos));
}
if (posInc > 1)
{
// Gap!
WriteArc(lastEndPos, pos, null, "dotted");
}
if (isFirst)
{
WriteNode(-1, null);
WriteArc(-1, pos, null, null);
}
string arcLabel = TermAtt.ToString();
if (OffsetAtt != null)
{
int startOffset = OffsetAtt.StartOffset;
int endOffset = OffsetAtt.EndOffset;
//System.out.println("start=" + startOffset + " end=" + endOffset + " len=" + inputText.length());
if (InputText != null)
{
arcLabel += " / " + InputText.Substring(startOffset, endOffset - startOffset);
}
else
{
arcLabel += " / " + startOffset + "-" + endOffset;
}
}
WriteArc(pos, pos + PosLengthAtt.PositionLength, arcLabel, null);
lastEndPos = pos + PosLengthAtt.PositionLength;
}
@in.End();
if (lastEndPos != -1)
{
// TODO: should we output any final text (from end
// offsets) on this arc...?
WriteNode(-2, null);
WriteArc(lastEndPos, -2, null, null);
}
WriteTrailer();
}
private static void AssertNext(TokenStream ts, String text, int positionIncrement, float boost, int startOffset,
int endOffset)
{
var termAtt = ts.AddAttribute<ITermAttribute>();
var posIncrAtt = ts.AddAttribute<IPositionIncrementAttribute>();
var payloadAtt = ts.AddAttribute<IPayloadAttribute>();
var offsetAtt = ts.AddAttribute<IOffsetAttribute>();
Assert.IsTrue(ts.IncrementToken());
Assert.AreEqual(text, termAtt.Term);
Assert.AreEqual(positionIncrement, posIncrAtt.PositionIncrement);
Assert.AreEqual(boost,
payloadAtt.Payload == null
? 1f
: PayloadHelper.DecodeFloat(payloadAtt.Payload.GetData()), 0);
Assert.AreEqual(startOffset, offsetAtt.StartOffset);
Assert.AreEqual(endOffset, offsetAtt.EndOffset);
}
internal virtual void AssertTermEquals(System.String expected, TokenStream stream, TermAttribute termAtt)
{
Assert.IsTrue(stream.IncrementToken());
Assert.AreEqual(expected, termAtt.Term());
}
private static void ConsumeStreamNewAPI(TokenStream stream)
{
stream.Reset();
PayloadAttribute payloadAtt = (PayloadAttribute) stream.AddAttribute(typeof(PayloadAttribute));
TermAttribute termAtt = (TermAttribute) stream.AddAttribute(typeof(TermAttribute));
int i = 0;
while (stream.IncrementToken())
{
System.String term = termAtt.Term();
Payload p = payloadAtt.GetPayload();
if (p != null && p.GetData().Length == 1 && p.GetData()[0] == PartOfSpeechAnnotatingFilter.PROPER_NOUN_ANNOTATION)
{
Assert.IsTrue("tokenstream".Equals(term), "only TokenStream is a proper noun");
}
else
{
Assert.IsFalse("tokenstream".Equals(term), "all other tokens (if this test fails, the special POSToken subclass is not correctly passed through the chain)");
}
Assert.AreEqual(results[i], term);
i++;
}
}
// assert-methods start here
private static void AssertNext(TokenStream ts, String text)
{
var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
Assert.IsTrue(ts.IncrementToken());
Assert.AreEqual(text, termAtt.Term());
}
/// <summary>
/// Pulls the graph (including <see cref="IPositionLengthAttribute"/>
/// from the provided <see cref="TokenStream"/>, and creates the corresponding
/// automaton where arcs are bytes (or Unicode code points
/// if unicodeArcs = true) from each term.
/// </summary>
public virtual Automaton ToAutomaton(TokenStream @in)
{
var a = new Automaton();
bool deterministic = true;
var posIncAtt = @in.AddAttribute <IPositionIncrementAttribute>();
var posLengthAtt = @in.AddAttribute <IPositionLengthAttribute>();
var offsetAtt = @in.AddAttribute <IOffsetAttribute>();
var termBytesAtt = @in.AddAttribute <ITermToBytesRefAttribute>();
BytesRef term = termBytesAtt.BytesRef;
@in.Reset();
// Only temporarily holds states ahead of our current
// position:
RollingBuffer <Position> positions = new Positions();
int pos = -1;
Position posData = null;
int maxOffset = 0;
while (@in.IncrementToken())
{
int posInc = posIncAtt.PositionIncrement;
if (!preservePositionIncrements && posInc > 1)
{
posInc = 1;
}
Debug.Assert(pos > -1 || posInc > 0);
if (posInc > 0)
{
// New node:
pos += posInc;
posData = positions.Get(pos);
Debug.Assert(posData.leaving == null);
if (posData.arriving == null)
{
// No token ever arrived to this position
if (pos == 0)
{
// OK: this is the first token
posData.leaving = a.GetInitialState();
}
else
{
// this means there's a hole (eg, StopFilter
// does this):
posData.leaving = new State();
AddHoles(a.GetInitialState(), positions, pos);
}
}
else
{
posData.leaving = new State();
posData.arriving.AddTransition(new Transition(POS_SEP, posData.leaving));
if (posInc > 1)
{
// A token spanned over a hole; add holes
// "under" it:
AddHoles(a.GetInitialState(), positions, pos);
}
}
positions.FreeBefore(pos);
}
else
{
// note: this isn't necessarily true. its just that we aren't surely det.
// we could optimize this further (e.g. buffer and sort synonyms at a position)
// but thats probably overkill. this is cheap and dirty
deterministic = false;
}
int endPos = pos + posLengthAtt.PositionLength;
termBytesAtt.FillBytesRef();
BytesRef termUTF8 = ChangeToken(term);
int[] termUnicode = null;
Position endPosData = positions.Get(endPos);
if (endPosData.arriving == null)
{
endPosData.arriving = new State();
}
State state = posData.leaving;
int termLen = termUTF8.Length;
if (unicodeArcs)
{
string utf16 = termUTF8.Utf8ToString();
termUnicode = new int[utf16.CodePointCount(0, utf16.Length)];
termLen = termUnicode.Length;
for (int cp, i = 0, j = 0; i < utf16.Length; i += Character.CharCount(cp))
{
termUnicode[j++] = cp = Character.CodePointAt(utf16, i);
}
}
else
{
termLen = termUTF8.Length;
}
for (int byteIDX = 0; byteIDX < termLen; byteIDX++)
{
State nextState = byteIDX == termLen - 1 ? endPosData.arriving : new State();
int c;
if (unicodeArcs)
{
c = termUnicode[byteIDX];
}
else
{
c = termUTF8.Bytes[termUTF8.Offset + byteIDX] & 0xff;
}
state.AddTransition(new Transition(c, nextState));
state = nextState;
}
maxOffset = Math.Max(maxOffset, offsetAtt.EndOffset);
}
@in.End();
State endState = null;
if (offsetAtt.EndOffset > maxOffset)
{
endState = new State();
endState.Accept = true;
}
pos++;
while (pos <= positions.MaxPos)
{
posData = positions.Get(pos);
if (posData.arriving != null)
{
if (endState != null)
{
posData.arriving.AddTransition(new Transition(POS_SEP, endState));
}
else
{
posData.arriving.Accept = true;
}
}
pos++;
}
//toDot(a);
a.IsDeterministic = deterministic;
return(a);
}
private static void AssertNext(TokenStream ts, String text, int startOffset, int endOffset)
{
var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
var offsetAtt = (OffsetAttribute) ts.AddAttribute(typeof (OffsetAttribute));
Assert.IsTrue(ts.IncrementToken());
Assert.AreEqual(text, termAtt.Term());
Assert.AreEqual(startOffset, offsetAtt.StartOffset());
Assert.AreEqual(endOffset, offsetAtt.EndOffset());
}
private static void AssertNext(TokenStream ts, String text, int positionIncrement, float boost, int startOffset,
int endOffset)
{
var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
var posIncrAtt = (PositionIncrementAttribute) ts.AddAttribute(typeof (PositionIncrementAttribute));
var payloadAtt = (PayloadAttribute) ts.AddAttribute(typeof (PayloadAttribute));
var offsetAtt = (OffsetAttribute) ts.AddAttribute(typeof (OffsetAttribute));
Assert.IsTrue(ts.IncrementToken());
Assert.AreEqual(text, termAtt.Term());
Assert.AreEqual(positionIncrement, posIncrAtt.GetPositionIncrement());
Assert.AreEqual(boost,
payloadAtt.GetPayload() == null
? 1f
: PayloadHelper.DecodeFloat(payloadAtt.GetPayload().GetData()), 0);
Assert.AreEqual(startOffset, offsetAtt.StartOffset());
Assert.AreEqual(endOffset, offsetAtt.EndOffset());
}
// offsetsAreCorrect also validates:
// - graph offsets are correct (all tokens leaving from
// pos X have the same startOffset; all tokens
// arriving to pos Y have the same endOffset)
// - offsets only move forwards (startOffset >=
// lastStartOffset)
public static void AssertTokenStreamContents(TokenStream ts, string[] output, int[] startOffsets, int[] endOffsets, string[] types, int[] posIncrements, int[] posLengths, int?finalOffset, int?finalPosInc, bool[] keywordAtts, bool offsetsAreCorrect)
{
Assert.IsNotNull(output);
var checkClearAtt = ts.AddAttribute <ICheckClearAttributesAttribute>();
ICharTermAttribute termAtt = null;
if (output.Length > 0)
{
Assert.IsTrue(ts.HasAttribute <ICharTermAttribute>(), "has no CharTermAttribute");
termAtt = ts.GetAttribute <ICharTermAttribute>();
}
IOffsetAttribute offsetAtt = null;
if (startOffsets != null || endOffsets != null || finalOffset != null)
{
Assert.IsTrue(ts.HasAttribute <IOffsetAttribute>(), "has no OffsetAttribute");
offsetAtt = ts.GetAttribute <IOffsetAttribute>();
}
ITypeAttribute typeAtt = null;
if (types != null)
{
Assert.IsTrue(ts.HasAttribute <ITypeAttribute>(), "has no TypeAttribute");
typeAtt = ts.GetAttribute <ITypeAttribute>();
}
IPositionIncrementAttribute posIncrAtt = null;
if (posIncrements != null || finalPosInc != null)
{
Assert.IsTrue(ts.HasAttribute <IPositionIncrementAttribute>(), "has no PositionIncrementAttribute");
posIncrAtt = ts.GetAttribute <IPositionIncrementAttribute>();
}
IPositionLengthAttribute posLengthAtt = null;
if (posLengths != null)
{
Assert.IsTrue(ts.HasAttribute <IPositionLengthAttribute>(), "has no PositionLengthAttribute");
posLengthAtt = ts.GetAttribute <IPositionLengthAttribute>();
}
IKeywordAttribute keywordAtt = null;
if (keywordAtts != null)
{
Assert.IsTrue(ts.HasAttribute <IKeywordAttribute>(), "has no KeywordAttribute");
keywordAtt = ts.GetAttribute <IKeywordAttribute>();
}
// Maps position to the start/end offset:
IDictionary <int?, int?> posToStartOffset = new Dictionary <int?, int?>();
IDictionary <int?, int?> posToEndOffset = new Dictionary <int?, int?>();
ts.Reset();
int pos = -1;
int lastStartOffset = 0;
for (int i = 0; i < output.Length; i++)
{
// extra safety to enforce, that the state is not preserved and also assign bogus values
ts.ClearAttributes();
termAtt.SetEmpty().Append("bogusTerm");
if (offsetAtt != null)
{
offsetAtt.SetOffset(14584724, 24683243);
}
if (typeAtt != null)
{
typeAtt.Type = "bogusType";
}
if (posIncrAtt != null)
{
posIncrAtt.PositionIncrement = 45987657;
}
if (posLengthAtt != null)
{
posLengthAtt.PositionLength = 45987653;
}
if (keywordAtt != null)
{
keywordAtt.Keyword = (i & 1) == 0;
}
bool reset = checkClearAtt.AndResetClearCalled; // reset it, because we called clearAttribute() before
Assert.IsTrue(ts.IncrementToken(), "token " + i + " does not exist");
Assert.IsTrue(reset, "ClearAttributes() was not called correctly in TokenStream chain");
Assert.AreEqual(output[i], termAtt.ToString(), "term " + i + ", output[i] = " + output[i] + ", termAtt = " + termAtt.ToString());
if (startOffsets != null)
{
Assert.AreEqual(startOffsets[i], offsetAtt.StartOffset(), "startOffset " + i);
}
if (endOffsets != null)
{
Assert.AreEqual(endOffsets[i], offsetAtt.EndOffset(), "endOffset " + i);
}
if (types != null)
{
Assert.AreEqual(types[i], typeAtt.Type, "type " + i);
}
if (posIncrements != null)
{
Assert.AreEqual(posIncrements[i], posIncrAtt.PositionIncrement, "posIncrement " + i);
}
if (posLengths != null)
{
Assert.AreEqual(posLengths[i], posLengthAtt.PositionLength, "posLength " + i);
}
if (keywordAtts != null)
{
Assert.AreEqual(keywordAtts[i], keywordAtt.Keyword, "keywordAtt " + i);
}
// we can enforce some basic things about a few attributes even if the caller doesn't check:
if (offsetAtt != null)
{
int startOffset = offsetAtt.StartOffset();
int endOffset = offsetAtt.EndOffset();
if (finalOffset != null)
{
Assert.IsTrue(startOffset <= (int)finalOffset, "startOffset must be <= finalOffset");
Assert.IsTrue(endOffset <= (int)finalOffset, "endOffset must be <= finalOffset: got endOffset=" + endOffset + " vs finalOffset=" + (int)finalOffset);
}
if (offsetsAreCorrect)
{
Assert.IsTrue(offsetAtt.StartOffset() >= lastStartOffset, "offsets must not go backwards startOffset=" + startOffset + " is < lastStartOffset=" + lastStartOffset);
lastStartOffset = offsetAtt.StartOffset();
}
if (offsetsAreCorrect && posLengthAtt != null && posIncrAtt != null)
{
// Validate offset consistency in the graph, ie
// all tokens leaving from a certain pos have the
// same startOffset, and all tokens arriving to a
// certain pos have the same endOffset:
int posInc = posIncrAtt.PositionIncrement;
pos += posInc;
int posLength = posLengthAtt.PositionLength;
if (!posToStartOffset.ContainsKey(pos))
{
// First time we've seen a token leaving from this position:
posToStartOffset[pos] = startOffset;
//System.out.println(" + s " + pos + " -> " + startOffset);
}
else
{
// We've seen a token leaving from this position
// before; verify the startOffset is the same:
//System.out.println(" + vs " + pos + " -> " + startOffset);
Assert.AreEqual((int)posToStartOffset[pos], startOffset, "pos=" + pos + " posLen=" + posLength + " token=" + termAtt);
}
int endPos = pos + posLength;
if (!posToEndOffset.ContainsKey(endPos))
{
// First time we've seen a token arriving to this position:
posToEndOffset[endPos] = endOffset;
//System.out.println(" + e " + endPos + " -> " + endOffset);
}
else
{
// We've seen a token arriving to this position
// before; verify the endOffset is the same:
//System.out.println(" + ve " + endPos + " -> " + endOffset);
Assert.AreEqual((int)posToEndOffset[endPos], endOffset, "pos=" + pos + " posLen=" + posLength + " token=" + termAtt);
}
}
}
if (posIncrAtt != null)
{
if (i == 0)
{
Assert.IsTrue(posIncrAtt.PositionIncrement >= 1, "first posIncrement must be >= 1");
}
else
{
Assert.IsTrue(posIncrAtt.PositionIncrement >= 0, "posIncrement must be >= 0");
}
}
if (posLengthAtt != null)
{
Assert.IsTrue(posLengthAtt.PositionLength >= 1, "posLength must be >= 1");
}
}
if (ts.IncrementToken())
{
Assert.Fail("TokenStream has more tokens than expected (expected count=" + output.Length + "); extra token=" + termAtt);
}
// repeat our extra safety checks for End()
ts.ClearAttributes();
if (termAtt != null)
{
termAtt.SetEmpty().Append("bogusTerm");
}
if (offsetAtt != null)
{
offsetAtt.SetOffset(14584724, 24683243);
}
if (typeAtt != null)
{
typeAtt.Type = "bogusType";
}
if (posIncrAtt != null)
{
posIncrAtt.PositionIncrement = 45987657;
}
if (posLengthAtt != null)
{
posLengthAtt.PositionLength = 45987653;
}
var reset_ = checkClearAtt.AndResetClearCalled; // reset it, because we called clearAttribute() before
ts.End();
Assert.IsTrue(checkClearAtt.AndResetClearCalled, "super.End()/ClearAttributes() was not called correctly in End()");
if (finalOffset != null)
{
Assert.AreEqual((int)finalOffset, offsetAtt.EndOffset(), "finalOffset");
}
if (offsetAtt != null)
{
Assert.IsTrue(offsetAtt.EndOffset() >= 0, "finalOffset must be >= 0");
}
if (finalPosInc != null)
{
Assert.AreEqual((int)finalPosInc, posIncrAtt.PositionIncrement, "finalPosInc");
}
ts.Dispose();
}
public static void AssertTokenStreamContents(TokenStream ts, System.String[] output, int[] startOffsets, int[] endOffsets, System.String[] types, int[] posIncrements, int?finalOffset)
{
Assert.IsNotNull(output);
CheckClearAttributesAttribute checkClearAtt = (CheckClearAttributesAttribute)ts.AddAttribute(typeof(CheckClearAttributesAttribute));
Assert.IsTrue(ts.HasAttribute(typeof(TermAttribute)), "has no TermAttribute");
TermAttribute termAtt = (TermAttribute)ts.GetAttribute(typeof(TermAttribute));
OffsetAttribute offsetAtt = null;
if (startOffsets != null || endOffsets != null || finalOffset != null)
{
Assert.IsTrue(ts.HasAttribute(typeof(OffsetAttribute)), "has no OffsetAttribute");
offsetAtt = (OffsetAttribute)ts.GetAttribute(typeof(OffsetAttribute));
}
TypeAttribute typeAtt = null;
if (types != null)
{
Assert.IsTrue(ts.HasAttribute(typeof(TypeAttribute)), "has no TypeAttribute");
typeAtt = (TypeAttribute)ts.GetAttribute(typeof(TypeAttribute));
}
PositionIncrementAttribute posIncrAtt = null;
if (posIncrements != null)
{
Assert.IsTrue(ts.HasAttribute(typeof(PositionIncrementAttribute)), "has no PositionIncrementAttribute");
posIncrAtt = (PositionIncrementAttribute)ts.GetAttribute(typeof(PositionIncrementAttribute));
}
ts.Reset();
for (int i = 0; i < output.Length; i++)
{
// extra safety to enforce, that the state is not preserved and also assign bogus values
ts.ClearAttributes();
termAtt.SetTermBuffer("bogusTerm");
if (offsetAtt != null)
{
offsetAtt.SetOffset(14584724, 24683243);
}
if (typeAtt != null)
{
typeAtt.SetType("bogusType");
}
if (posIncrAtt != null)
{
posIncrAtt.SetPositionIncrement(45987657);
}
checkClearAtt.GetAndResetClearCalled(); // reset it, because we called clearAttribute() before
Assert.IsTrue(ts.IncrementToken(), "token " + i + " does not exist");
Assert.IsTrue(checkClearAtt.GetAndResetClearCalled(), "clearAttributes() was not called correctly in TokenStream chain");
Assert.AreEqual(output[i], termAtt.Term(), "term " + i);
if (startOffsets != null)
{
Assert.AreEqual(startOffsets[i], offsetAtt.StartOffset(), "startOffset " + i);
}
if (endOffsets != null)
{
Assert.AreEqual(endOffsets[i], offsetAtt.EndOffset(), "endOffset " + i);
}
if (types != null)
{
Assert.AreEqual(types[i], typeAtt.Type(), "type " + i);
}
if (posIncrements != null)
{
Assert.AreEqual(posIncrements[i], posIncrAtt.GetPositionIncrement(), "posIncrement " + i);
}
}
Assert.IsFalse(ts.IncrementToken(), "end of stream");
ts.End();
if (finalOffset.HasValue)
{
Assert.AreEqual(finalOffset, offsetAtt.EndOffset(), "finalOffset ");
}
ts.Close();
}
private void checkTokens(TokenStream stream)
{
int count = 0;
TermAttribute termAtt = (TermAttribute) stream.GetAttribute(typeof(TermAttribute));
Assert.IsNotNull(termAtt);
while (stream.IncrementToken())
{
Assert.IsTrue(count < tokens.Length);
Assert.AreEqual(tokens[count], termAtt.Term());
count++;
}
Assert.AreEqual(tokens.Length, count);
}
void AssertTermEquals(String expected, TokenStream stream, ITermAttribute termAtt, IPayloadAttribute payAtt, byte[] expectPay)
{
Assert.True(stream.IncrementToken());
Assert.AreEqual(expected, termAtt.Term);
Payload payload = payAtt.Payload;
if (payload != null)
{
Assert.True(payload.Length == expectPay.Length, payload.Length + " does not equal: " + expectPay.Length);
for (int i = 0; i < expectPay.Length; i++)
{
Assert.True(expectPay[i] == payload.ByteAt(i), expectPay[i] + " does not equal: " + payload.ByteAt(i));
}
}
else
{
Assert.True(expectPay == null, "expectPay is not null and it should be");
}
}
private void CheckTokens(TokenStream stream)
{
int count = 0;
ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>();
while (stream.IncrementToken())
{
Assert.IsTrue(count < Tokens.Length);
Assert.AreEqual(Tokens[count], termAtt.ToString());
count++;
}
Assert.AreEqual(Tokens.Length, count);
}
/// <summary> Not an explicit test, just useful to print out some info on performance
///
/// </summary>
/// <throws> Exception </throws>
public virtual void Performance()
{
int[] tokCount = new int[] { 100, 500, 1000, 2000, 5000, 10000 };
int[] modCounts = new int[] { 1, 2, 5, 10, 20, 50, 100, 200, 500 };
for (int k = 0; k < tokCount.Length; k++)
{
System.Text.StringBuilder buffer = new System.Text.StringBuilder();
System.Console.Out.WriteLine("-----Tokens: " + tokCount[k] + "-----");
for (int i = 0; i < tokCount[k]; i++)
{
buffer.Append(English.IntToEnglish(i).ToUpper()).Append(' ');
}
//make sure we produce the same tokens
TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(new StandardTokenizer(Version.LUCENE_CURRENT, new System.IO.StringReader(buffer.ToString()))));
TokenStream sink = teeStream.NewSinkTokenStream(new ModuloSinkFilter(this, 100));
teeStream.ConsumeAllTokens();
TokenStream stream = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(Version.LUCENE_CURRENT, new System.IO.StringReader(buffer.ToString()))), 100);
ITermAttribute tfTok = stream.AddAttribute <ITermAttribute>();
ITermAttribute sinkTok = sink.AddAttribute <ITermAttribute>();
for (int i = 0; stream.IncrementToken(); i++)
{
Assert.IsTrue(sink.IncrementToken());
Assert.IsTrue(tfTok.Equals(sinkTok) == true, tfTok + " is not equal to " + sinkTok + " at token: " + i);
}
//simulate two fields, each being analyzed once, for 20 documents
for (int j = 0; j < modCounts.Length; j++)
{
int tfPos = 0;
long start = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
for (int i = 0; i < 20; i++)
{
stream = new StandardFilter(new StandardTokenizer(Version.LUCENE_CURRENT, new System.IO.StringReader(buffer.ToString())));
IPositionIncrementAttribute posIncrAtt = stream.GetAttribute <IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
tfPos += posIncrAtt.PositionIncrement;
}
stream = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(Version.LUCENE_CURRENT, new System.IO.StringReader(buffer.ToString()))), modCounts[j]);
posIncrAtt = stream.GetAttribute <IPositionIncrementAttribute>();
while (stream.IncrementToken())
{
tfPos += posIncrAtt.PositionIncrement;
}
}
long finish = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
int sinkPos = 0;
//simulate one field with one sink
start = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
for (int i = 0; i < 20; i++)
{
teeStream = new TeeSinkTokenFilter(new StandardFilter(new StandardTokenizer(Version.LUCENE_CURRENT, new System.IO.StringReader(buffer.ToString()))));
sink = teeStream.NewSinkTokenStream(new ModuloSinkFilter(this, modCounts[j]));
IPositionIncrementAttribute posIncrAtt = teeStream.GetAttribute <IPositionIncrementAttribute>();
while (teeStream.IncrementToken())
{
sinkPos += posIncrAtt.PositionIncrement;
}
//System.out.println("Modulo--------");
posIncrAtt = sink.GetAttribute <IPositionIncrementAttribute>();
while (sink.IncrementToken())
{
sinkPos += posIncrAtt.PositionIncrement;
}
}
finish = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
Assert.IsTrue(sinkPos == tfPos, sinkPos + " does not equal: " + tfPos);
}
System.Console.Out.WriteLine("- End Tokens: " + tokCount[k] + "-----");
}
}
// offsetsAreCorrect also validates:
// - graph offsets are correct (all tokens leaving from
// pos X have the same startOffset; all tokens
// arriving to pos Y have the same endOffset)
// - offsets only move forwards (startOffset >=
// lastStartOffset)
public static void AssertTokenStreamContents(TokenStream ts, string[] output, int[] startOffsets, int[] endOffsets, string[] types, int[] posIncrements, int[] posLengths, int? finalOffset, int? finalPosInc, bool[] keywordAtts, bool offsetsAreCorrect)
{
// LUCENENET: Bug fix: NUnit throws an exception when something fails.
// This causes Dispose() to be skipped and it pollutes other tests indicating false negatives.
// Added this try-finally block to fix this.
try
{
Assert.IsNotNull(output);
var checkClearAtt = ts.AddAttribute<ICheckClearAttributesAttribute>();
ICharTermAttribute termAtt = null;
if (output.Length > 0)
{
Assert.IsTrue(ts.HasAttribute<ICharTermAttribute>(), "has no CharTermAttribute");
termAtt = ts.GetAttribute<ICharTermAttribute>();
}
IOffsetAttribute offsetAtt = null;
if (startOffsets != null || endOffsets != null || finalOffset != null)
{
Assert.IsTrue(ts.HasAttribute<IOffsetAttribute>(), "has no OffsetAttribute");
offsetAtt = ts.GetAttribute<IOffsetAttribute>();
}
ITypeAttribute typeAtt = null;
if (types != null)
{
Assert.IsTrue(ts.HasAttribute<ITypeAttribute>(), "has no TypeAttribute");
typeAtt = ts.GetAttribute<ITypeAttribute>();
}
IPositionIncrementAttribute posIncrAtt = null;
if (posIncrements != null || finalPosInc != null)
{
Assert.IsTrue(ts.HasAttribute<IPositionIncrementAttribute>(), "has no PositionIncrementAttribute");
posIncrAtt = ts.GetAttribute<IPositionIncrementAttribute>();
}
IPositionLengthAttribute posLengthAtt = null;
if (posLengths != null)
{
Assert.IsTrue(ts.HasAttribute<IPositionLengthAttribute>(), "has no PositionLengthAttribute");
posLengthAtt = ts.GetAttribute<IPositionLengthAttribute>();
}
IKeywordAttribute keywordAtt = null;
if (keywordAtts != null)
{
Assert.IsTrue(ts.HasAttribute<IKeywordAttribute>(), "has no KeywordAttribute");
keywordAtt = ts.GetAttribute<IKeywordAttribute>();
}
// Maps position to the start/end offset:
IDictionary<int?, int?> posToStartOffset = new Dictionary<int?, int?>();
IDictionary<int?, int?> posToEndOffset = new Dictionary<int?, int?>();
ts.Reset();
int pos = -1;
int lastStartOffset = 0;
for (int i = 0; i < output.Length; i++)
{
// extra safety to enforce, that the state is not preserved and also assign bogus values
ts.ClearAttributes();
termAtt.SetEmpty().Append("bogusTerm");
if (offsetAtt != null)
{
offsetAtt.SetOffset(14584724, 24683243);
}
if (typeAtt != null)
{
typeAtt.Type = "bogusType";
}
if (posIncrAtt != null)
{
posIncrAtt.PositionIncrement = 45987657;
}
if (posLengthAtt != null)
{
posLengthAtt.PositionLength = 45987653;
}
if (keywordAtt != null)
{
keywordAtt.Keyword = (i & 1) == 0;
}
bool reset = checkClearAtt.AndResetClearCalled; // reset it, because we called clearAttribute() before
Assert.IsTrue(ts.IncrementToken(), "token " + i + " does not exist");
Assert.IsTrue(reset, "ClearAttributes() was not called correctly in TokenStream chain");
Assert.AreEqual(output[i], termAtt.ToString(), "term " + i + ", output[i] = " + output[i] + ", termAtt = " + termAtt.ToString());
if (startOffsets != null)
{
Assert.AreEqual(startOffsets[i], offsetAtt.StartOffset(), "startOffset " + i);
}
if (endOffsets != null)
{
Assert.AreEqual(endOffsets[i], offsetAtt.EndOffset(), "endOffset " + i);
}
if (types != null)
{
Assert.AreEqual(types[i], typeAtt.Type, "type " + i);
}
if (posIncrements != null)
{
Assert.AreEqual(posIncrements[i], posIncrAtt.PositionIncrement, "posIncrement " + i);
}
if (posLengths != null)
{
Assert.AreEqual(posLengths[i], posLengthAtt.PositionLength, "posLength " + i);
}
if (keywordAtts != null)
{
Assert.AreEqual(keywordAtts[i], keywordAtt.Keyword, "keywordAtt " + i);
}
// we can enforce some basic things about a few attributes even if the caller doesn't check:
if (offsetAtt != null)
{
int startOffset = offsetAtt.StartOffset();
int endOffset = offsetAtt.EndOffset();
if (finalOffset != null)
{
Assert.IsTrue(startOffset <= (int)finalOffset, "startOffset must be <= finalOffset");
Assert.IsTrue(endOffset <= (int)finalOffset, "endOffset must be <= finalOffset: got endOffset=" + endOffset + " vs finalOffset=" + (int)finalOffset);
}
if (offsetsAreCorrect)
{
Assert.IsTrue(offsetAtt.StartOffset() >= lastStartOffset, "offsets must not go backwards startOffset=" + startOffset + " is < lastStartOffset=" + lastStartOffset);
lastStartOffset = offsetAtt.StartOffset();
}
if (offsetsAreCorrect && posLengthAtt != null && posIncrAtt != null)
{
// Validate offset consistency in the graph, ie
// all tokens leaving from a certain pos have the
// same startOffset, and all tokens arriving to a
// certain pos have the same endOffset:
int posInc = posIncrAtt.PositionIncrement;
pos += posInc;
int posLength = posLengthAtt.PositionLength;
if (!posToStartOffset.ContainsKey(pos))
{
// First time we've seen a token leaving from this position:
posToStartOffset[pos] = startOffset;
//System.out.println(" + s " + pos + " -> " + startOffset);
}
else
{
// We've seen a token leaving from this position
// before; verify the startOffset is the same:
//System.out.println(" + vs " + pos + " -> " + startOffset);
Assert.AreEqual((int)posToStartOffset[pos], startOffset, "pos=" + pos + " posLen=" + posLength + " token=" + termAtt);
}
int endPos = pos + posLength;
if (!posToEndOffset.ContainsKey(endPos))
{
// First time we've seen a token arriving to this position:
posToEndOffset[endPos] = endOffset;
//System.out.println(" + e " + endPos + " -> " + endOffset);
}
else
{
// We've seen a token arriving to this position
// before; verify the endOffset is the same:
//System.out.println(" + ve " + endPos + " -> " + endOffset);
Assert.AreEqual((int)posToEndOffset[endPos], endOffset, "pos=" + pos + " posLen=" + posLength + " token=" + termAtt);
}
}
}
if (posIncrAtt != null)
{
if (i == 0)
{
Assert.IsTrue(posIncrAtt.PositionIncrement >= 1, "first posIncrement must be >= 1");
}
else
{
Assert.IsTrue(posIncrAtt.PositionIncrement >= 0, "posIncrement must be >= 0");
}
}
if (posLengthAtt != null)
{
Assert.IsTrue(posLengthAtt.PositionLength >= 1, "posLength must be >= 1");
}
}
if (ts.IncrementToken())
{
Assert.Fail("TokenStream has more tokens than expected (expected count=" + output.Length + "); extra token=" + termAtt);
}
// repeat our extra safety checks for End()
ts.ClearAttributes();
if (termAtt != null)
{
termAtt.SetEmpty().Append("bogusTerm");
}
if (offsetAtt != null)
{
offsetAtt.SetOffset(14584724, 24683243);
}
if (typeAtt != null)
{
typeAtt.Type = "bogusType";
}
if (posIncrAtt != null)
{
posIncrAtt.PositionIncrement = 45987657;
}
if (posLengthAtt != null)
{
posLengthAtt.PositionLength = 45987653;
}
var reset_ = checkClearAtt.AndResetClearCalled; // reset it, because we called clearAttribute() before
ts.End();
Assert.IsTrue(checkClearAtt.AndResetClearCalled, "super.End()/ClearAttributes() was not called correctly in End()");
if (finalOffset != null)
{
Assert.AreEqual((int)finalOffset, offsetAtt.EndOffset(), "finalOffset");
}
if (offsetAtt != null)
{
Assert.IsTrue(offsetAtt.EndOffset() >= 0, "finalOffset must be >= 0");
}
if (finalPosInc != null)
{
Assert.AreEqual((int)finalPosInc, posIncrAtt.PositionIncrement, "finalPosInc");
}
ts.Dispose();
}
catch (Exception)
{
//ts.Reset();
ts.ClearAttributes();
ts.End();
ts.Dispose();
throw;
}
}
