//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { // capture state lazily - maybe no SinkFilter accepts this state AttributeSource.State state = null; foreach (WeakReference <SinkTokenStream> @ref in sinks) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final SinkTokenStream sink = ref.get(); SinkTokenStream sink = @ref.get(); if (sink != null) { if (sink.accept(this)) { if (state == null) { state = this.captureState(); } sink.addState(state); } } } return(true); } return(false); }
public override sealed bool IncrementToken() { if (m_tokens.Count > 0) { Debug.Assert(current != null); CompoundToken token = m_tokens.First.Value; m_tokens.Remove(token); RestoreState(current); // keep all other attributes untouched m_termAtt.SetEmpty().Append(token.Text); m_offsetAtt.SetOffset(token.StartOffset, token.EndOffset); posIncAtt.PositionIncrement = 0; return(true); } current = null; // not really needed, but for safety if (m_input.IncrementToken()) { // Only words longer than minWordSize get processed if (m_termAtt.Length >= this.m_minWordSize) { Decompose(); // only capture the state if we really need it for producing new tokens if (m_tokens.Count > 0) { current = CaptureState(); } } // return original token: return(true); } else { return(false); } }
public sealed override bool IncrementToken() { if (state != null) { RestoreState(state); payloadAtt.Payload = null; posIncrAtt.PositionIncrement = 0; termAtt.SetEmpty().Append("b"); state = null; return(true); } bool hasNext = m_input.IncrementToken(); if (!hasNext) { return(false); } if (char.IsDigit(termAtt.Buffer[0])) { posIncrAtt.PositionIncrement = termAtt.Buffer[0] - '0'; } if (first) { // set payload on first position only payloadAtt.Payload = new BytesRef(new byte[] { 100 }); first = false; } // index a "synonym" for every token state = CaptureState(); return(true); }
public void Reset() { state = null; consumed = true; keepOrig = false; matched = false; }
public override bool IncrementToken() { if (m_input.IncrementToken()) { // capture state lazily - maybe no SinkFilter accepts this state AttributeSource.State state = null; foreach (WeakReference <SinkTokenStream> @ref in sinks) { SinkTokenStream sink; if (@ref.TryGetTarget(out sink)) { if (sink.Accept(this)) { if (state == null) { state = CaptureState(); } sink.AddState(state); } } } return(true); } return(false); }
internal /*private*/ void AddState(AttributeSource.State state) { if (it != null) { throw new System.SystemException("The tee must be consumed before sinks are consumed."); } cachedStates.AddLast(state); }
internal void AddState(AttributeSource.State state) { if (it != null) { throw new InvalidOperationException("The tee must be consumed before sinks are consumed."); } cachedStates.Add(state); }
internal void AddState(AttributeSource.State state) { if (it != null) { throw IllegalStateException.Create("The tee must be consumed before sinks are consumed."); } cachedStates.Add(state); }
private void FillCache() { while (m_input.IncrementToken()) { cache.Add(CaptureState()); } // capture final state m_input.End(); finalState = CaptureState(); }
public override void Reset() { base.Reset(); nextToken = null; shingleBufferPosition = 0; shingleBuf.Clear(); numFillerTokensToInsert = 0; currentToken = null; hasCurrentToken = false; }
private void FillCache() { while (Input.IncrementToken()) { Cache.AddLast(CaptureState()); } // capture final state Input.End(); FinalState = CaptureState(); }
public override sealed void End() { base.End(); AttributeSource.State finalState = CaptureState(); foreach (WeakReference <SinkTokenStream> @ref in sinks) { SinkTokenStream sink;; if (@ref.TryGetTarget(out sink)) { sink.SetFinalState(finalState); } } }
protected override void Swap(int i, int j) { AttributeSource.State tmp = outerInstance.buffered[i]; outerInstance.buffered[i] = outerInstance.buffered[j]; outerInstance.buffered[j] = tmp; int tmp2 = outerInstance.startOff[i]; outerInstance.startOff[i] = outerInstance.startOff[j]; outerInstance.startOff[j] = tmp2; tmp2 = outerInstance.posInc[i]; outerInstance.posInc[i] = outerInstance.posInc[j]; outerInstance.posInc[j] = tmp2; }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public final void end() throws java.io.IOException public override void end() { base.end(); AttributeSource.State finalState = captureState(); foreach (WeakReference <SinkTokenStream> @ref in sinks) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final SinkTokenStream sink = ref.get(); SinkTokenStream sink = @ref.get(); if (sink != null) { sink.FinalState = finalState; } } }
/* * (non-Javadoc) * * @see org.apache.lucene.analysis.TokenStream#next() */ //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException public override bool incrementToken() { //JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops: if (tokens != null && tokens.hasNext()) { //JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops: AttributeSource.State state = tokens.next(); restoreState(state); return(true); } clearAttributes(); int tokenType = scanner.NextToken; if (tokenType == WikipediaTokenizerImpl.YYEOF) { return(false); } string type = WikipediaTokenizerImpl.TOKEN_TYPES[tokenType]; if (tokenOutput == TOKENS_ONLY || untokenizedTypes.Contains(type) == false) { setupToken(); } else if (tokenOutput == UNTOKENIZED_ONLY && untokenizedTypes.Contains(type) == true) { collapseTokens(tokenType); } else if (tokenOutput == BOTH) { //collapse into a single token, add it to tokens AND output the individual tokens //output the untokenized Token first collapseAndSaveTokens(tokenType, type); } int posinc = scanner.PositionIncrement; if (first && posinc == 0) { posinc = 1; // don't emit posinc=0 for the first token! } posIncrAtt.PositionIncrement = posinc; typeAtt.Type = type; first = false; return(true); }
/* * Get the next token from the input stream and push it on the token buffer. * If we encounter a token with position increment > 1, we put filler tokens * on the token buffer. * <p/> * Returns null when the end of the input stream is reached. * @return the next token, or null if at end of input stream * @throws IOException if the input stream has a problem */ private bool GetNextToken() { while (true) { if (numFillerTokensToInsert > 0) { if (currentToken == null) { currentToken = CaptureState(); } else { RestoreState(currentToken); } numFillerTokensToInsert--; // A filler token occupies no space offsetAtt.SetOffset(offsetAtt.StartOffset, offsetAtt.StartOffset); termAtt.SetTermBuffer(FILLER_TOKEN, 0, FILLER_TOKEN.Length); return(true); } if (hasCurrentToken) { if (currentToken != null) { RestoreState(currentToken); currentToken = null; } hasCurrentToken = false; return(true); } if (!input.IncrementToken()) { return(false); } hasCurrentToken = true; if (posIncrAtt.PositionIncrement > 1) { numFillerTokensToInsert = posIncrAtt.PositionIncrement - 1; } } }
public override bool incrementToken() { // lazy init the iterator if (it == null) { it = cachedStates.GetEnumerator(); } //JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops: if (!it.hasNext()) { return(false); } //JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops: AttributeSource.State state = it.next(); restoreState(state); return(true); }
/* * (non-Javadoc) * * @see org.apache.lucene.analysis.TokenStream#next() */ public override bool IncrementToken() { if (tokens != null && tokens.MoveNext()) { AttributeSource.State state = tokens.Current; RestoreState(state); return(true); } ClearAttributes(); int tokenType = scanner.GetNextToken(); if (tokenType == WikipediaTokenizerImpl.YYEOF) { return(false); } string type = WikipediaTokenizerImpl.TOKEN_TYPES[tokenType]; if (tokenOutput == TOKENS_ONLY || untokenizedTypes.Contains(type) == false) { setupToken(); } else if (tokenOutput == UNTOKENIZED_ONLY && untokenizedTypes.Contains(type) == true) { collapseTokens(tokenType); } else if (tokenOutput == BOTH) { //collapse into a single token, add it to tokens AND output the individual tokens //output the untokenized Token first collapseAndSaveTokens(tokenType, type); } int posinc = scanner.PositionIncrement; if (first && posinc == 0) { posinc = 1; // don't emit posinc=0 for the first token! } posIncrAtt.PositionIncrement = posinc; typeAtt.Type = type; first = false; return(true); }
public sealed override bool IncrementToken() { if (permuterms.Count > 0) { char[] permuterm = permuterms.Pop(); RestoreState(current); termAttr.CopyBuffer(permuterm, 0, permuterm.Length); posIncAttr.PositionIncrement = 0; return(true); } if (!input.IncrementToken()) { return(false); } if (addPermuterms()) { current = CaptureState(); } return(true); }
public override bool IncrementToken() { if (synonymStack.Count > 0) { var syn = synonymStack.Pop(); RestoreState(current); termAtt.SetTermBuffer(syn); posIncrAtt.PositionIncrement = 0; return(true); } if (!input.IncrementToken()) { return(false); } if (addAliasesToStack()) { current = CaptureState(); } return(true); }
public override bool IncrementToken() { if (state != null) { RestoreState(state); payloadAtt.Payload = null; posIncrAtt.PositionIncrement = 0; termAtt.SetEmpty().Append("b"); state = null; return true; } bool hasNext = input.IncrementToken(); if (!hasNext) { return false; } if (char.IsDigit(termAtt.Buffer()[0])) { posIncrAtt.PositionIncrement = termAtt.Buffer()[0] - '0'; } if (first) { // set payload on first position only payloadAtt.Payload = new BytesRef(new byte[] { 100 }); first = false; } // index a "synonym" for every token state = CaptureState(); return true; }
public sealed override void Reset() { base.Reset(); first = true; state = null; }
public virtual void reset() { state = null; consumed = true; keepOrig = false; matched = false; }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public void reset() throws java.io.IOException public override void reset() { base.reset(); tokens.Clear(); current = null; }
/* (non-Javadoc) * @see org.apache.lucene.analysis.TokenStream#next() */ public sealed override bool IncrementToken() { while (true) { if (nextToken == null) { if (!FillShingleBuffer()) { return(false); } } nextToken = shingleBuf.First.Value; if (outputUnigrams) { if (shingleBufferPosition == 0) { RestoreState(nextToken); posIncrAtt.PositionIncrement = 1; shingleBufferPosition++; return(true); } } else if (shingleBufferPosition % this.maxShingleSize == 0) { shingleBufferPosition++; } if (shingleBufferPosition < shingleBuf.Count) { RestoreState(nextToken); typeAtt.Type = tokenType; offsetAtt.SetOffset(offsetAtt.StartOffset, endOffsets[shingleBufferPosition]); StringBuilder buf = shingles[shingleBufferPosition]; int termLength = buf.Length; char[] TermBuffer = termAtt.TermBuffer(); if (TermBuffer.Length < termLength) { TermBuffer = termAtt.ResizeTermBuffer(termLength); } buf.CopyTo(0, TermBuffer, 0, termLength); termAtt.SetTermLength(termLength); if ((!outputUnigrams) && shingleBufferPosition % this.maxShingleSize == 1) { posIncrAtt.PositionIncrement = 1; } else { posIncrAtt.PositionIncrement = 0; } shingleBufferPosition++; if (shingleBufferPosition == shingleBuf.Count) { nextToken = null; shingleBufferPosition = 0; } return(true); } else { nextToken = null; shingleBufferPosition = 0; } } }
public override void Reset() { base.Reset(); m_tokens.Clear(); current = null; }
public override void Reset() { base.Reset(); first = true; state = null; }
public virtual void Add(AttributeSource.State state) { InputTokens.Add(state); }
internal void SetFinalState(AttributeSource.State finalState) { this.finalState = finalState; }