//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final char[] buffer = termAtt.buffer(); char[] buffer = termAtt.buffer(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int length = termAtt.length(); int length = termAtt.length(); for (int i = 0; i < length; i++) { if (buffer[i] == delimiter) { payAtt.Payload = encoder.encode(buffer, i + 1, (length - (i + 1))); termAtt.Length = i; // simply set a new length return(true); } } // we have not seen the delimiter payAtt.Payload = null; return(true); } else { return(false); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { char[] chArray = termAtt.buffer(); int chLen = termAtt.length(); int idx = 0; if (chLen > 1 && (chArray[0] == 'n' || chArray[0] == 't') && isUpperVowel(chArray[1])) { chArray = termAtt.resizeBuffer(chLen + 1); for (int i = chLen; i > 1; i--) { chArray[i] = chArray[i - 1]; } chArray[1] = '-'; termAtt.Length = chLen + 1; idx = 2; chLen = chLen + 1; } for (int i = idx; i < chLen;) { i += char.toChars(char.ToLower(chArray[i]), chArray, i); } return(true); } else { return(false); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException public override bool incrementToken() { clearAttributes(); skippedPositions = 0; while (true) { int tokenType = scanner.NextToken; if (tokenType == StandardTokenizerInterface_Fields.YYEOF) { return(false); } if (scanner.yylength() <= maxTokenLength) { posIncrAtt.PositionIncrement = skippedPositions + 1; scanner.getText(termAtt); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int start = scanner.yychar(); int start = scanner.yychar(); offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length())); typeAtt.Type = TOKEN_TYPES[tokenType]; return(true); } else // When we skip a too-long term, we still increment the // position increment { skippedPositions++; } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length()); termAtt.Length = newlen; return(true); } return(false); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length()); int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length()); termAtt.Length = newlen; return(true); } return(false); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { if (!keywordAtt.Keyword) { termAtt.Length = normalizer.normalize(termAtt.buffer(), termAtt.length()); } return(true); } return(false); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (!input.incrementToken()) { return(false); } if ((!keywordAttr.Keyword) && stemmer.stem(termAtt.buffer(), 0, termAtt.length())) { termAtt.copyBuffer(stemmer.ResultBuffer, 0, stemmer.ResultLength); } return(true); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private org.apache.lucene.analysis.Token getNextPrefixInputToken(org.apache.lucene.analysis.Token token) throws java.io.IOException private Token getNextPrefixInputToken(Token token) { if (!prefix.incrementToken()) { return(null); } token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length()); token.PositionIncrement = p_posIncrAtt.PositionIncrement; token.Flags = p_flagsAtt.Flags; token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset()); token.Type = p_typeAtt.type(); token.Payload = p_payloadAtt.Payload; return(token); }
/// <summary> /// refills buffers with new data from the current token. /// </summary> private void refill() { // compact buffers to keep them smallish if they become large // just a safety check, but technically we only need the last codepoint if (bufferLen > 64) { int last = bufferLen - 1; buffer[0] = buffer[last]; startOffset[0] = startOffset[last]; endOffset[0] = endOffset[last]; bufferLen = 1; index -= last; } char[] termBuffer = termAtt.buffer(); int len = termAtt.length(); int start = offsetAtt.startOffset(); int end = offsetAtt.endOffset(); int newSize = bufferLen + len; buffer = ArrayUtil.grow(buffer, newSize); startOffset = ArrayUtil.grow(startOffset, newSize); endOffset = ArrayUtil.grow(endOffset, newSize); lastEndOffset = end; if (end - start != len) { // crazy offsets (modified by synonym or charfilter): just preserve for (int i = 0, cp = 0; i < len; i += char.charCount(cp)) { cp = buffer[bufferLen] = char.codePointAt(termBuffer, i, len); startOffset[bufferLen] = start; endOffset[bufferLen] = end; bufferLen++; } } else { // normal offsets for (int i = 0, cp = 0, cpLen = 0; i < len; i += cpLen) { cp = buffer[bufferLen] = char.codePointAt(termBuffer, i, len); cpLen = char.charCount(cp); startOffset[bufferLen] = start; start = endOffset[bufferLen] = start + cpLen; bufferLen++; } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private org.apache.lucene.analysis.Token getNextSuffixInputToken(org.apache.lucene.analysis.Token token) throws java.io.IOException private Token getNextSuffixInputToken(Token token) { if (!suffix.incrementToken()) { return(null); } token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.PositionIncrement = posIncrAtt.PositionIncrement; token.Flags = flagsAtt.Flags; token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.Type = typeAtt.type(); token.Payload = payloadAtt.Payload; return(token); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { if (!keywordAtt.Keyword) { termAtt.Length = stemmer.stem(termAtt.buffer(), termAtt.length()); } return(true); } else { return(false); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { if (!keywordAttr.Keyword && termAttribute.length() > length) { termAttribute.Length = length; } return(true); } else { return(false); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { char[] chArray = termAtt.buffer(); int chLen = termAtt.length(); for (int i = 0; i < chLen;) { i += char.toChars(lowerCase(charUtils.codePointAt(chArray, i, chLen)), chArray, i); } return(true); } else { return(false); } }
/// <summary> /// Returns the next, stemmed, input Token. </summary> /// <returns> The stemmed form of a token. </returns> /// <exception cref="IOException"> If there is a low-level I/O error. </exception> //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (!input.incrementToken()) { return(false); } char[] term = termAttribute.buffer(); int len = termAttribute.length(); if ((!keywordAtt.Keyword) && stemmer.stem(term, len)) { termAttribute.setEmpty().append(stemmer.asCharSequence()); } return(true); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { if (!keywordAttr.Keyword) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length()); int newlen = stemmer.stem(termAtt.buffer(), termAtt.length()); termAtt.Length = newlen; } return(true); } else { return(false); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { if (!keywordAttr.Keyword) { // this stemmer increases word length by 1: worst case '*çom' -> '*ción' //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int len = termAtt.length(); int len = termAtt.length(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int newlen = stemmer.stem(termAtt.resizeBuffer(len+1), len); int newlen = stemmer.stem(termAtt.resizeBuffer(len + 1), len); termAtt.Length = newlen; } return(true); } else { return(false); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public final boolean incrementTokenClassic() throws java.io.IOException public bool incrementTokenClassic() { if (!input.incrementToken()) { return(false); } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final char[] buffer = termAtt.buffer(); char[] buffer = termAtt.buffer(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int bufferLength = termAtt.length(); int bufferLength = termAtt.length(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final String type = typeAtt.type(); string type = typeAtt.type(); if (type == APOSTROPHE_TYPE && bufferLength >= 2 && buffer[bufferLength - 2] == '\'' && (buffer[bufferLength - 1] == 's' || buffer[bufferLength - 1] == 'S')) // remove 's { // Strip last 2 characters off termAtt.Length = bufferLength - 2; } // remove dots else if (type == ACRONYM_TYPE) { int upto = 0; for (int i = 0; i < bufferLength; i++) { char c = buffer[i]; if (c != '.') { buffer[upto++] = c; } } termAtt.Length = upto; } return(true); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (input.incrementToken()) { char[] text = termAtt.buffer(); int length = termAtt.length(); for (int i = 0; i < length; i++) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final char ch = text[i]; char ch = text[i]; if (ch >= 0xFF01 && ch <= 0xFF5E) { // Fullwidth ASCII variants text[i] -= 0xFEE0; } else if (ch >= 0xFF65 && ch <= 0xFF9F) { // Halfwidth Katakana variants if ((ch == 0xFF9E || ch == 0xFF9F) && i > 0 && combine(text, i, ch)) { length = StemmerUtil.delete(text, i--, length); } else { text[i] = KANA_NORM[ch - 0xFF65]; } } } termAtt.Length = length; return(true); } else { return(false); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException public override bool incrementToken() { if (!input.incrementToken()) { return(false); } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final char[] buffer = termAtt.buffer(); char[] buffer = termAtt.buffer(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int length = termAtt.length(); int length = termAtt.length(); for (int i = 0; i < length; i++) { if (buffer[i] == '\'' || buffer[i] == '\u2019') { termAtt.Length = i; return(true); } } return(true); }