/* * (non-Javadoc) * * @see Mono.Lucene.Net.Analysis.TokenStream#next() */ public override bool IncrementToken() { ClearAttributes(); int posIncr = 1; while (true) { int tokenType = scanner.GetNextToken(); if (tokenType == StandardTokenizerImpl.YYEOF) { return(false); } if (scanner.Yylength() <= maxTokenLength) { posIncrAtt.SetPositionIncrement(posIncr); scanner.GetText(termAtt); int start = scanner.Yychar(); offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + termAtt.TermLength())); // This 'if' should be removed in the next release. For now, it converts // invalid acronyms to HOST. When removed, only the 'else' part should // remain. if (tokenType == StandardTokenizerImpl.ACRONYM_DEP) { if (replaceInvalidAcronym) { typeAtt.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]); termAtt.SetTermLength(termAtt.TermLength() - 1); // remove extra '.' } else { typeAtt.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]); } } else { typeAtt.SetType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]); } return(true); } // When we skip a too-long term, we still increment the // position increment else { posIncr++; } } }
public override bool IncrementToken() { if (input.IncrementToken()) { char[] buffer = termAtt.TermBuffer(); int length = termAtt.TermLength(); // If no characters actually require rewriting then we // just return token as-is: for (int i = 0; i < length; i++) { char c = buffer[i]; if (c >= '\u00c0' && c <= '\uFB06') { RemoveAccents(buffer, length); termAtt.SetTermBuffer(output, 0, outputPos); break; } } return(true); } else { return(false); } }
/// <summary>Returns the next token in the stream, or null at EOS. /// <p/>Removes <tt>'s</tt> from the end of words. /// <p/>Removes dots from acronyms. /// </summary> public override bool IncrementToken() { if (!input.IncrementToken()) { return(false); } char[] buffer = termAtt.TermBuffer(); int bufferLength = termAtt.TermLength(); System.String type = typeAtt.Type(); if ((System.Object)type == (System.Object)APOSTROPHE_TYPE && bufferLength >= 2 && buffer[bufferLength - 2] == '\'' && (buffer[bufferLength - 1] == 's' || buffer[bufferLength - 1] == 'S')) { // Strip last 2 characters off termAtt.SetTermLength(bufferLength - 2); } else if ((System.Object)type == (System.Object)ACRONYM_TYPE) { // remove dots int upto = 0; for (int i = 0; i < bufferLength; i++) { char c = buffer[i]; if (c != '.') { buffer[upto++] = c; } } termAtt.SetTermLength(upto); } return(true); }
public override bool IncrementToken() { if (!input.IncrementToken()) { return(false); } if (stemmer.Stem(termAtt.TermBuffer(), 0, termAtt.TermLength())) { termAtt.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength()); } return(true); }
/// <summary> Returns the next input Token whose term() is the right len</summary> public override bool IncrementToken() { // return the first non-stop word found while (input.IncrementToken()) { int len = termAtt.TermLength(); if (len >= min && len <= max) { return(true); } // note: else we ignore it but should we index each part of it? } // reached EOS -- return null return(false); }
public override bool IncrementToken() { if (input.IncrementToken()) { char[] buffer = termAtt.TermBuffer(); int length = termAtt.TermLength(); for (int i = 0; i < length; i++) { buffer[i] = System.Char.ToLower(buffer[i]); } return(true); } else { return(false); } }
/// <summary> Returns the next input Token whose term() is not a stop word.</summary> public override bool IncrementToken() { // return the first non-stop word found int skippedPositions = 0; while (input.IncrementToken()) { if (!stopWords.Contains(termAtt.TermBuffer(), 0, termAtt.TermLength())) { if (enablePositionIncrements) { posIncrAtt.SetPositionIncrement(posIncrAtt.GetPositionIncrement() + skippedPositions); } return true; } skippedPositions += posIncrAtt.GetPositionIncrement(); } // reached EOS -- return null return false; }
// Primary entry point (for first TermsHash) internal override void Add() { System.Diagnostics.Debug.Assert(!postingsCompacted); // We are first in the chain so we must "intern" the // term text into textStart address // Get the text of this term. char[] tokenText = termAtt.TermBuffer(); ; int tokenTextLen = termAtt.TermLength(); // Compute hashcode & replace any invalid UTF16 sequences int downto = tokenTextLen; int code = 0; while (downto > 0) { char ch = tokenText[--downto]; if (ch >= UnicodeUtil.UNI_SUR_LOW_START && ch <= UnicodeUtil.UNI_SUR_LOW_END) { if (0 == downto) { // Unpaired ch = tokenText[downto] = (char)(UnicodeUtil.UNI_REPLACEMENT_CHAR); } else { char ch2 = tokenText[downto - 1]; if (ch2 >= UnicodeUtil.UNI_SUR_HIGH_START && ch2 <= UnicodeUtil.UNI_SUR_HIGH_END) { // OK: high followed by low. This is a valid // surrogate pair. code = ((code * 31) + ch) * 31 + ch2; downto--; continue; } else { // Unpaired ch = tokenText[downto] = (char)(UnicodeUtil.UNI_REPLACEMENT_CHAR); } } } else if (ch >= UnicodeUtil.UNI_SUR_HIGH_START && (ch <= UnicodeUtil.UNI_SUR_HIGH_END || ch == 0xffff)) { // Unpaired or 0xffff ch = tokenText[downto] = (char)(UnicodeUtil.UNI_REPLACEMENT_CHAR); } code = (code * 31) + ch; } int hashPos = code & postingsHashMask; // Locate RawPostingList in hash p = postingsHash[hashPos]; if (p != null && !PostingEquals(tokenText, tokenTextLen)) { // Conflict: keep searching different locations in // the hash table. int inc = ((code >> 8) + code) | 1; do { code += inc; hashPos = code & postingsHashMask; p = postingsHash[hashPos]; }while (p != null && !PostingEquals(tokenText, tokenTextLen)); } if (p == null) { // First time we are seeing this token since we last // flushed the hash. int textLen1 = 1 + tokenTextLen; if (textLen1 + charPool.charUpto > DocumentsWriter.CHAR_BLOCK_SIZE) { if (textLen1 > DocumentsWriter.CHAR_BLOCK_SIZE) { // Just skip this term, to remain as robust as // possible during indexing. A TokenFilter // can be inserted into the analyzer chain if // other behavior is wanted (pruning the term // to a prefix, throwing an exception, etc). if (docState.maxTermPrefix == null) { docState.maxTermPrefix = new System.String(tokenText, 0, 30); } consumer.SkippingLongTerm(); return; } charPool.NextBuffer(); } // Refill? if (0 == perThread.freePostingsCount) { perThread.MorePostings(); } // Pull next free RawPostingList from free list p = perThread.freePostings[--perThread.freePostingsCount]; System.Diagnostics.Debug.Assert(p != null); char[] text = charPool.buffer; int textUpto = charPool.charUpto; p.textStart = textUpto + charPool.charOffset; charPool.charUpto += textLen1; Array.Copy(tokenText, 0, text, textUpto, tokenTextLen); text[textUpto + tokenTextLen] = (char)(0xffff); System.Diagnostics.Debug.Assert(postingsHash [hashPos] == null); postingsHash[hashPos] = p; numPostings++; if (numPostings == postingsHashHalfSize) { RehashPostings(2 * postingsHashSize); } // Init stream slices if (numPostingInt + intPool.intUpto > DocumentsWriter.INT_BLOCK_SIZE) { intPool.NextBuffer(); } if (DocumentsWriter.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt * ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.NextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; p.intStart = intUptoStart + intPool.intOffset; for (int i = 0; i < streamCount; i++) { int upto = bytePool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart + i] = upto + bytePool.byteOffset; } p.byteStart = intUptos[intUptoStart]; consumer.NewTerm(p); } else { intUptos = intPool.buffers[p.intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; intUptoStart = p.intStart & DocumentsWriter.INT_BLOCK_MASK; consumer.AddTerm(p); } if (doNextCall) { nextPerField.Add(p.textStart); } }