/// <exception cref="ParseException">throw in overridden method to disallow /// </exception> public /*protected internal*/ virtual Query GetFieldQuery(System.String field, System.String queryText) { // Use the analyzer to get all the tokens, and then build a TermQuery, // PhraseQuery, or nothing based on the term count TokenStream source; try { source = analyzer.ReusableTokenStream(field, new System.IO.StringReader(queryText)); source.Reset(); } catch (System.IO.IOException e) { source = analyzer.TokenStream(field, new System.IO.StringReader(queryText)); } CachingTokenFilter buffer = new CachingTokenFilter(source); TermAttribute termAtt = null; PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; bool success = false; try { buffer.Reset(); success = true; } catch (System.IO.IOException e) { // success==false if we hit an exception } if (success) { if (buffer.HasAttribute(typeof(TermAttribute))) { termAtt = (TermAttribute) buffer.GetAttribute(typeof(TermAttribute)); } if (buffer.HasAttribute(typeof(PositionIncrementAttribute))) { posIncrAtt = (PositionIncrementAttribute) buffer.GetAttribute(typeof(PositionIncrementAttribute)); } } int positionCount = 0; bool severalTokensAtSamePosition = false; bool hasMoreTokens = false; if (termAtt != null) { try { hasMoreTokens = buffer.IncrementToken(); while (hasMoreTokens) { numTokens++; int positionIncrement = (posIncrAtt != null)?posIncrAtt.GetPositionIncrement():1; if (positionIncrement != 0) { positionCount += positionIncrement; } else { severalTokensAtSamePosition = true; } hasMoreTokens = buffer.IncrementToken(); } } catch (System.IO.IOException e) { // ignore } } try { // rewind the buffer stream buffer.Reset(); // close original stream - all tokens buffered source.Close(); } catch (System.IO.IOException e) { // ignore } if (numTokens == 0) return null; else if (numTokens == 1) { System.String term = null; try { bool hasNext = buffer.IncrementToken(); System.Diagnostics.Debug.Assert(hasNext == true); term = termAtt.Term(); } catch (System.IO.IOException e) { // safe to ignore, because we know the number of tokens } return NewTermQuery(new Term(field, term)); } else { if (severalTokensAtSamePosition) { if (positionCount == 1) { // no phrase query: BooleanQuery q = NewBooleanQuery(true); for (int i = 0; i < numTokens; i++) { System.String term = null; try { bool hasNext = buffer.IncrementToken(); System.Diagnostics.Debug.Assert(hasNext == true); term = termAtt.Term(); } catch (System.IO.IOException e) { // safe to ignore, because we know the number of tokens } Query currentQuery = NewTermQuery(new Term(field, term)); q.Add(currentQuery, BooleanClause.Occur.SHOULD); } return q; } else { // phrase query: MultiPhraseQuery mpq = NewMultiPhraseQuery(); mpq.SetSlop(phraseSlop); System.Collections.ArrayList multiTerms = new System.Collections.ArrayList(); int position = - 1; for (int i = 0; i < numTokens; i++) { System.String term = null; int positionIncrement = 1; try { bool hasNext = buffer.IncrementToken(); System.Diagnostics.Debug.Assert(hasNext == true); term = termAtt.Term(); if (posIncrAtt != null) { positionIncrement = posIncrAtt.GetPositionIncrement(); } } catch (System.IO.IOException e) { // safe to ignore, because we know the number of tokens } if (positionIncrement > 0 && multiTerms.Count > 0) { if (enablePositionIncrements) { mpq.Add((Term[]) multiTerms.ToArray(typeof(Term)), position); } else { mpq.Add((Term[]) multiTerms.ToArray(typeof(Term))); } multiTerms.Clear(); } position += positionIncrement; multiTerms.Add(new Term(field, term)); } if (enablePositionIncrements) { mpq.Add((Term[]) multiTerms.ToArray(typeof(Term)), position); } else { mpq.Add((Term[]) multiTerms.ToArray(typeof(Term))); } return mpq; } } else { PhraseQuery pq = NewPhraseQuery(); pq.SetSlop(phraseSlop); int position = - 1; for (int i = 0; i < numTokens; i++) { System.String term = null; int positionIncrement = 1; try { bool hasNext = buffer.IncrementToken(); System.Diagnostics.Debug.Assert(hasNext == true); term = termAtt.Term(); if (posIncrAtt != null) { positionIncrement = posIncrAtt.GetPositionIncrement(); } } catch (System.IO.IOException e) { // safe to ignore, because we know the number of tokens } if (enablePositionIncrements) { position += positionIncrement; pq.Add(new Term(field, term), position); } else { pq.Add(new Term(field, term)); } } return pq; } } }
public virtual void TestEndOffsetPositionWithCachingTokenFilter() { MockRAMDirectory dir = new MockRAMDirectory(); Analyzer analyzer = new WhitespaceAnalyzer(); IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); TokenStream stream = new CachingTokenFilter(analyzer.TokenStream("field", new System.IO.StringReader("abcd "))); Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS); doc.Add(f); doc.Add(f); w.AddDocument(doc); w.Close(); IndexReader r = IndexReader.Open(dir); TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.GetTermFreqVector(0, "field")).GetOffsets(0); Assert.AreEqual(2, termOffsets.Length); Assert.AreEqual(0, termOffsets[0].GetStartOffset()); Assert.AreEqual(4, termOffsets[0].GetEndOffset()); Assert.AreEqual(8, termOffsets[1].GetStartOffset()); Assert.AreEqual(12, termOffsets[1].GetEndOffset()); r.Close(); dir.Close(); }
/// <summary> /// Creates a query from the analysis chain. /// <p> /// Expert: this is more useful for subclasses such as queryparsers. /// If using this class directly, just use <seealso cref="#createBooleanQuery(String, String)"/> /// and <seealso cref="#createPhraseQuery(String, String)"/> </summary> /// <param name="analyzer"> analyzer used for this query </param> /// <param name="operator"> default boolean operator used for this query </param> /// <param name="field"> field to create queries against </param> /// <param name="queryText"> text to be passed to the analysis chain </param> /// <param name="quoted"> true if phrases should be generated when terms occur at more than one position </param> /// <param name="phraseSlop"> slop factor for phrase/multiphrase queries </param> protected internal Query CreateFieldQuery(Analyzer analyzer, BooleanClause.Occur @operator, string field, string queryText, bool quoted, int phraseSlop) { Debug.Assert(@operator == BooleanClause.Occur.SHOULD || @operator == BooleanClause.Occur.MUST); // Use the analyzer to get all the tokens, and then build a TermQuery, // PhraseQuery, or nothing based on the term count CachingTokenFilter buffer = null; ITermToBytesRefAttribute termAtt = null; IPositionIncrementAttribute posIncrAtt = null; int numTokens = 0; int positionCount = 0; bool severalTokensAtSamePosition = false; bool hasMoreTokens = false; TokenStream source = null; try { source = analyzer.TokenStream(field, new StringReader(queryText)); source.Reset(); buffer = new CachingTokenFilter(source); buffer.Reset(); if (buffer.HasAttribute<ITermToBytesRefAttribute>()) { termAtt = buffer.GetAttribute<ITermToBytesRefAttribute>(); } if (buffer.HasAttribute<IPositionIncrementAttribute>()) { posIncrAtt = buffer.GetAttribute<IPositionIncrementAttribute>(); } if (termAtt != null) { try { hasMoreTokens = buffer.IncrementToken(); while (hasMoreTokens) { numTokens++; int positionIncrement = (posIncrAtt != null) ? posIncrAtt.PositionIncrement : 1; if (positionIncrement != 0) { positionCount += positionIncrement; } else { severalTokensAtSamePosition = true; } hasMoreTokens = buffer.IncrementToken(); } } catch (System.IO.IOException) { // ignore } } } catch (System.IO.IOException e) { throw new Exception("Error analyzing query text", e); } finally { IOUtils.CloseWhileHandlingException(source); } // rewind the buffer stream buffer.Reset(); BytesRef bytes = termAtt == null ? null : termAtt.BytesRef; if (numTokens == 0) { return null; } else if (numTokens == 1) { try { bool hasNext = buffer.IncrementToken(); Debug.Assert(hasNext == true); termAtt.FillBytesRef(); } catch (System.IO.IOException) { // safe to ignore, because we know the number of tokens } return NewTermQuery(new Term(field, BytesRef.DeepCopyOf(bytes))); } else { if (severalTokensAtSamePosition || (!quoted)) { if (positionCount == 1 || (!quoted)) { // no phrase query: if (positionCount == 1) { // simple case: only one position, with synonyms BooleanQuery q = NewBooleanQuery(true); for (int i = 0; i < numTokens; i++) { try { bool hasNext = buffer.IncrementToken(); Debug.Assert(hasNext == true); termAtt.FillBytesRef(); } catch (System.IO.IOException) { // safe to ignore, because we know the number of tokens } Query currentQuery = NewTermQuery(new Term(field, BytesRef.DeepCopyOf(bytes))); q.Add(currentQuery, BooleanClause.Occur.SHOULD); } return q; } else { // multiple positions BooleanQuery q = NewBooleanQuery(false); Query currentQuery = null; for (int i = 0; i < numTokens; i++) { try { bool hasNext = buffer.IncrementToken(); Debug.Assert(hasNext == true); termAtt.FillBytesRef(); } catch (System.IO.IOException) { // safe to ignore, because we know the number of tokens } if (posIncrAtt != null && posIncrAtt.PositionIncrement == 0) { if (!(currentQuery is BooleanQuery)) { Query t = currentQuery; currentQuery = NewBooleanQuery(true); ((BooleanQuery)currentQuery).Add(t, BooleanClause.Occur.SHOULD); } ((BooleanQuery)currentQuery).Add(NewTermQuery(new Term(field, BytesRef.DeepCopyOf(bytes))), BooleanClause.Occur.SHOULD); } else { if (currentQuery != null) { q.Add(currentQuery, @operator); } currentQuery = NewTermQuery(new Term(field, BytesRef.DeepCopyOf(bytes))); } } q.Add(currentQuery, @operator); return q; } } else { // phrase query: MultiPhraseQuery mpq = NewMultiPhraseQuery(); mpq.Slop = phraseSlop; IList<Term> multiTerms = new List<Term>(); int position = -1; for (int i = 0; i < numTokens; i++) { int positionIncrement = 1; try { bool hasNext = buffer.IncrementToken(); Debug.Assert(hasNext == true); termAtt.FillBytesRef(); if (posIncrAtt != null) { positionIncrement = posIncrAtt.PositionIncrement; } } catch (System.IO.IOException) { // safe to ignore, because we know the number of tokens } if (positionIncrement > 0 && multiTerms.Count > 0) { if (EnablePositionIncrements_Renamed) { mpq.Add(multiTerms.ToArray(), position); } else { mpq.Add(multiTerms.ToArray()); } multiTerms.Clear(); } position += positionIncrement; multiTerms.Add(new Term(field, BytesRef.DeepCopyOf(bytes))); } if (EnablePositionIncrements_Renamed) { mpq.Add(multiTerms.ToArray(), position); } else { mpq.Add(multiTerms.ToArray()); } return mpq; } } else { PhraseQuery pq = NewPhraseQuery(); pq.Slop = phraseSlop; int position = -1; for (int i = 0; i < numTokens; i++) { int positionIncrement = 1; try { bool hasNext = buffer.IncrementToken(); Debug.Assert(hasNext == true); termAtt.FillBytesRef(); if (posIncrAtt != null) { positionIncrement = posIncrAtt.PositionIncrement; } } catch (System.IO.IOException) { // safe to ignore, because we know the number of tokens } if (EnablePositionIncrements_Renamed) { position += positionIncrement; pq.Add(new Term(field, BytesRef.DeepCopyOf(bytes)), position); } else { pq.Add(new Term(field, BytesRef.DeepCopyOf(bytes))); } } return pq; } } }
public virtual void TestEndOffsetPositionWithCachingTokenFilter() { Directory dir = NewDirectory(); Analyzer analyzer = new MockAnalyzer(Random()); IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); IOException priorException = null; TokenStream stream = analyzer.TokenStream("field", new StringReader("abcd ")); try { stream.Reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct? TokenStream cachedStream = new CachingTokenFilter(stream); FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.StoreTermVectors = true; customType.StoreTermVectorPositions = true; customType.StoreTermVectorOffsets = true; Field f = new Field("field", cachedStream, customType); doc.Add(f); doc.Add(f); w.AddDocument(doc); } catch (IOException e) { priorException = e; } finally { IOUtils.CloseWhileHandlingException(priorException, stream); } w.Dispose(); IndexReader r = DirectoryReader.Open(dir); TermsEnum termsEnum = r.GetTermVectors(0).Terms("field").Iterator(null); Assert.IsNotNull(termsEnum.Next()); DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null); Assert.AreEqual(2, termsEnum.TotalTermFreq()); Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.NextPosition(); Assert.AreEqual(0, dpEnum.StartOffset()); Assert.AreEqual(4, dpEnum.EndOffset()); dpEnum.NextPosition(); Assert.AreEqual(8, dpEnum.StartOffset()); Assert.AreEqual(12, dpEnum.EndOffset()); Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc()); r.Dispose(); dir.Dispose(); }