/// <summary>Creates one of these objects for a custom comparator/parser. </summary> internal Entry(System.String field, System.Object custom) { this.field = StringHelper.Intern(field); this.type = SortField.CUSTOM; this.custom = custom; this.locale = null; }
protected internal override System.Object CreateValue(IndexReader reader, Entry entryKey) { System.String field = StringHelper.Intern((System.String)entryKey.field); System.String[] retArray = new System.String[reader.MaxDoc()]; TermDocs termDocs = reader.TermDocs(); TermEnum termEnum = reader.Terms(new Term(field)); try { do { Term term = termEnum.Term(); if (term == null || (System.Object)term.Field() != (System.Object)field) { break; } System.String termval = term.Text(); termDocs.Seek(termEnum); while (termDocs.Next()) { retArray[termDocs.Doc()] = termval; } }while (termEnum.Next()); } finally { termDocs.Close(); termEnum.Close(); } return(retArray); }
internal Entry(System.String field, int type, System.Globalization.CultureInfo locale) { this.field = StringHelper.Intern(field); this.type = type; this.custom = null; this.locale = locale; }
/// <summary> Create a tokenized and indexed field that is not stored, optionally with /// storing term vectors. This is useful for pre-analyzed fields. /// The TokenStream is read only when the Document is added to the index, /// i.e. you may not close the TokenStream until {@link IndexWriter#AddDocument(Document)} /// has been called. /// /// </summary> /// <param name="name">The name of the field /// </param> /// <param name="tokenStream">The TokenStream with the content /// </param> /// <param name="termVector">Whether term vector should be stored /// </param> /// <throws> NullPointerException if name or tokenStream is <code>null</code> </throws> public Field(System.String name, TokenStream tokenStream, TermVector termVector) { if (name == null) { throw new System.NullReferenceException("name cannot be null"); } if (tokenStream == null) { throw new System.NullReferenceException("tokenStream cannot be null"); } this.name = StringHelper.Intern(name); // field names are interned this.fieldsData = null; this.tokenStream = tokenStream; this.isStored = false; this.isCompressed = false; this.isIndexed = true; this.isTokenized = true; this.isBinary = false; SetStoreTermVector(termVector); }
/// <summary> Enumerates all terms greater/equal than <code>lowerTerm</code> /// but less/equal than <code>upperTerm</code>. /// /// If an endpoint is null, it is said to be "open". Either or both /// endpoints may be open. Open endpoints may not be exclusive /// (you can't select all but the first or last term without /// explicitly specifying the term to exclude.) /// /// </summary> /// <param name="reader"> /// </param> /// <param name="field">An interned field that holds both lower and upper terms. /// </param> /// <param name="lowerTermText">The term text at the lower end of the range /// </param> /// <param name="upperTermText">The term text at the upper end of the range /// </param> /// <param name="includeLower">If true, the <code>lowerTerm</code> is included in the range. /// </param> /// <param name="includeUpper">If true, the <code>upperTerm</code> is included in the range. /// </param> /// <param name="collator">The collator to use to collate index Terms, to determine their /// membership in the range bounded by <code>lowerTerm</code> and /// <code>upperTerm</code>. /// /// </param> /// <throws> IOException </throws> public TermRangeTermEnum(IndexReader reader, System.String field, System.String lowerTermText, System.String upperTermText, bool includeLower, bool includeUpper, System.Globalization.CompareInfo collator) { this.collator = collator; this.upperTermText = upperTermText; this.lowerTermText = lowerTermText; this.includeLower = includeLower; this.includeUpper = includeUpper; this.field = StringHelper.Intern(field); // do a little bit of normalization... // open ended range queries should always be inclusive. if (this.lowerTermText == null) { this.lowerTermText = ""; this.includeLower = true; } if (this.upperTermText == null) { this.includeUpper = true; } System.String startTermText = collator == null?this.lowerTermText:""; SetEnum(reader.Terms(new Term(this.field, startTermText))); }
internal Entry(System.String field, int type, Parser parser) { this.field = StringHelper.Intern(field); this.type = type; this.custom = parser; this.locale = null; }
private FieldInfo AddInternal(System.String name, bool isIndexed, bool storeTermVector, bool storePositionWithTermVector, bool storeOffsetWithTermVector, bool omitNorms, bool storePayloads, bool omitTermFreqAndPositions) { name = StringHelper.Intern(name); FieldInfo fi = new FieldInfo(name, isIndexed, byNumber.Count, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); byNumber.Add(fi); byName[name] = fi; return(fi); }
protected internal override System.Object CreateValue(IndexReader reader, Entry entryKey) { System.String field = StringHelper.Intern((System.String)entryKey.field); TermEnum enumerator = reader.Terms(new Term(field)); try { Term term = enumerator.Term(); if (term == null) { throw new System.SystemException("no terms in field " + field + " - cannot determine type"); } System.Object ret = null; if ((System.Object)term.Field() == (System.Object)field) { System.String termtext = term.Text().Trim(); try { System.Int32.Parse(termtext); ret = wrapper.GetInts(reader, field); } catch (System.FormatException nfe1) { try { System.Int64.Parse(termtext); ret = wrapper.GetLongs(reader, field); } catch (System.FormatException nfe2) { try { SupportClass.Single.Parse(termtext); ret = wrapper.GetFloats(reader, field); } catch (System.FormatException nfe3) { ret = wrapper.GetStringIndex(reader, field); } } } } else { throw new System.SystemException("field \"" + field + "\" does not appear to be indexed"); } return(ret); } finally { enumerator.Close(); } }
private void Read(IndexInput input, System.String fileName) { int firstInt = input.ReadVInt(); if (firstInt < 0) { // This is a real format format = firstInt; } else { format = FORMAT_PRE; } if (format != FORMAT_PRE & format != FORMAT_START) { throw new CorruptIndexException("unrecognized format " + format + " in file \"" + fileName + "\""); } int size; if (format == FORMAT_PRE) { size = firstInt; } else { size = input.ReadVInt(); //read in the size } for (int i = 0; i < size; i++) { System.String name = StringHelper.Intern(input.ReadString()); byte bits = input.ReadByte(); bool isIndexed = (bits & IS_INDEXED) != 0; bool storeTermVector = (bits & STORE_TERMVECTOR) != 0; bool storePositionsWithTermVector = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0; bool storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0; bool omitNorms = (bits & OMIT_NORMS) != 0; bool storePayloads = (bits & STORE_PAYLOADS) != 0; bool omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0; AddInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); } if (input.GetFilePointer() != input.Length()) { throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.GetFilePointer() + " vs size " + input.Length()); } }
// Sets field & type, and ensures field is not NULL unless // type is SCORE or DOC private void InitFieldType(System.String field, int type) { this.type = type; if (field == null) { if (type != SCORE && type != DOC) { throw new System.ArgumentException("field can only be null when type is SCORE or DOC"); } } else { this.field = StringHelper.Intern(field); } }
internal static int DetectFieldType(IndexReader reader, System.String fieldKey) { System.String field = StringHelper.Intern(fieldKey); TermEnum enumerator = reader.Terms(new Term(field)); try { Term term = enumerator.Term(); if (term == null) { throw new System.SystemException("no terms in field " + field + " - cannot determine sort type"); } int ret = 0; if ((System.Object)term.Field() == (System.Object)field) { System.String termtext = term.Text().Trim(); int tmpI32; long tmpI64; float tmpF; if (System.Int32.TryParse(termtext, out tmpI32)) { ret = SortField.INT; } else if (System.Int64.TryParse(termtext, out tmpI64)) { ret = SortField.LONG; } else if (SupportClass.Single.TryParse(termtext, out tmpF)) { ret = SortField.FLOAT; } else { ret = SortField.STRING; } } else { throw new System.SystemException("field \"" + field + "\" does not appear to be indexed"); } return(ret); } finally { enumerator.Close(); } }
/// <summary> Create a stored field with binary value. Optionally the value may be compressed. /// /// </summary> /// <param name="name">The name of the field /// </param> /// <param name="value">The binary value /// </param> /// <param name="offset">Starting offset in value where this Field's bytes are /// </param> /// <param name="length">Number of bytes to use for this Field, starting at offset /// </param> /// <param name="store">How <code>value</code> should be stored (compressed or not) /// </param> /// <throws> IllegalArgumentException if store is <code>Store.NO</code> </throws> public Field(System.String name, byte[] value_Renamed, int offset, int length, Store store) { if (name == null) { throw new System.ArgumentException("name cannot be null"); } if (value_Renamed == null) { throw new System.ArgumentException("value cannot be null"); } this.name = StringHelper.Intern(name); // field names are interned fieldsData = value_Renamed; if (store == Store.YES) { isStored = true; isCompressed = false; } else if (store == Store.COMPRESS) { isStored = true; isCompressed = true; } else if (store == Store.NO) { throw new System.ArgumentException("binary values can't be unstored"); } else { throw new System.ArgumentException("unknown store parameter " + store); } isIndexed = false; isTokenized = false; omitTermFreqAndPositions = false; omitNorms = true; isBinary = true; binaryLength = length; binaryOffset = offset; SetStoreTermVector(TermVector.NO); }
public FieldForMerge(System.Object value_Renamed, FieldInfo fi, bool binary, bool compressed, bool tokenize) { this.isStored = true; this.fieldsData = value_Renamed; this.isCompressed = compressed; this.isBinary = binary; if (binary) { binaryLength = ((byte[])value_Renamed).Length; } this.isTokenized = tokenize; this.name = StringHelper.Intern(fi.name); this.isIndexed = fi.isIndexed; this.omitNorms = fi.omitNorms; this.omitTermFreqAndPositions = fi.omitTermFreqAndPositions; this.storeOffsetWithTermVector = fi.storeOffsetWithTermVector; this.storePositionWithTermVector = fi.storePositionWithTermVector; this.storeTermVector = fi.storeTermVector; }
private NumericRangeQuery(System.String field, int precisionStep, int valSize, System.ValueType min, System.ValueType max, bool minInclusive, bool maxInclusive) { System.Diagnostics.Debug.Assert((valSize == 32 || valSize == 64)); if (precisionStep < 1) { throw new System.ArgumentException("precisionStep must be >=1"); } this.field = StringHelper.Intern(field); this.precisionStep = precisionStep; this.valSize = valSize; this.min = min; this.max = max; this.minInclusive = minInclusive; this.maxInclusive = maxInclusive; // For bigger precisionSteps this query likely // hits too many terms, so set to CONSTANT_SCORE_FILTER right off // (especially as the FilteredTermEnum is costly if wasted only for AUTO tests because it // creates new enums from IndexReader for each sub-range) switch (valSize) { case 64: SetRewriteMethod((precisionStep > 6)?CONSTANT_SCORE_FILTER_REWRITE:CONSTANT_SCORE_AUTO_REWRITE_DEFAULT); break; case 32: SetRewriteMethod((precisionStep > 8)?CONSTANT_SCORE_FILTER_REWRITE:CONSTANT_SCORE_AUTO_REWRITE_DEFAULT); break; default: // should never happen throw new System.ArgumentException("valSize must be 32 or 64"); } // shortcut if upper bound == lower bound if (min != null && min.Equals(max)) { SetRewriteMethod(CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE); } }
/// <summary> Create a tokenized and indexed field that is not stored, optionally with /// storing term vectors. The Reader is read only when the Document is added to the index, /// i.e. you may not close the Reader until {@link IndexWriter#AddDocument(Document)} /// has been called. /// /// </summary> /// <param name="name">The name of the field /// </param> /// <param name="reader">The reader with the content /// </param> /// <param name="termVector">Whether term vector should be stored /// </param> /// <throws> NullPointerException if name or reader is <code>null</code> </throws> public Field(System.String name, System.IO.TextReader reader, TermVector termVector) { if (name == null) { throw new System.NullReferenceException("name cannot be null"); } if (reader == null) { throw new System.NullReferenceException("reader cannot be null"); } this.name = StringHelper.Intern(name); // field names are interned this.fieldsData = reader; this.isStored = false; this.isCompressed = false; this.isIndexed = true; this.isTokenized = true; this.isBinary = false; SetStoreTermVector(termVector); }
/// <summary>Constructs a Term with the given field and text. /// <p/>Note that a null field or null text value results in undefined /// behavior for most Lucene APIs that accept a Term parameter. /// </summary> public Term(System.String fld, System.String txt) { field = StringHelper.Intern(fld); text = txt; }
protected internal AbstractField(System.String name, Field.Store store, Field.Index index, Field.TermVector termVector) { if (name == null) { throw new System.NullReferenceException("name cannot be null"); } this.name = StringHelper.Intern(name); // field names are interned if (store == Field.Store.YES) { this.isStored = true; this.isCompressed = false; } else if (store == Field.Store.COMPRESS) { this.isStored = true; this.isCompressed = true; } else if (store == Field.Store.NO) { this.isStored = false; this.isCompressed = false; } else { throw new System.ArgumentException("unknown store parameter " + store); } if (index == Field.Index.NO) { this.isIndexed = false; this.isTokenized = false; } else if (index == Field.Index.ANALYZED) { this.isIndexed = true; this.isTokenized = true; } else if (index == Field.Index.NOT_ANALYZED) { this.isIndexed = true; this.isTokenized = false; } else if (index == Field.Index.NOT_ANALYZED_NO_NORMS) { this.isIndexed = true; this.isTokenized = false; this.omitNorms = true; } else if (index == Field.Index.ANALYZED_NO_NORMS) { this.isIndexed = true; this.isTokenized = true; this.omitNorms = true; } else { throw new System.ArgumentException("unknown index parameter " + index); } this.isBinary = false; SetStoreTermVector(termVector); }
protected internal override System.Object CreateValue(IndexReader reader, Entry entryKey) { System.String field = StringHelper.Intern((System.String)entryKey.field); int[] retArray = new int[reader.MaxDoc()]; System.String[] mterms = new System.String[reader.MaxDoc() + 1]; TermDocs termDocs = reader.TermDocs(); TermEnum termEnum = reader.Terms(new Term(field)); int t = 0; // current term number // an entry for documents that have no terms in this field // should a document with no terms be at top or bottom? // this puts them at the top - if it is changed, FieldDocSortedHitQueue // needs to change as well. mterms[t++] = null; try { do { Term term = termEnum.Term(); if (term == null || term.Field() != field || t >= mterms.Length) { break; } // store term text mterms[t] = term.Text(); termDocs.Seek(termEnum); while (termDocs.Next()) { retArray[termDocs.Doc()] = t; } t++; }while (termEnum.Next()); } finally { termDocs.Close(); termEnum.Close(); } if (t == 0) { // if there are no terms, make the term array // have a single null entry mterms = new System.String[1]; } else if (t < mterms.Length) { // if there are less terms than documents, // trim off the dead array space System.String[] terms = new System.String[t]; Array.Copy(mterms, 0, terms, 0, t); mterms = terms; } StringIndex value_Renamed = new StringIndex(retArray, mterms); return(value_Renamed); }
internal Term(System.String fld, System.String txt, bool intern) { field = intern?StringHelper.Intern(fld):fld; // field names are interned text = txt; // unless already known to be }
internal void OnDeserialized(System.Runtime.Serialization.StreamingContext context) { field = StringHelper.Intern(field); }
/// <summary> Create a field by specifying its name, value and how it will /// be saved in the index. /// /// </summary> /// <param name="name">The name of the field /// </param> /// <param name="internName">Whether to .intern() name or not /// </param> /// <param name="value">The string to process /// </param> /// <param name="store">Whether <code>value</code> should be stored in the index /// </param> /// <param name="index">Whether the field should be indexed, and if so, if it should /// be tokenized before indexing /// </param> /// <param name="termVector">Whether term vector should be stored /// </param> /// <throws> NullPointerException if name or value is <code>null</code> </throws> /// <throws> IllegalArgumentException in any of the following situations: </throws> /// <summary> <ul> /// <li>the field is neither stored nor indexed</li> /// <li>the field is not indexed but termVector is <code>TermVector.YES</code></li> /// </ul> /// </summary> public Field(System.String name, bool internName, System.String value_Renamed, Store store, Index index, TermVector termVector) { if (name == null) { throw new System.NullReferenceException("name cannot be null"); } if (value_Renamed == null) { throw new System.NullReferenceException("value cannot be null"); } if (name.Length == 0 && value_Renamed.Length == 0) { throw new System.ArgumentException("name and value cannot both be empty"); } if (index == Index.NO && store == Store.NO) { throw new System.ArgumentException("it doesn't make sense to have a field that " + "is neither indexed nor stored"); } if (index == Index.NO && termVector != TermVector.NO) { throw new System.ArgumentException("cannot store term vector information " + "for a field that is not indexed"); } if (internName) { // field names are optionally interned name = StringHelper.Intern(name); } this.name = name; this.fieldsData = value_Renamed; if (store == Store.YES) { this.isStored = true; this.isCompressed = false; } else if (store == Store.COMPRESS) { this.isStored = true; this.isCompressed = true; } else if (store == Store.NO) { this.isStored = false; this.isCompressed = false; } else { throw new System.ArgumentException("unknown store parameter " + store); } if (index == Index.NO) { this.isIndexed = false; this.isTokenized = false; this.omitTermFreqAndPositions = false; this.omitNorms = true; } else if (index == Index.ANALYZED) { this.isIndexed = true; this.isTokenized = true; } else if (index == Index.NOT_ANALYZED) { this.isIndexed = true; this.isTokenized = false; } else if (index == Index.NOT_ANALYZED_NO_NORMS) { this.isIndexed = true; this.isTokenized = false; this.omitNorms = true; } else if (index == Index.ANALYZED_NO_NORMS) { this.isIndexed = true; this.isTokenized = true; this.omitNorms = true; } else { throw new System.ArgumentException("unknown index parameter " + index); } this.isBinary = false; SetStoreTermVector(termVector); }