/// <summary>Creates one of these objects for a custom comparator/parser. </summary>
internal Entry(System.String field, System.Object custom)
{
this.field = StringHelper.Intern(field);
this.type = SortField.CUSTOM;
this.custom = custom;
this.locale = null;
}
/// <summary> Enumerates all terms greater/equal than <code>lowerTerm</code>
/// but less/equal than <code>upperTerm</code>.
///
/// If an endpoint is null, it is said to be "open". Either or both
/// endpoints may be open. Open endpoints may not be exclusive
/// (you can't select all but the first or last term without
/// explicitly specifying the term to exclude.)
///
/// </summary>
/// <param name="reader">
/// </param>
/// <param name="field">An interned field that holds both lower and upper terms.
/// </param>
/// <param name="lowerTermText">The term text at the lower end of the range
/// </param>
/// <param name="upperTermText">The term text at the upper end of the range
/// </param>
/// <param name="includeLower">If true, the <code>lowerTerm</code> is included in the range.
/// </param>
/// <param name="includeUpper">If true, the <code>upperTerm</code> is included in the range.
/// </param>
/// <param name="collator">The collator to use to collate index Terms, to determine their
/// membership in the range bounded by <code>lowerTerm</code> and
/// <code>upperTerm</code>.
///
/// </param>
/// <throws> IOException </throws>
public TermRangeTermEnum(IndexReader reader, System.String field, System.String lowerTermText, System.String upperTermText, bool includeLower, bool includeUpper, System.Globalization.CompareInfo collator)
{
this.collator = collator;
this.upperTermText = upperTermText;
this.lowerTermText = lowerTermText;
this.includeLower = includeLower;
this.includeUpper = includeUpper;
this.field = StringHelper.Intern(field);
// do a little bit of normalization...
// open ended range queries should always be inclusive.
if (this.lowerTermText == null)
{
this.lowerTermText = "";
this.includeLower = true;
}
if (this.upperTermText == null)
{
this.includeUpper = true;
}
System.String startTermText = collator == null?this.lowerTermText:"";
SetEnum(reader.Terms(new Term(this.field, startTermText)));
}
internal Entry(System.String field, int type, System.Globalization.CultureInfo locale)
{
this.field = StringHelper.Intern(field);
this.type = type;
this.custom = null;
this.locale = locale;
}
/// <summary> Create a tokenized and indexed field that is not stored, optionally with
/// storing term vectors. This is useful for pre-analyzed fields.
/// The TokenStream is read only when the Document is added to the index,
/// i.e. you may not close the TokenStream until {@link IndexWriter#AddDocument(Document)}
/// has been called.
///
/// </summary>
/// <param name="name">The name of the field
/// </param>
/// <param name="tokenStream">The TokenStream with the content
/// </param>
/// <param name="termVector">Whether term vector should be stored
/// </param>
/// <throws> NullPointerException if name or tokenStream is <code>null</code> </throws>
public Field(System.String name, TokenStream tokenStream, TermVector termVector)
{
if (name == null)
{
throw new System.NullReferenceException("name cannot be null");
}
if (tokenStream == null)
{
throw new System.NullReferenceException("tokenStream cannot be null");
}
this.name = StringHelper.Intern(name); // field names are interned
this.fieldsData = null;
this.tokenStream = tokenStream;
this.isStored = false;
this.isCompressed = false;
this.isIndexed = true;
this.isTokenized = true;
this.isBinary = false;
SetStoreTermVector(termVector);
}
protected internal override System.Object CreateValue(IndexReader reader, Entry entryKey)
{
System.String field = StringHelper.Intern((System.String)entryKey.field);
System.String[] retArray = new System.String[reader.MaxDoc()];
TermDocs termDocs = reader.TermDocs();
TermEnum termEnum = reader.Terms(new Term(field));
try
{
do
{
Term term = termEnum.Term();
if (term == null || (System.Object)term.Field() != (System.Object)field)
{
break;
}
System.String termval = term.Text();
termDocs.Seek(termEnum);
while (termDocs.Next())
{
retArray[termDocs.Doc()] = termval;
}
}while (termEnum.Next());
}
finally
{
termDocs.Close();
termEnum.Close();
}
return(retArray);
}
internal Entry(System.String field, int type, Parser parser)
{
this.field = StringHelper.Intern(field);
this.type = type;
this.custom = parser;
this.locale = null;
}
private FieldInfo AddInternal(System.String name, bool isIndexed, bool storeTermVector, bool storePositionWithTermVector, bool storeOffsetWithTermVector, bool omitNorms, bool storePayloads, bool omitTermFreqAndPositions)
{
name = StringHelper.Intern(name);
FieldInfo fi = new FieldInfo(name, isIndexed, byNumber.Count, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
byNumber.Add(fi);
byName[name] = fi;
return(fi);
}
protected internal override System.Object CreateValue(IndexReader reader, Entry entryKey)
{
System.String field = StringHelper.Intern((System.String)entryKey.field);
TermEnum enumerator = reader.Terms(new Term(field));
try
{
Term term = enumerator.Term();
if (term == null)
{
throw new System.SystemException("no terms in field " + field + " - cannot determine type");
}
System.Object ret = null;
if ((System.Object)term.Field() == (System.Object)field)
{
System.String termtext = term.Text().Trim();
try
{
System.Int32.Parse(termtext);
ret = wrapper.GetInts(reader, field);
}
catch (System.FormatException nfe1)
{
try
{
System.Int64.Parse(termtext);
ret = wrapper.GetLongs(reader, field);
}
catch (System.FormatException nfe2)
{
try
{
SupportClass.Single.Parse(termtext);
ret = wrapper.GetFloats(reader, field);
}
catch (System.FormatException nfe3)
{
ret = wrapper.GetStringIndex(reader, field);
}
}
}
}
else
{
throw new System.SystemException("field \"" + field + "\" does not appear to be indexed");
}
return(ret);
}
finally
{
enumerator.Close();
}
}
private void Read(IndexInput input, System.String fileName)
{
int firstInt = input.ReadVInt();
if (firstInt < 0)
{
// This is a real format
format = firstInt;
}
else
{
format = FORMAT_PRE;
}
if (format != FORMAT_PRE & format != FORMAT_START)
{
throw new CorruptIndexException("unrecognized format " + format + " in file \"" + fileName + "\"");
}
int size;
if (format == FORMAT_PRE)
{
size = firstInt;
}
else
{
size = input.ReadVInt(); //read in the size
}
for (int i = 0; i < size; i++)
{
System.String name = StringHelper.Intern(input.ReadString());
byte bits = input.ReadByte();
bool isIndexed = (bits & IS_INDEXED) != 0;
bool storeTermVector = (bits & STORE_TERMVECTOR) != 0;
bool storePositionsWithTermVector = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
bool storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
bool omitNorms = (bits & OMIT_NORMS) != 0;
bool storePayloads = (bits & STORE_PAYLOADS) != 0;
bool omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;
AddInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
}
if (input.GetFilePointer() != input.Length())
{
throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.GetFilePointer() + " vs size " + input.Length());
}
}
// Sets field & type, and ensures field is not NULL unless
// type is SCORE or DOC
private void InitFieldType(System.String field, int type)
{
this.type = type;
if (field == null)
{
if (type != SCORE && type != DOC)
{
throw new System.ArgumentException("field can only be null when type is SCORE or DOC");
}
}
else
{
this.field = StringHelper.Intern(field);
}
}
internal static int DetectFieldType(IndexReader reader, System.String fieldKey)
{
System.String field = StringHelper.Intern(fieldKey);
TermEnum enumerator = reader.Terms(new Term(field));
try
{
Term term = enumerator.Term();
if (term == null)
{
throw new System.SystemException("no terms in field " + field + " - cannot determine sort type");
}
int ret = 0;
if ((System.Object)term.Field() == (System.Object)field)
{
System.String termtext = term.Text().Trim();
int tmpI32; long tmpI64; float tmpF;
if (System.Int32.TryParse(termtext, out tmpI32))
{
ret = SortField.INT;
}
else if (System.Int64.TryParse(termtext, out tmpI64))
{
ret = SortField.LONG;
}
else if (SupportClass.Single.TryParse(termtext, out tmpF))
{
ret = SortField.FLOAT;
}
else
{
ret = SortField.STRING;
}
}
else
{
throw new System.SystemException("field \"" + field + "\" does not appear to be indexed");
}
return(ret);
}
finally
{
enumerator.Close();
}
}
/// <summary> Create a stored field with binary value. Optionally the value may be compressed.
///
/// </summary>
/// <param name="name">The name of the field
/// </param>
/// <param name="value">The binary value
/// </param>
/// <param name="offset">Starting offset in value where this Field's bytes are
/// </param>
/// <param name="length">Number of bytes to use for this Field, starting at offset
/// </param>
/// <param name="store">How <code>value</code> should be stored (compressed or not)
/// </param>
/// <throws> IllegalArgumentException if store is <code>Store.NO</code> </throws>
public Field(System.String name, byte[] value_Renamed, int offset, int length, Store store)
{
if (name == null)
{
throw new System.ArgumentException("name cannot be null");
}
if (value_Renamed == null)
{
throw new System.ArgumentException("value cannot be null");
}
this.name = StringHelper.Intern(name); // field names are interned
fieldsData = value_Renamed;
if (store == Store.YES)
{
isStored = true;
isCompressed = false;
}
else if (store == Store.COMPRESS)
{
isStored = true;
isCompressed = true;
}
else if (store == Store.NO)
{
throw new System.ArgumentException("binary values can't be unstored");
}
else
{
throw new System.ArgumentException("unknown store parameter " + store);
}
isIndexed = false;
isTokenized = false;
omitTermFreqAndPositions = false;
omitNorms = true;
isBinary = true;
binaryLength = length;
binaryOffset = offset;
SetStoreTermVector(TermVector.NO);
}
public FieldForMerge(System.Object value_Renamed, FieldInfo fi, bool binary, bool compressed, bool tokenize)
{
this.isStored = true;
this.fieldsData = value_Renamed;
this.isCompressed = compressed;
this.isBinary = binary;
if (binary)
{
binaryLength = ((byte[])value_Renamed).Length;
}
this.isTokenized = tokenize;
this.name = StringHelper.Intern(fi.name);
this.isIndexed = fi.isIndexed;
this.omitNorms = fi.omitNorms;
this.omitTermFreqAndPositions = fi.omitTermFreqAndPositions;
this.storeOffsetWithTermVector = fi.storeOffsetWithTermVector;
this.storePositionWithTermVector = fi.storePositionWithTermVector;
this.storeTermVector = fi.storeTermVector;
}
private NumericRangeQuery(System.String field, int precisionStep, int valSize, System.ValueType min, System.ValueType max, bool minInclusive, bool maxInclusive)
{
System.Diagnostics.Debug.Assert((valSize == 32 || valSize == 64));
if (precisionStep < 1)
{
throw new System.ArgumentException("precisionStep must be >=1");
}
this.field = StringHelper.Intern(field);
this.precisionStep = precisionStep;
this.valSize = valSize;
this.min = min;
this.max = max;
this.minInclusive = minInclusive;
this.maxInclusive = maxInclusive;
// For bigger precisionSteps this query likely
// hits too many terms, so set to CONSTANT_SCORE_FILTER right off
// (especially as the FilteredTermEnum is costly if wasted only for AUTO tests because it
// creates new enums from IndexReader for each sub-range)
switch (valSize)
{
case 64:
SetRewriteMethod((precisionStep > 6)?CONSTANT_SCORE_FILTER_REWRITE:CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
break;
case 32:
SetRewriteMethod((precisionStep > 8)?CONSTANT_SCORE_FILTER_REWRITE:CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
break;
default:
// should never happen
throw new System.ArgumentException("valSize must be 32 or 64");
}
// shortcut if upper bound == lower bound
if (min != null && min.Equals(max))
{
SetRewriteMethod(CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
}
}
/// <summary> Create a tokenized and indexed field that is not stored, optionally with
/// storing term vectors. The Reader is read only when the Document is added to the index,
/// i.e. you may not close the Reader until {@link IndexWriter#AddDocument(Document)}
/// has been called.
///
/// </summary>
/// <param name="name">The name of the field
/// </param>
/// <param name="reader">The reader with the content
/// </param>
/// <param name="termVector">Whether term vector should be stored
/// </param>
/// <throws> NullPointerException if name or reader is <code>null</code> </throws>
public Field(System.String name, System.IO.TextReader reader, TermVector termVector)
{
if (name == null)
{
throw new System.NullReferenceException("name cannot be null");
}
if (reader == null)
{
throw new System.NullReferenceException("reader cannot be null");
}
this.name = StringHelper.Intern(name); // field names are interned
this.fieldsData = reader;
this.isStored = false;
this.isCompressed = false;
this.isIndexed = true;
this.isTokenized = true;
this.isBinary = false;
SetStoreTermVector(termVector);
}
/// <summary>Constructs a Term with the given field and text.
/// <p/>Note that a null field or null text value results in undefined
/// behavior for most Lucene APIs that accept a Term parameter.
/// </summary>
public Term(System.String fld, System.String txt)
{
field = StringHelper.Intern(fld);
text = txt;
}
protected internal AbstractField(System.String name, Field.Store store, Field.Index index, Field.TermVector termVector)
{
if (name == null)
{
throw new System.NullReferenceException("name cannot be null");
}
this.name = StringHelper.Intern(name); // field names are interned
if (store == Field.Store.YES)
{
this.isStored = true;
this.isCompressed = false;
}
else if (store == Field.Store.COMPRESS)
{
this.isStored = true;
this.isCompressed = true;
}
else if (store == Field.Store.NO)
{
this.isStored = false;
this.isCompressed = false;
}
else
{
throw new System.ArgumentException("unknown store parameter " + store);
}
if (index == Field.Index.NO)
{
this.isIndexed = false;
this.isTokenized = false;
}
else if (index == Field.Index.ANALYZED)
{
this.isIndexed = true;
this.isTokenized = true;
}
else if (index == Field.Index.NOT_ANALYZED)
{
this.isIndexed = true;
this.isTokenized = false;
}
else if (index == Field.Index.NOT_ANALYZED_NO_NORMS)
{
this.isIndexed = true;
this.isTokenized = false;
this.omitNorms = true;
}
else if (index == Field.Index.ANALYZED_NO_NORMS)
{
this.isIndexed = true;
this.isTokenized = true;
this.omitNorms = true;
}
else
{
throw new System.ArgumentException("unknown index parameter " + index);
}
this.isBinary = false;
SetStoreTermVector(termVector);
}
/// <summary> Add a complete document specified by all its term vectors. If document has no
/// term vectors, add value for tvx.
///
/// </summary>
/// <param name="vectors">
/// </param>
/// <throws> IOException </throws>
public void AddAllDocVectors(TermFreqVector[] vectors)
{
tvx.WriteLong(tvd.GetFilePointer());
tvx.WriteLong(tvf.GetFilePointer());
if (vectors != null)
{
int numFields = vectors.Length;
tvd.WriteVInt(numFields);
long[] fieldPointers = new long[numFields];
for (int i = 0; i < numFields; i++)
{
fieldPointers[i] = tvf.GetFilePointer();
int fieldNumber = fieldInfos.FieldNumber(vectors[i].GetField());
// 1st pass: write field numbers to tvd
tvd.WriteVInt(fieldNumber);
int numTerms = vectors[i].Size();
tvf.WriteVInt(numTerms);
TermPositionVector tpVector;
byte bits;
bool storePositions;
bool storeOffsets;
if (vectors[i] is TermPositionVector)
{
// May have positions & offsets
tpVector = (TermPositionVector)vectors[i];
storePositions = tpVector.Size() > 0 && tpVector.GetTermPositions(0) != null;
storeOffsets = tpVector.Size() > 0 && tpVector.GetOffsets(0) != null;
bits = (byte)((storePositions?TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR: (byte)0) + (storeOffsets?TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR: (byte)0));
}
else
{
tpVector = null;
bits = 0;
storePositions = false;
storeOffsets = false;
}
tvf.WriteVInt(bits);
System.String[] terms = vectors[i].GetTerms();
int[] freqs = vectors[i].GetTermFrequencies();
int utf8Upto = 0;
utf8Results[1].length = 0;
for (int j = 0; j < numTerms; j++)
{
UnicodeUtil.UTF16toUTF8(terms[j], 0, terms[j].Length, utf8Results[utf8Upto]);
int start = StringHelper.BytesDifference(utf8Results[1 - utf8Upto].result, utf8Results[1 - utf8Upto].length, utf8Results[utf8Upto].result, utf8Results[utf8Upto].length);
int length = utf8Results[utf8Upto].length - start;
tvf.WriteVInt(start); // write shared prefix length
tvf.WriteVInt(length); // write delta length
tvf.WriteBytes(utf8Results[utf8Upto].result, start, length); // write delta bytes
utf8Upto = 1 - utf8Upto;
int termFreq = freqs[j];
tvf.WriteVInt(termFreq);
if (storePositions)
{
int[] positions = tpVector.GetTermPositions(j);
if (positions == null)
{
throw new System.SystemException("Trying to write positions that are null!");
}
System.Diagnostics.Debug.Assert(positions.Length == termFreq);
// use delta encoding for positions
int lastPosition = 0;
for (int k = 0; k < positions.Length; k++)
{
int position = positions[k];
tvf.WriteVInt(position - lastPosition);
lastPosition = position;
}
}
if (storeOffsets)
{
TermVectorOffsetInfo[] offsets = tpVector.GetOffsets(j);
if (offsets == null)
{
throw new System.SystemException("Trying to write offsets that are null!");
}
System.Diagnostics.Debug.Assert(offsets.Length == termFreq);
// use delta encoding for offsets
int lastEndOffset = 0;
for (int k = 0; k < offsets.Length; k++)
{
int startOffset = offsets[k].GetStartOffset();
int endOffset = offsets[k].GetEndOffset();
tvf.WriteVInt(startOffset - lastEndOffset);
tvf.WriteVInt(endOffset - startOffset);
lastEndOffset = endOffset;
}
}
}
}
// 2nd pass: write field pointers to tvd
if (numFields > 1)
{
long lastFieldPointer = fieldPointers[0];
for (int i = 1; i < numFields; i++)
{
long fieldPointer = fieldPointers[i];
tvd.WriteVLong(fieldPointer - lastFieldPointer);
lastFieldPointer = fieldPointer;
}
}
}
else
{
tvd.WriteVInt(0);
}
}
protected internal override System.Object CreateValue(IndexReader reader, Entry entryKey)
{
System.String field = StringHelper.Intern((System.String)entryKey.field);
int[] retArray = new int[reader.MaxDoc()];
System.String[] mterms = new System.String[reader.MaxDoc() + 1];
TermDocs termDocs = reader.TermDocs();
TermEnum termEnum = reader.Terms(new Term(field));
int t = 0; // current term number
// an entry for documents that have no terms in this field
// should a document with no terms be at top or bottom?
// this puts them at the top - if it is changed, FieldDocSortedHitQueue
// needs to change as well.
mterms[t++] = null;
try
{
do
{
Term term = termEnum.Term();
if (term == null || term.Field() != field || t >= mterms.Length)
{
break;
}
// store term text
mterms[t] = term.Text();
termDocs.Seek(termEnum);
while (termDocs.Next())
{
retArray[termDocs.Doc()] = t;
}
t++;
}while (termEnum.Next());
}
finally
{
termDocs.Close();
termEnum.Close();
}
if (t == 0)
{
// if there are no terms, make the term array
// have a single null entry
mterms = new System.String[1];
}
else if (t < mterms.Length)
{
// if there are less terms than documents,
// trim off the dead array space
System.String[] terms = new System.String[t];
Array.Copy(mterms, 0, terms, 0, t);
mterms = terms;
}
StringIndex value_Renamed = new StringIndex(retArray, mterms);
return(value_Renamed);
}
internal Term(System.String fld, System.String txt, bool intern)
{
field = intern?StringHelper.Intern(fld):fld; // field names are interned
text = txt; // unless already known to be
}
internal void OnDeserialized(System.Runtime.Serialization.StreamingContext context)
{
field = StringHelper.Intern(field);
}
/// <summary> Create a field by specifying its name, value and how it will
/// be saved in the index.
///
/// </summary>
/// <param name="name">The name of the field
/// </param>
/// <param name="internName">Whether to .intern() name or not
/// </param>
/// <param name="value">The string to process
/// </param>
/// <param name="store">Whether <code>value</code> should be stored in the index
/// </param>
/// <param name="index">Whether the field should be indexed, and if so, if it should
/// be tokenized before indexing
/// </param>
/// <param name="termVector">Whether term vector should be stored
/// </param>
/// <throws> NullPointerException if name or value is <code>null</code> </throws>
/// <throws> IllegalArgumentException in any of the following situations: </throws>
/// <summary> <ul>
/// <li>the field is neither stored nor indexed</li>
/// <li>the field is not indexed but termVector is <code>TermVector.YES</code></li>
/// </ul>
/// </summary>
public Field(System.String name, bool internName, System.String value_Renamed, Store store, Index index, TermVector termVector)
{
if (name == null)
{
throw new System.NullReferenceException("name cannot be null");
}
if (value_Renamed == null)
{
throw new System.NullReferenceException("value cannot be null");
}
if (name.Length == 0 && value_Renamed.Length == 0)
{
throw new System.ArgumentException("name and value cannot both be empty");
}
if (index == Index.NO && store == Store.NO)
{
throw new System.ArgumentException("it doesn't make sense to have a field that " + "is neither indexed nor stored");
}
if (index == Index.NO && termVector != TermVector.NO)
{
throw new System.ArgumentException("cannot store term vector information " + "for a field that is not indexed");
}
if (internName)
{
// field names are optionally interned
name = StringHelper.Intern(name);
}
this.name = name;
this.fieldsData = value_Renamed;
if (store == Store.YES)
{
this.isStored = true;
this.isCompressed = false;
}
else if (store == Store.COMPRESS)
{
this.isStored = true;
this.isCompressed = true;
}
else if (store == Store.NO)
{
this.isStored = false;
this.isCompressed = false;
}
else
{
throw new System.ArgumentException("unknown store parameter " + store);
}
if (index == Index.NO)
{
this.isIndexed = false;
this.isTokenized = false;
this.omitTermFreqAndPositions = false;
this.omitNorms = true;
}
else if (index == Index.ANALYZED)
{
this.isIndexed = true;
this.isTokenized = true;
}
else if (index == Index.NOT_ANALYZED)
{
this.isIndexed = true;
this.isTokenized = false;
}
else if (index == Index.NOT_ANALYZED_NO_NORMS)
{
this.isIndexed = true;
this.isTokenized = false;
this.omitNorms = true;
}
else if (index == Index.ANALYZED_NO_NORMS)
{
this.isIndexed = true;
this.isTokenized = true;
this.omitNorms = true;
}
else
{
throw new System.ArgumentException("unknown index parameter " + index);
}
this.isBinary = false;
SetStoreTermVector(termVector);
}