public override void AddField(int docID, IndexableField field, FieldInfo fieldInfo) { DocValuesType_e? dvType = field.FieldType().DocValueType; if (dvType != null) { fieldInfo.DocValuesType = dvType; if (dvType == DocValuesType_e.BINARY) { AddBinaryField(fieldInfo, docID, field.BinaryValue()); } else if (dvType == DocValuesType_e.SORTED) { AddSortedField(fieldInfo, docID, field.BinaryValue()); } else if (dvType == DocValuesType_e.SORTED_SET) { AddSortedSetField(fieldInfo, docID, field.BinaryValue()); } else if (dvType == DocValuesType_e.NUMERIC) { if (!(field.NumericValue is long?)) { throw new System.ArgumentException("illegal type " + field.NumericValue.GetType() + ": DocValues types must be Long"); } AddNumericField(fieldInfo, docID, (long)field.NumericValue); } else { Debug.Assert(false, "unrecognized DocValues.Type: " + dvType); } } }
public override void AddField(int docID, IndexableField field, FieldInfo fieldInfo) { DocValuesType_e?dvType = field.FieldType().DocValueType; if (dvType != null) { fieldInfo.DocValuesType = dvType; if (dvType == DocValuesType_e.BINARY) { AddBinaryField(fieldInfo, docID, field.BinaryValue()); } else if (dvType == DocValuesType_e.SORTED) { AddSortedField(fieldInfo, docID, field.BinaryValue()); } else if (dvType == DocValuesType_e.SORTED_SET) { AddSortedSetField(fieldInfo, docID, field.BinaryValue()); } else if (dvType == DocValuesType_e.NUMERIC) { if (!(field.NumericValue is long?)) { throw new System.ArgumentException("illegal type " + field.NumericValue.GetType() + ": DocValues types must be Long"); } AddNumericField(fieldInfo, docID, (long)field.NumericValue); } else { Debug.Assert(false, "unrecognized DocValues.Type: " + dvType); } } }
public static void CheckNorms(AtomicReader reader) { // test omit norms for (int i = 0; i < DocHelper.Fields.Length; i++) { IndexableField f = DocHelper.Fields[i]; if (f.FieldType().Indexed) { Assert.AreEqual(reader.GetNormValues(f.Name()) != null, !f.FieldType().OmitNorms); Assert.AreEqual(reader.GetNormValues(f.Name()) != null, !DocHelper.NoNorms.ContainsKey(f.Name())); if (reader.GetNormValues(f.Name()) == null) { // test for norms of null NumericDocValues norms = MultiDocValues.GetNormValues(reader, f.Name()); Assert.IsNull(norms); } } } }
public override void AddField(int docID, IndexableField field, FieldInfo fieldInfo) { if (field.FieldType().Stored) { if (NumStoredFields == StoredFields.Length) { int newSize = ArrayUtil.Oversize(NumStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF); IndexableField[] newArray = new IndexableField[newSize]; Array.Copy(StoredFields, 0, newArray, 0, NumStoredFields); StoredFields = newArray; FieldInfo[] newInfoArray = new FieldInfo[newSize]; Array.Copy(FieldInfos, 0, newInfoArray, 0, NumStoredFields); FieldInfos = newInfoArray; } StoredFields[NumStoredFields] = field; FieldInfos[NumStoredFields] = fieldInfo; NumStoredFields++; Debug.Assert(DocState.TestPoint("StoredFieldsWriterPerThread.processFields.writeField")); } }
static DocHelper() { CustomType = new FieldType(TextField.TYPE_STORED); TextField1 = new Field(TEXT_FIELD_1_KEY, FIELD_1_TEXT, CustomType); CustomType2 = new FieldType(TextField.TYPE_STORED); CustomType2.StoreTermVectors = true; CustomType2.StoreTermVectorPositions = true; CustomType2.StoreTermVectorOffsets = true; TextField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, CustomType2); CustomType3 = new FieldType(TextField.TYPE_STORED); CustomType3.OmitNorms = true; TextField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, CustomType3); KeyField = new StringField(KEYWORD_FIELD_KEY, KEYWORD_TEXT, Field.Store.YES); CustomType5 = new FieldType(TextField.TYPE_STORED); CustomType5.OmitNorms = true; CustomType5.Tokenized = false; NoNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT, CustomType5); CustomType6 = new FieldType(TextField.TYPE_STORED); CustomType6.IndexOptions = FieldInfo.IndexOptions.DOCS_ONLY; NoTFField = new Field(NO_TF_KEY, NO_TF_TEXT, CustomType6); CustomType7 = new FieldType(); CustomType7.Stored = true; UnIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT, CustomType7); CustomType8 = new FieldType(TextField.TYPE_NOT_STORED); CustomType8.StoreTermVectors = true; UnStoredField2 = new Field(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT, CustomType8); UnStoredField1 = new TextField(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT, Field.Store.NO); LazyField = new Field(LAZY_FIELD_KEY, LAZY_FIELD_TEXT, CustomType); TextUtfField1 = new Field(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT, CustomType); TextUtfField2 = new Field(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT, CustomType2); Fields = new Field[] { TextField1, TextField2, TextField3, KeyField, NoNormsField, NoTFField, UnIndField, UnStoredField1, UnStoredField2, TextUtfField1, TextUtfField2, LazyField, LazyFieldBinary, LargeLazyField }; //Initialize the large Lazy Field StringBuilder buffer = new StringBuilder(); for (int i = 0; i < 10000; i++) { buffer.Append("Lazily loading lengths of language in lieu of laughing "); } try { LAZY_FIELD_BINARY_BYTES = "These are some binary field bytes".GetBytes(IOUtils.CHARSET_UTF_8); } catch (EncoderFallbackException e) { } LazyFieldBinary = new StoredField(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES); Fields[Fields.Length - 2] = LazyFieldBinary; LARGE_LAZY_FIELD_TEXT = buffer.ToString(); LargeLazyField = new Field(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT, CustomType); Fields[Fields.Length - 1] = LargeLazyField; for (int i = 0; i < Fields.Length; i++) { IndexableField f = Fields[i]; Add(All, f); if (f.FieldType().Indexed) { Add(Indexed, f); } else { Add(Unindexed, f); } if (f.FieldType().StoreTermVectors) { Add(Termvector, f); } if (f.FieldType().Indexed&& !f.FieldType().StoreTermVectors) { Add(Notermvector, f); } if (f.FieldType().Stored) { Add(Stored, f); } else { Add(Unstored, f); } if (f.FieldType().IndexOptions == FieldInfo.IndexOptions.DOCS_ONLY) { Add(NoTf, f); } if (f.FieldType().OmitNorms) { Add(NoNorms, f); } if (f.FieldType().IndexOptions == FieldInfo.IndexOptions.DOCS_ONLY) { Add(NoTf, f); } //if (f.isLazy()) add(lazy, f); } NameValues = new Dictionary <string, object>(); NameValues[TEXT_FIELD_1_KEY] = FIELD_1_TEXT; NameValues[TEXT_FIELD_2_KEY] = FIELD_2_TEXT; NameValues[TEXT_FIELD_3_KEY] = FIELD_3_TEXT; NameValues[KEYWORD_FIELD_KEY] = KEYWORD_TEXT; NameValues[NO_NORMS_KEY] = NO_NORMS_TEXT; NameValues[NO_TF_KEY] = NO_TF_TEXT; NameValues[UNINDEXED_FIELD_KEY] = UNINDEXED_FIELD_TEXT; NameValues[UNSTORED_FIELD_1_KEY] = UNSTORED_1_FIELD_TEXT; NameValues[UNSTORED_FIELD_2_KEY] = UNSTORED_2_FIELD_TEXT; NameValues[LAZY_FIELD_KEY] = LAZY_FIELD_TEXT; NameValues[LAZY_FIELD_BINARY_KEY] = LAZY_FIELD_BINARY_BYTES; NameValues[LARGE_LAZY_FIELD_KEY] = LARGE_LAZY_FIELD_TEXT; NameValues[TEXT_FIELD_UTF1_KEY] = FIELD_UTF1_TEXT; NameValues[TEXT_FIELD_UTF2_KEY] = FIELD_UTF2_TEXT; }
public override void ProcessFields(IndexableField[] fields, int count) { FieldState.Reset(); bool doInvert = Consumer.Start(fields, count); for (int i = 0; i < count; i++) { IndexableField field = fields[i]; IndexableFieldType fieldType = field.FieldType(); // TODO FI: this should be "genericized" to querying // consumer if it wants to see this particular field // tokenized. if (fieldType.Indexed && doInvert) { bool analyzed = fieldType.Tokenized && DocState.Analyzer != null; // if the field omits norms, the boost cannot be indexed. if (fieldType.OmitNorms && field.GetBoost() != 1.0f) { throw new System.NotSupportedException("You cannot set an index-time boost: norms are omitted for field '" + field.Name() + "'"); } // only bother checking offsets if something will consume them. // TODO: after we fix analyzers, also check if termVectorOffsets will be indexed. bool checkOffsets = fieldType.IndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; int lastStartOffset = 0; if (i > 0) { FieldState.Position_Renamed += analyzed ? DocState.Analyzer.GetPositionIncrementGap(fieldInfo.Name) : 0; } /* * To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream * when we fail. We expect some caller to eventually deal with the real exception, so we don't want any 'catch' clauses, * but rather a finally that takes note of the problem. */ bool succeededInProcessingField = false; TokenStream stream = field.GetTokenStream(DocState.Analyzer); // reset the TokenStream to the first token stream.Reset(); try { bool hasMoreTokens = stream.IncrementToken(); FieldState.AttributeSource_Renamed = stream; IOffsetAttribute offsetAttribute = FieldState.AttributeSource_Renamed.AddAttribute <IOffsetAttribute>(); IPositionIncrementAttribute posIncrAttribute = FieldState.AttributeSource_Renamed.AddAttribute <IPositionIncrementAttribute>(); if (hasMoreTokens) { Consumer.Start(field); do { // If we hit an exception in stream.next below // (which is fairly common, eg if analyzer // chokes on a given document), then it's // non-aborting and (above) this one document // will be marked as deleted, but still // consume a docID int posIncr = posIncrAttribute.PositionIncrement; if (posIncr < 0) { throw new System.ArgumentException("position increment must be >=0 (got " + posIncr + ") for field '" + field.Name() + "'"); } if (FieldState.Position_Renamed == 0 && posIncr == 0) { throw new System.ArgumentException("first position increment must be > 0 (got 0) for field '" + field.Name() + "'"); } int position = FieldState.Position_Renamed + posIncr; if (position > 0) { // NOTE: confusing: this "mirrors" the // position++ we do below position--; } else if (position < 0) { throw new System.ArgumentException("position overflow for field '" + field.Name() + "'"); } // position is legal, we can safely place it in fieldState now. // not sure if anything will use fieldState after non-aborting exc... FieldState.Position_Renamed = position; if (posIncr == 0) { FieldState.NumOverlap_Renamed++; } if (checkOffsets) { int startOffset = FieldState.Offset_Renamed + offsetAttribute.StartOffset(); int endOffset = FieldState.Offset_Renamed + offsetAttribute.EndOffset(); if (startOffset < 0 || endOffset < startOffset) { throw new System.ArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, " + "startOffset=" + startOffset + ",endOffset=" + endOffset + " for field '" + field.Name() + "'"); } if (startOffset < lastStartOffset) { throw new System.ArgumentException("offsets must not go backwards startOffset=" + startOffset + " is < lastStartOffset=" + lastStartOffset + " for field '" + field.Name() + "'"); } lastStartOffset = startOffset; } bool success = false; try { // If we hit an exception in here, we abort // all buffered documents since the last // flush, on the likelihood that the // internal state of the consumer is now // corrupt and should not be flushed to a // new segment: Consumer.Add(); success = true; } finally { if (!success) { DocState.DocWriter.SetAborting(); } } FieldState.Length_Renamed++; FieldState.Position_Renamed++; } while (stream.IncrementToken()); } // trigger streams to perform end-of-stream operations stream.End(); // TODO: maybe add some safety? then again, its already checked // when we come back around to the field... FieldState.Position_Renamed += posIncrAttribute.PositionIncrement; FieldState.Offset_Renamed += offsetAttribute.EndOffset(); if (DocState.MaxTermPrefix != null) { string msg = "Document contains at least one immense term in field=\"" + fieldInfo.Name + "\" (whose UTF8 encoding is longer than the max length " + DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + DocState.MaxTermPrefix + "...'"; if (DocState.InfoStream.IsEnabled("IW")) { DocState.InfoStream.Message("IW", "ERROR: " + msg); } DocState.MaxTermPrefix = null; throw new System.ArgumentException(msg); } /* if success was false above there is an exception coming through and we won't get here.*/ succeededInProcessingField = true; } finally { if (!succeededInProcessingField) { IOUtils.CloseWhileHandlingException(stream); } else { stream.Dispose(); } if (!succeededInProcessingField && DocState.InfoStream.IsEnabled("DW")) { DocState.InfoStream.Message("DW", "An exception was thrown while processing field " + fieldInfo.Name); } } FieldState.Offset_Renamed += analyzed ? DocState.Analyzer.GetOffsetGap(fieldInfo.Name) : 0; FieldState.Boost_Renamed *= field.GetBoost(); } // LUCENE-2387: don't hang onto the field, so GC can // reclaim fields[i] = null; } Consumer.Finish(); EndConsumer.Finish(); }
public override void AddField(int docID, IndexableField field, FieldInfo fieldInfo) { if (field.FieldType().Stored) { if (NumStoredFields == StoredFields.Length) { int newSize = ArrayUtil.Oversize(NumStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF); IndexableField[] newArray = new IndexableField[newSize]; Array.Copy(StoredFields, 0, newArray, 0, NumStoredFields); StoredFields = newArray; FieldInfo[] newInfoArray = new FieldInfo[newSize]; Array.Copy(FieldInfos, 0, newInfoArray, 0, NumStoredFields); FieldInfos = newInfoArray; } StoredFields[NumStoredFields] = field; FieldInfos[NumStoredFields] = fieldInfo; NumStoredFields++; Debug.Assert(DocState.TestPoint("StoredFieldsWriterPerThread.processFields.writeField")); } }
internal override bool Start(IndexableField[] fields, int count) { DoVectors = false; DoVectorPositions = false; DoVectorOffsets = false; DoVectorPayloads = false; HasPayloads = false; for (int i = 0; i < count; i++) { IndexableField field = fields[i]; if (field.FieldType().Indexed) { if (field.FieldType().StoreTermVectors) { DoVectors = true; DoVectorPositions |= field.FieldType().StoreTermVectorPositions; DoVectorOffsets |= field.FieldType().StoreTermVectorOffsets; if (DoVectorPositions) { DoVectorPayloads |= field.FieldType().StoreTermVectorPayloads; } else if (field.FieldType().StoreTermVectorPayloads) { // TODO: move this check somewhere else, and impl the other missing ones throw new System.ArgumentException("cannot index term vector payloads without term vector positions (field=\"" + field.Name() + "\")"); } } else { if (field.FieldType().StoreTermVectorOffsets) { throw new System.ArgumentException("cannot index term vector offsets when term vectors are not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPositions) { throw new System.ArgumentException("cannot index term vector positions when term vectors are not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPayloads) { throw new System.ArgumentException("cannot index term vector payloads when term vectors are not indexed (field=\"" + field.Name() + "\")"); } } } else { if (field.FieldType().StoreTermVectors) { throw new System.ArgumentException("cannot index term vectors when field is not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorOffsets) { throw new System.ArgumentException("cannot index term vector offsets when field is not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPositions) { throw new System.ArgumentException("cannot index term vector positions when field is not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPayloads) { throw new System.ArgumentException("cannot index term vector payloads when field is not indexed (field=\"" + field.Name() + "\")"); } } } if (DoVectors) { TermsWriter.HasVectors = true; if (TermsHashPerField.BytesHash.Size() != 0) { // Only necessary if previous doc hit a // non-aborting exception while writing vectors in // this field: TermsHashPerField.Reset(); } } // TODO: only if needed for performance //perThread.postingsCount = 0; return(DoVectors); }