public static void CheckNorms(AtomicReader reader) { // test omit norms for (int i = 0; i < DocHelper.Fields.Length; i++) { IndexableField f = DocHelper.Fields[i]; if (f.FieldType().Indexed) { Assert.AreEqual(reader.GetNormValues(f.Name()) != null, !f.FieldType().OmitNorms); Assert.AreEqual(reader.GetNormValues(f.Name()) != null, !DocHelper.NoNorms.ContainsKey(f.Name())); if (reader.GetNormValues(f.Name()) == null) { // test for norms of null NumericDocValues norms = MultiDocValues.GetNormValues(reader, f.Name()); Assert.IsNull(norms); } } } }
private static void Add(IDictionary <string, IndexableField> map, IndexableField field) { map[field.Name()] = field; }
public override void WriteField(FieldInfo info, IndexableField field) { int bits = 0; BytesRef bytes; string @string; object number = (object)field.NumericValue; if (number != null) { if (number is string) { string numStr = number.ToString(); sbyte dummySbyte; short dummyShort; int dummyInt; long dummyLong; float dummyFloat; double dummyDouble; if (sbyte.TryParse(numStr, out dummySbyte) || short.TryParse(numStr, out dummyShort) || int.TryParse(numStr, out dummyInt)) { bits = NUMERIC_INT; } else if (long.TryParse(numStr, out dummyLong)) { bits = NUMERIC_LONG; } else if (float.TryParse(numStr, out dummyFloat)) { bits = NUMERIC_FLOAT; } else if (double.TryParse(numStr, out dummyDouble)) { bits = NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } } else { if (number is sbyte || number is short || number is int) { bits = NUMERIC_INT; } else if (number is long) { bits = NUMERIC_LONG; } else if (number is float) { bits = NUMERIC_FLOAT; } else if (number is double) { bits = NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } } @string = null; bytes = null; } else { bytes = field.BinaryValue(); if (bytes != null) { bits = BYTE_ARR; @string = null; } else { bits = STRING; @string = field.StringValue; if (@string == null) { throw new System.ArgumentException("field " + field.Name() + " is stored but does not have binaryValue, stringValue nor numericValue"); } } } long infoAndBits = (((long)info.Number) << TYPE_BITS) | bits; BufferedDocs.WriteVLong(infoAndBits); if (bytes != null) { BufferedDocs.WriteVInt(bytes.Length); BufferedDocs.WriteBytes(bytes.Bytes, bytes.Offset, bytes.Length); } else if (@string != null) { BufferedDocs.WriteString(field.StringValue); } else { if (number is string) { string numStr = number.ToString(); sbyte dummySbyte; short dummyShort; int dummyInt; long dummyLong; float dummyFloat; double dummyDouble; if (sbyte.TryParse(numStr, out dummySbyte) || short.TryParse(numStr, out dummyShort) || int.TryParse(numStr, out dummyInt)) { bits = NUMERIC_INT; } else if (long.TryParse(numStr, out dummyLong)) { bits = NUMERIC_LONG; } else if (float.TryParse(numStr, out dummyFloat)) { bits = NUMERIC_FLOAT; } else if (double.TryParse(numStr, out dummyDouble)) { bits = NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } } else { if (number is sbyte || number is short || number is int) { BufferedDocs.WriteInt((int)number); } else if (number is long) { BufferedDocs.WriteLong((long)number); } else if (number is float) { BufferedDocs.WriteInt(Number.FloatToIntBits((float)number)); } else if (number is double) { BufferedDocs.WriteLong(BitConverter.DoubleToInt64Bits((double)number)); } else { throw new Exception("Cannot get here"); } } } }
/// <summary> /// checks that two stored fields are equivalent /// </summary> public void AssertStoredFieldEquals(string info, IndexableField leftField, IndexableField rightField) { Assert.AreEqual(leftField.Name(), rightField.Name(), info); Assert.AreEqual(leftField.BinaryValue(), rightField.BinaryValue(), info); Assert.AreEqual(leftField.StringValue, rightField.StringValue, info); Assert.AreEqual(leftField.NumericValue, rightField.NumericValue, info); // TODO: should we check the FT at all? }
public override void WriteField(FieldInfo info, IndexableField field) { FieldsStream.WriteVInt(info.Number); int bits = 0; BytesRef bytes; string @string; // TODO: maybe a field should serialize itself? // this way we don't bake into indexer all these // specific encodings for different fields? and apps // can customize... object number = (object)field.NumericValue; if (number != null) { if (number is sbyte || number is short || number is int) { bits |= FIELD_IS_NUMERIC_INT; } else if (number is long) { bits |= FIELD_IS_NUMERIC_LONG; } else if (number is float) { bits |= FIELD_IS_NUMERIC_FLOAT; } else if (number is double) { bits |= FIELD_IS_NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } @string = null; bytes = null; } else { bytes = field.BinaryValue(); if (bytes != null) { bits |= FIELD_IS_BINARY; @string = null; } else { @string = field.StringValue; if (@string == null) { throw new System.ArgumentException("field " + field.Name() + " is stored but does not have binaryValue, stringValue nor numericValue"); } } } FieldsStream.WriteByte((byte)(sbyte)bits); if (bytes != null) { FieldsStream.WriteVInt(bytes.Length); FieldsStream.WriteBytes(bytes.Bytes, bytes.Offset, bytes.Length); } else if (@string != null) { FieldsStream.WriteString(field.StringValue); } else { if (number is sbyte || number is short || number is int) { FieldsStream.WriteInt((int)number); } else if (number is long) { FieldsStream.WriteLong((long)number); } else if (number is float) { FieldsStream.WriteInt(Number.FloatToIntBits((float)number)); } else if (number is double) { FieldsStream.WriteLong(BitConverter.DoubleToInt64Bits((double)number)); } else { throw new InvalidOperationException("Cannot get here"); } } }
public override void WriteField(FieldInfo info, IndexableField field) { int bits = 0; BytesRef bytes; string @string; object number = (object)field.NumericValue; if (number != null) { if (number is string) { string numStr = number.ToString(); sbyte dummySbyte; short dummyShort; int dummyInt; long dummyLong; float dummyFloat; double dummyDouble; if (sbyte.TryParse(numStr, out dummySbyte) || short.TryParse(numStr, out dummyShort) || int.TryParse(numStr, out dummyInt)) { bits = NUMERIC_INT; } else if (long.TryParse(numStr, out dummyLong)) { bits = NUMERIC_LONG; } else if (float.TryParse(numStr, out dummyFloat)) { bits = NUMERIC_FLOAT; } else if (double.TryParse(numStr, out dummyDouble)) { bits = NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } } else { if (number is sbyte || number is short || number is int) { bits = NUMERIC_INT; } else if (number is long) { bits = NUMERIC_LONG; } else if (number is float) { bits = NUMERIC_FLOAT; } else if (number is double) { bits = NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } } @string = null; bytes = null; } else { bytes = field.BinaryValue(); if (bytes != null) { bits = BYTE_ARR; @string = null; } else { bits = STRING; @string = field.StringValue; if (@string == null) { throw new System.ArgumentException("field " + field.Name() + " is stored but does not have binaryValue, stringValue nor numericValue"); } } } long infoAndBits = (((long)info.Number) << TYPE_BITS) | bits; BufferedDocs.WriteVLong(infoAndBits); if (bytes != null) { BufferedDocs.WriteVInt(bytes.Length); BufferedDocs.WriteBytes(bytes.Bytes, bytes.Offset, bytes.Length); } else if (@string != null) { BufferedDocs.WriteString(field.StringValue); } else { if (number is string) { string numStr = number.ToString(); sbyte dummySbyte; short dummyShort; int dummyInt; long dummyLong; float dummyFloat; double dummyDouble; if (sbyte.TryParse(numStr, out dummySbyte) || short.TryParse(numStr, out dummyShort) || int.TryParse(numStr, out dummyInt)) { bits = NUMERIC_INT; } else if (long.TryParse(numStr, out dummyLong)) { bits = NUMERIC_LONG; } else if (float.TryParse(numStr, out dummyFloat)) { bits = NUMERIC_FLOAT; } else if (double.TryParse(numStr, out dummyDouble)) { bits = NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } } else { if (number is sbyte || number is short || number is int) { BufferedDocs.WriteInt((int)number); } else if (number is long) { BufferedDocs.WriteLong((long)number); } else if (number is float) { BufferedDocs.WriteInt(Number.FloatToIntBits((float)number)); } else if (number is double) { BufferedDocs.WriteLong(BitConverter.DoubleToInt64Bits((double)number)); } else { throw new Exception("Cannot get here"); } } } }
private static void Add(IDictionary<string, IndexableField> map, IndexableField field) { map[field.Name()] = field; }
public override void ProcessFields(IndexableField[] fields, int count) { FieldState.Reset(); bool doInvert = Consumer.Start(fields, count); for (int i = 0; i < count; i++) { IndexableField field = fields[i]; IndexableFieldType fieldType = field.FieldType(); // TODO FI: this should be "genericized" to querying // consumer if it wants to see this particular field // tokenized. if (fieldType.Indexed && doInvert) { bool analyzed = fieldType.Tokenized && DocState.Analyzer != null; // if the field omits norms, the boost cannot be indexed. if (fieldType.OmitNorms && field.GetBoost() != 1.0f) { throw new System.NotSupportedException("You cannot set an index-time boost: norms are omitted for field '" + field.Name() + "'"); } // only bother checking offsets if something will consume them. // TODO: after we fix analyzers, also check if termVectorOffsets will be indexed. bool checkOffsets = fieldType.IndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; int lastStartOffset = 0; if (i > 0) { FieldState.Position_Renamed += analyzed ? DocState.Analyzer.GetPositionIncrementGap(fieldInfo.Name) : 0; } /* * To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream * when we fail. We expect some caller to eventually deal with the real exception, so we don't want any 'catch' clauses, * but rather a finally that takes note of the problem. */ bool succeededInProcessingField = false; TokenStream stream = field.GetTokenStream(DocState.Analyzer); // reset the TokenStream to the first token stream.Reset(); try { bool hasMoreTokens = stream.IncrementToken(); FieldState.AttributeSource_Renamed = stream; IOffsetAttribute offsetAttribute = FieldState.AttributeSource_Renamed.AddAttribute <IOffsetAttribute>(); IPositionIncrementAttribute posIncrAttribute = FieldState.AttributeSource_Renamed.AddAttribute <IPositionIncrementAttribute>(); if (hasMoreTokens) { Consumer.Start(field); do { // If we hit an exception in stream.next below // (which is fairly common, eg if analyzer // chokes on a given document), then it's // non-aborting and (above) this one document // will be marked as deleted, but still // consume a docID int posIncr = posIncrAttribute.PositionIncrement; if (posIncr < 0) { throw new System.ArgumentException("position increment must be >=0 (got " + posIncr + ") for field '" + field.Name() + "'"); } if (FieldState.Position_Renamed == 0 && posIncr == 0) { throw new System.ArgumentException("first position increment must be > 0 (got 0) for field '" + field.Name() + "'"); } int position = FieldState.Position_Renamed + posIncr; if (position > 0) { // NOTE: confusing: this "mirrors" the // position++ we do below position--; } else if (position < 0) { throw new System.ArgumentException("position overflow for field '" + field.Name() + "'"); } // position is legal, we can safely place it in fieldState now. // not sure if anything will use fieldState after non-aborting exc... FieldState.Position_Renamed = position; if (posIncr == 0) { FieldState.NumOverlap_Renamed++; } if (checkOffsets) { int startOffset = FieldState.Offset_Renamed + offsetAttribute.StartOffset(); int endOffset = FieldState.Offset_Renamed + offsetAttribute.EndOffset(); if (startOffset < 0 || endOffset < startOffset) { throw new System.ArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, " + "startOffset=" + startOffset + ",endOffset=" + endOffset + " for field '" + field.Name() + "'"); } if (startOffset < lastStartOffset) { throw new System.ArgumentException("offsets must not go backwards startOffset=" + startOffset + " is < lastStartOffset=" + lastStartOffset + " for field '" + field.Name() + "'"); } lastStartOffset = startOffset; } bool success = false; try { // If we hit an exception in here, we abort // all buffered documents since the last // flush, on the likelihood that the // internal state of the consumer is now // corrupt and should not be flushed to a // new segment: Consumer.Add(); success = true; } finally { if (!success) { DocState.DocWriter.SetAborting(); } } FieldState.Length_Renamed++; FieldState.Position_Renamed++; } while (stream.IncrementToken()); } // trigger streams to perform end-of-stream operations stream.End(); // TODO: maybe add some safety? then again, its already checked // when we come back around to the field... FieldState.Position_Renamed += posIncrAttribute.PositionIncrement; FieldState.Offset_Renamed += offsetAttribute.EndOffset(); if (DocState.MaxTermPrefix != null) { string msg = "Document contains at least one immense term in field=\"" + fieldInfo.Name + "\" (whose UTF8 encoding is longer than the max length " + DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + DocState.MaxTermPrefix + "...'"; if (DocState.InfoStream.IsEnabled("IW")) { DocState.InfoStream.Message("IW", "ERROR: " + msg); } DocState.MaxTermPrefix = null; throw new System.ArgumentException(msg); } /* if success was false above there is an exception coming through and we won't get here.*/ succeededInProcessingField = true; } finally { if (!succeededInProcessingField) { IOUtils.CloseWhileHandlingException(stream); } else { stream.Dispose(); } if (!succeededInProcessingField && DocState.InfoStream.IsEnabled("DW")) { DocState.InfoStream.Message("DW", "An exception was thrown while processing field " + fieldInfo.Name); } } FieldState.Offset_Renamed += analyzed ? DocState.Analyzer.GetOffsetGap(fieldInfo.Name) : 0; FieldState.Boost_Renamed *= field.GetBoost(); } // LUCENE-2387: don't hang onto the field, so GC can // reclaim fields[i] = null; } Consumer.Finish(); EndConsumer.Finish(); }
internal override bool Start(IndexableField[] fields, int count) { DoVectors = false; DoVectorPositions = false; DoVectorOffsets = false; DoVectorPayloads = false; HasPayloads = false; for (int i = 0; i < count; i++) { IndexableField field = fields[i]; if (field.FieldType().Indexed) { if (field.FieldType().StoreTermVectors) { DoVectors = true; DoVectorPositions |= field.FieldType().StoreTermVectorPositions; DoVectorOffsets |= field.FieldType().StoreTermVectorOffsets; if (DoVectorPositions) { DoVectorPayloads |= field.FieldType().StoreTermVectorPayloads; } else if (field.FieldType().StoreTermVectorPayloads) { // TODO: move this check somewhere else, and impl the other missing ones throw new System.ArgumentException("cannot index term vector payloads without term vector positions (field=\"" + field.Name() + "\")"); } } else { if (field.FieldType().StoreTermVectorOffsets) { throw new System.ArgumentException("cannot index term vector offsets when term vectors are not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPositions) { throw new System.ArgumentException("cannot index term vector positions when term vectors are not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPayloads) { throw new System.ArgumentException("cannot index term vector payloads when term vectors are not indexed (field=\"" + field.Name() + "\")"); } } } else { if (field.FieldType().StoreTermVectors) { throw new System.ArgumentException("cannot index term vectors when field is not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorOffsets) { throw new System.ArgumentException("cannot index term vector offsets when field is not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPositions) { throw new System.ArgumentException("cannot index term vector positions when field is not indexed (field=\"" + field.Name() + "\")"); } if (field.FieldType().StoreTermVectorPayloads) { throw new System.ArgumentException("cannot index term vector payloads when field is not indexed (field=\"" + field.Name() + "\")"); } } } if (DoVectors) { TermsWriter.HasVectors = true; if (TermsHashPerField.BytesHash.Size() != 0) { // Only necessary if previous doc hit a // non-aborting exception while writing vectors in // this field: TermsHashPerField.Reset(); } } // TODO: only if needed for performance //perThread.postingsCount = 0; return(DoVectors); }
public override void WriteField(FieldInfo info, IndexableField field) { FieldsStream.WriteVInt(info.Number); int bits = 0; BytesRef bytes; string @string; // TODO: maybe a field should serialize itself? // this way we don't bake into indexer all these // specific encodings for different fields? and apps // can customize... object number = (object)field.NumericValue; if (number != null) { if (number is sbyte || number is short || number is int) { bits |= FIELD_IS_NUMERIC_INT; } else if (number is long) { bits |= FIELD_IS_NUMERIC_LONG; } else if (number is float) { bits |= FIELD_IS_NUMERIC_FLOAT; } else if (number is double) { bits |= FIELD_IS_NUMERIC_DOUBLE; } else { throw new System.ArgumentException("cannot store numeric type " + number.GetType()); } @string = null; bytes = null; } else { bytes = field.BinaryValue(); if (bytes != null) { bits |= FIELD_IS_BINARY; @string = null; } else { @string = field.StringValue; if (@string == null) { throw new System.ArgumentException("field " + field.Name() + " is stored but does not have binaryValue, stringValue nor numericValue"); } } } FieldsStream.WriteByte((sbyte)bits); if (bytes != null) { FieldsStream.WriteVInt(bytes.Length); FieldsStream.WriteBytes(bytes.Bytes, bytes.Offset, bytes.Length); } else if (@string != null) { FieldsStream.WriteString(field.StringValue); } else { if (number is sbyte || number is short || number is int) { FieldsStream.WriteInt((int)number); } else if (number is long) { FieldsStream.WriteLong((long)number); } else if (number is float) { FieldsStream.WriteInt(Number.FloatToIntBits((float)number)); } else if (number is double) { FieldsStream.WriteLong(BitConverter.DoubleToInt64Bits((double)number)); } else { throw new InvalidOperationException("Cannot get here"); } } }