private object ConvertType(Lucene.Net.Documents.Document indexDocument, IFieldable field, FieldType fieldType) { if (field.IsBinary) { throw new NotImplementedException("Support for binary values"); } var stringValue = field.StringValue; if (stringValue == Constants.NullValue || stringValue == null) { return(null); } if (stringValue == Constants.EmptyString || stringValue == string.Empty) { return(string.Empty); } if (fieldType.IsJson == false) { return(stringValue); } var bytes = _context.Encoding.GetBytes(stringValue); var ms = new MemoryStream(bytes); return(_context.ReadForMemory(ms, field.Name)); }
public virtual void TestLazyFieldsAfterClose() { Assert.IsTrue(dir != null); Assert.IsTrue(fieldInfos != null); FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null); Assert.IsTrue(reader != null); Assert.IsTrue(reader.Size() == 1); ISet <string> loadFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>(); loadFieldNames.Add(DocHelper.TEXT_FIELD_1_KEY); loadFieldNames.Add(DocHelper.TEXT_FIELD_UTF1_KEY); ISet <string> lazyFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>(); lazyFieldNames.Add(DocHelper.LARGE_LAZY_FIELD_KEY); lazyFieldNames.Add(DocHelper.LAZY_FIELD_KEY); lazyFieldNames.Add(DocHelper.LAZY_FIELD_BINARY_KEY); lazyFieldNames.Add(DocHelper.TEXT_FIELD_UTF2_KEY); SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames); Document doc = reader.Doc(0, fieldSelector, null); Assert.IsTrue(doc != null, "doc is null and it shouldn't be"); IFieldable field = doc.GetFieldable(DocHelper.LAZY_FIELD_KEY); Assert.IsTrue(field != null, "field is null and it shouldn't be"); Assert.IsTrue(field.IsLazy, "field is not lazy and it should be"); reader.Dispose(); Assert.Throws <AlreadyClosedException>(() => { var value = field.StringValue(null); }, "did not hit AlreadyClosedException as expected"); }
private static object ConvertType(JsonOperationContext context, IFieldable field, FieldType fieldType, IState state) { if (field.IsBinary) { ThrowBinaryValuesNotSupported(); } var stringValue = field.StringValue(state); if (stringValue == Constants.Documents.Indexing.Fields.NullValue || stringValue == null) { return(null); } if (stringValue == Constants.Documents.Indexing.Fields.EmptyString || stringValue == string.Empty) { return(string.Empty); } if (fieldType.IsJson == false) { return(stringValue); } var bytes = Encodings.Utf8.GetBytes(stringValue); var ms = new MemoryStream(bytes); return(context.ReadForMemory(ms, field.Name)); }
internal void WriteField(FieldInfo fi, IFieldable field, IState state) { fieldsStream.WriteVInt(fi.number); byte bits = 0; if (field.IsTokenized) { bits |= FieldsWriter.FIELD_IS_TOKENIZED; } if (field.IsBinary) { bits |= FieldsWriter.FIELD_IS_BINARY; } fieldsStream.WriteByte(bits); // compression is disabled for the current field if (field.IsBinary) { byte[] data = field.GetBinaryValue(state); int len = field.BinaryLength; int offset = field.BinaryOffset; fieldsStream.WriteVInt(len); fieldsStream.WriteBytes(data, offset, len); } else { fieldsStream.WriteString(field.StringValue(state)); } }
public static void CheckNorms(IndexReader reader) { // test omit norms for (int i = 0; i < DocHelper.fields.Length; i++) { IFieldable f = DocHelper.fields[i]; if (f.IsIndexed) { Assert.AreEqual(reader.HasNorms(f.Name, null), !f.OmitNorms); Assert.AreEqual(reader.HasNorms(f.Name, null), !DocHelper.noNorms.Contains(f.Name)); if (!reader.HasNorms(f.Name, null)) { // test for fake norms of 1.0 or null depending on the flag byte[] norms = reader.Norms(f.Name, null); byte norm1 = DefaultSimilarity.EncodeNorm(1.0f); Assert.IsNull(norms); norms = new byte[reader.MaxDoc]; reader.Norms(f.Name, norms, 0, null); for (int j = 0; j < reader.MaxDoc; j++) { Assert.AreEqual(norms[j], norm1); } } } } }
private static FieldType GetFieldType(IFieldable field, Lucene.Net.Documents.Document indexDocument) { return(new FieldType { IsArray = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.IsArrayFieldSuffix) != null, IsJson = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.ConvertToJsonSuffix) != null, }); }
internal virtual void ValidateField(IFieldable f) { System.String val = f.StringValue; if (!val.StartsWith("^") || !val.EndsWith("$")) { throw new System.SystemException("Invalid field:" + f.ToString() + " val=" + val); } }
/// <summary> /// Allows method chaining of add field /// </summary> /// <param name="document">Document to add field to</param> /// <param name="field">Field to add</param> /// <returns>Document</returns> public static Document AddField(this Document document, IFieldable field) { if (document != null) { document.Add(field); } return(document); }
internal override void Start(IFieldable f) { termAtt = fieldState.attributeSource.AddAttribute <ITermAttribute>(); consumer.Start(f); if (nextPerField != null) { nextPerField.Start(f); } }
protected internal virtual object ConvertFieldValue(IFieldable field) { var fieldValue = (object)field.StringValue; if (converter != null) { fieldValue = converter.ConvertFrom(fieldValue); } return(fieldValue); }
public virtual void TestLazyFields() { Assert.IsTrue(dir != null); Assert.IsTrue(fieldInfos != null); FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null); Assert.IsTrue(reader != null); Assert.IsTrue(reader.Size() == 1); ISet <string> loadFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>(); loadFieldNames.Add(DocHelper.TEXT_FIELD_1_KEY); loadFieldNames.Add(DocHelper.TEXT_FIELD_UTF1_KEY); ISet <string> lazyFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>(); //new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY}; lazyFieldNames.Add(DocHelper.LARGE_LAZY_FIELD_KEY); lazyFieldNames.Add(DocHelper.LAZY_FIELD_KEY); lazyFieldNames.Add(DocHelper.LAZY_FIELD_BINARY_KEY); lazyFieldNames.Add(DocHelper.TEXT_FIELD_UTF2_KEY); SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames); Document doc = reader.Doc(0, fieldSelector, null); Assert.IsTrue(doc != null, "doc is null and it shouldn't be"); IFieldable field = doc.GetFieldable(DocHelper.LAZY_FIELD_KEY); Assert.IsTrue(field != null, "field is null and it shouldn't be"); Assert.IsTrue(field.IsLazy, "field is not lazy and it should be"); System.String value_Renamed = field.StringValue(null); Assert.IsTrue(value_Renamed != null, "value is null and it shouldn't be"); Assert.IsTrue(value_Renamed.Equals(DocHelper.LAZY_FIELD_TEXT) == true, value_Renamed + " is not equal to " + DocHelper.LAZY_FIELD_TEXT); field = doc.GetFieldable(DocHelper.TEXT_FIELD_1_KEY); Assert.IsTrue(field != null, "field is null and it shouldn't be"); Assert.IsTrue(field.IsLazy == false, "Field is lazy and it should not be"); field = doc.GetFieldable(DocHelper.TEXT_FIELD_UTF1_KEY); Assert.IsTrue(field != null, "field is null and it shouldn't be"); Assert.IsTrue(field.IsLazy == false, "Field is lazy and it should not be"); Assert.IsTrue(field.StringValue(null).Equals(DocHelper.FIELD_UTF1_TEXT) == true, field.StringValue(null) + " is not equal to " + DocHelper.FIELD_UTF1_TEXT); field = doc.GetFieldable(DocHelper.TEXT_FIELD_UTF2_KEY); Assert.IsTrue(field != null, "field is null and it shouldn't be"); Assert.IsTrue(field.IsLazy == true, "Field is lazy and it should not be"); Assert.IsTrue(field.StringValue(null).Equals(DocHelper.FIELD_UTF2_TEXT) == true, field.StringValue(null) + " is not equal to " + DocHelper.FIELD_UTF2_TEXT); field = doc.GetFieldable(DocHelper.LAZY_FIELD_BINARY_KEY); Assert.IsTrue(field != null, "field is null and it shouldn't be"); Assert.IsTrue(field.StringValue(null) == null, "stringValue isn't null for lazy binary field"); byte[] bytes = field.GetBinaryValue(null); Assert.IsTrue(bytes != null, "bytes is null and it shouldn't be"); Assert.IsTrue(DocHelper.LAZY_FIELD_BINARY_BYTES.Length == bytes.Length, ""); for (int i = 0; i < bytes.Length; i++) { Assert.IsTrue(bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i], "byte[" + i + "] is mismatched"); } }
protected internal override object ConvertFieldValue(IFieldable field) { var value = base.ConvertFieldValue(field); if (typeToValueTypeConverter != null) { value = typeToValueTypeConverter.ConvertFrom(value); } return(value); }
/// <summary> <p/>Removes all fields with the given name from the document. /// If there is no field with the specified name, the document remains unchanged.<p/> /// <p/> Note that the removeField(s) methods like the add method only make sense /// prior to adding a document to an index. These methods cannot /// be used to change the content of an existing index! In order to achieve this, /// a document has to be deleted from an index and a new changed version of that /// document has to be added.<p/> /// </summary> public void RemoveFields(System.String name) { for (int i = fields.Count - 1; i >= 0; i--) { IFieldable field = fields[i]; if (field.Name.Equals(name)) { fields.RemoveAt(i); } } }
internal override void Start(IFieldable f) { if (fieldState.attributeSource.HasAttribute <IPayloadAttribute>()) { payloadAttribute = fieldState.attributeSource.GetAttribute <IPayloadAttribute>(); } else { payloadAttribute = null; } }
internal override void Start(IFieldable f) { if (doVectorOffsets) { offsetAttribute = fieldState.attributeSource.AddAttribute <IOffsetAttribute>(); } else { offsetAttribute = null; } }
private static string GetStringValue(IFieldable field) { switch (field.StringValue) { case Constants.NullValue: return null; case Constants.EmptyString: return string.Empty; default: return field.StringValue; } }
void Select(IFieldable toSelect, bool state, bool additive) { //Clear by click on empty space if (toSelect == null) { DeselectAll(); } else { if (additive) { if (state) { ConsumateSelection(); } else { ConsumateSelection(); } } else { //if(selected.Contains(toSelect)) if (selected.Count > 0 || !state) { DeselectAll(toSelect); } //else //{ //state = !state; //} if (state) { ConsumateSelection(); } } } void ConsumateSelection() { if (state) { selected.Add(toSelect); } else { selected.Remove(toSelect); } toSelect.ShowSelection(state); _UImanager.selectionmetric.UpdateText(); } }
internal virtual void loadDoc(IndexReader ir) { // beware of deleted docs in the future Document doc = ir.Document(rand.Next(ir.MaxDoc), new AnonymousClassFieldSelector(this)); var fields = doc.GetFields(); for (int i = 0; i < fields.Count; i++) { IFieldable f = fields[i]; Enclosing_Instance.ValidateField(f); } }
/// <summary> <p/>Removes field with the specified name from the document. /// If multiple fields exist with this name, this method removes the first field that has been added. /// If there is no field with the specified name, the document remains unchanged.<p/> /// <p/> Note that the removeField(s) methods like the add method only make sense /// prior to adding a document to an index. These methods cannot /// be used to change the content of an existing index! In order to achieve this, /// a document has to be deleted from an index and a new changed version of that /// document has to be added.<p/> /// </summary> public void RemoveField(System.String name) { System.Collections.Generic.IEnumerator <IFieldable> it = fields.GetEnumerator(); while (it.MoveNext()) { IFieldable field = it.Current; if (field.Name.Equals(name)) { fields.Remove(field); return; } } }
public IFieldable CreateFieldItem(Sprite sprite, Vector3 scale, Vector3 position, Vector3 pixelSize, Color color) { IFieldable fieldable = _itemPool.Get(); _Container.Inject(fieldable); fieldable.Initialize(); fieldable.SetParent(_UImanager.fieldPanel); fieldable.SetScale(scale); fieldable.SetPosition(position); fieldable.SetSprite(sprite); fieldable.SetPixelSize(pixelSize); fieldable.SetColor(color); all.Add(fieldable); return(fieldable); }
private static JToken GetInt(IndexSearcher searcher, int doc, string name) { IFieldable field = searcher.Doc(doc).GetFieldable(name); if (field != null) { string s = field.StringValue; int i = 0; if (int.TryParse(s, out i)) { return(new JValue(i)); } } return(null); }
/// <summary> /// 将T转换成doc /// </summary> /// <param name="model"></param> /// <param name="fieldModelList"></param> /// <returns></returns> private Document ParseModeltoDoc(T model, IEnumerable <FieldDataModel> fieldModelList) { Document document = new Document(); Type type = model.GetType(); foreach (var item in fieldModelList) { PropertyInfo propertyInfo = type.GetProperty(item.PropertyName); var propertyValue = propertyInfo.GetValue(model); if (propertyValue != null) { string valueString = propertyValue.ToString(); IFieldable fieldable = null; if (item.FieldType == TypeCode.String) { fieldable = new Field(item.FieldName, valueString, item.Store, item.Index, item.TermVector); } else { NumericField numericField = new NumericField(item.FieldName, item.Store, item.Index == Field.Index.ANALYZED_NO_NORMS); switch (item.FieldType) { case TypeCode.Double: numericField.SetDoubleValue(Convert.ToDouble(valueString)); break; case TypeCode.Single: numericField.SetFloatValue(Convert.ToSingle(valueString)); break; case TypeCode.Int32: numericField.SetIntValue(Convert.ToInt32(valueString)); break; case TypeCode.Int64: numericField.SetLongValue(Convert.ToInt64(valueString)); break; default: break; } fieldable = numericField; } document.Add(fieldable); } } return(document); }
public void AddField(IFieldable field, FieldInfo fieldInfo) { if (doc == null) { doc = storedFieldsWriter.GetPerDoc(); doc.docID = docState.docID; localFieldsWriter.SetFieldsStream(doc.fdt); System.Diagnostics.Debug.Assert(doc.numStoredFields == 0, "doc.numStoredFields=" + doc.numStoredFields); System.Diagnostics.Debug.Assert(0 == doc.fdt.Length); System.Diagnostics.Debug.Assert(0 == doc.fdt.FilePointer); } localFieldsWriter.WriteField(fieldInfo, field); System.Diagnostics.Debug.Assert(docState.TestPoint("StoredFieldsWriterPerThread.processFields.writeField")); doc.numStoredFields++; }
public void DeselectAll(IFieldable except = null) { for (int i = selected.Count - 1; i > -1; i--) { if (except == null || selected[i] != except) { selected[i].ShowSelection(false); selected.RemoveAt(i); } else { continue; } } _UImanager.selectionmetric.UpdateText(); }
/// <summary>Prints the fields of a document for human consumption. </summary> public override System.String ToString() { System.Text.StringBuilder buffer = new System.Text.StringBuilder(); buffer.Append("Document<"); for (int i = 0; i < fields.Count; i++) { IFieldable field = fields[i]; buffer.Append(field.ToString()); if (i != fields.Count - 1) { buffer.Append(" "); } } buffer.Append(">"); return(buffer.ToString()); }
public static FieldIndex ToFieldIndex(this IFieldable fieldable) { if (!fieldable.IsIndexed) { return(FieldIndex.NO); } if (fieldable.IsTokenized) { return((fieldable.OmitNorms) ? FieldIndex.ANALYZED_NO_NORMS : FieldIndex.ANALYZED); } else { return((fieldable.OmitNorms) ? FieldIndex.NOT_ANALYZED_NO_NORMS : FieldIndex.NOT_ANALYZED); } }
internal override bool Start(IFieldable[] fields, int count) { doVectors = false; doVectorPositions = false; doVectorOffsets = false; for (int i = 0; i < count; i++) { IFieldable field = fields[i]; if (field.IsIndexed && field.IsTermVectorStored) { doVectors = true; doVectorPositions |= field.IsStorePositionWithTermVector; doVectorOffsets |= field.IsStoreOffsetWithTermVector; } } if (doVectors) { if (perThread.doc == null) { perThread.doc = termsWriter.GetPerDoc(); perThread.doc.docID = docState.docID; System.Diagnostics.Debug.Assert(perThread.doc.numVectorFields == 0); System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.Length); System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.FilePointer); } System.Diagnostics.Debug.Assert(perThread.doc.docID == docState.docID); if (termsHashPerField.numPostings != 0) { // Only necessary if previous doc hit a // non-aborting exception while writing vectors in // this field: termsHashPerField.Reset(); perThread.termsHashPerThread.Reset(false); } } // TODO: only if needed for performance //perThread.postingsCount = 0; return(doVectors); }
internal override bool Start(IFieldable[] fields, int count) { doVectors = false; doVectorPositions = false; doVectorOffsets = false; for (int i = 0; i < count; i++) { IFieldable field = fields[i]; if (field.IsIndexed && field.IsTermVectorStored) { doVectors = true; doVectorPositions |= field.IsStorePositionWithTermVector; doVectorOffsets |= field.IsStoreOffsetWithTermVector; } } if (doVectors) { if (perThread.doc == null) { perThread.doc = termsWriter.GetPerDoc(); perThread.doc.docID = docState.docID; System.Diagnostics.Debug.Assert(perThread.doc.numVectorFields == 0); System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.Length); System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.FilePointer); } System.Diagnostics.Debug.Assert(perThread.doc.docID == docState.docID); if (termsHashPerField.numPostings != 0) { // Only necessary if previous doc hit a // non-aborting exception while writing vectors in // this field: termsHashPerField.Reset(); perThread.termsHashPerThread.Reset(false); } } // TODO: only if needed for performance //perThread.postingsCount = 0; return doVectors; }
public virtual void TestLoadSize() { FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null); Document doc; doc = reader.Doc(0, new AnonymousClassFieldSelector(this), null); IFieldable f1 = doc.GetFieldable(DocHelper.TEXT_FIELD_1_KEY); IFieldable f3 = doc.GetFieldable(DocHelper.TEXT_FIELD_3_KEY); IFieldable fb = doc.GetFieldable(DocHelper.LAZY_FIELD_BINARY_KEY); Assert.IsTrue(f1.IsBinary); Assert.IsTrue(!f3.IsBinary); Assert.IsTrue(fb.IsBinary); AssertSizeEquals(2 * DocHelper.FIELD_1_TEXT.Length, f1.GetBinaryValue(null)); Assert.AreEqual(DocHelper.FIELD_3_TEXT, f3.StringValue(null)); AssertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.Length, fb.GetBinaryValue(null)); reader.Dispose(); }
public virtual void Test() { Assert.IsTrue(dir != null); Assert.IsTrue(fieldInfos != null); FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null); Assert.IsTrue(reader != null); Assert.IsTrue(reader.Size() == 1); Document doc = reader.Doc(0, null, null); Assert.IsTrue(doc != null); Assert.IsTrue(doc.GetField(DocHelper.TEXT_FIELD_1_KEY) != null); IFieldable field = doc.GetField(DocHelper.TEXT_FIELD_2_KEY); Assert.IsTrue(field != null); Assert.IsTrue(field.IsTermVectorStored == true); Assert.IsTrue(field.IsStoreOffsetWithTermVector == true); Assert.IsTrue(field.IsStorePositionWithTermVector == true); Assert.IsTrue(field.OmitNorms == false); Assert.IsTrue(field.OmitTermFreqAndPositions == false); field = doc.GetField(DocHelper.TEXT_FIELD_3_KEY); Assert.IsTrue(field != null); Assert.IsTrue(field.IsTermVectorStored == false); Assert.IsTrue(field.IsStoreOffsetWithTermVector == false); Assert.IsTrue(field.IsStorePositionWithTermVector == false); Assert.IsTrue(field.OmitNorms == true); Assert.IsTrue(field.OmitTermFreqAndPositions == false); field = doc.GetField(DocHelper.NO_TF_KEY); Assert.IsTrue(field != null); Assert.IsTrue(field.IsTermVectorStored == false); Assert.IsTrue(field.IsStoreOffsetWithTermVector == false); Assert.IsTrue(field.IsStorePositionWithTermVector == false); Assert.IsTrue(field.OmitNorms == false); Assert.IsTrue(field.OmitTermFreqAndPositions == true); reader.Dispose(); }
private static object ConvertType(JsonOperationContext context, IFieldable field, FieldType fieldType, IState state) { if (field.IsBinary) { ThrowBinaryValuesNotSupported(); } var stringValue = field.StringValue(state); if (stringValue == null) { return(null); } if (stringValue == string.Empty) { return(string.Empty); } if (field.IsTokenized == false) { // NULL_VALUE and EMPTY_STRING fields aren't tokenized // this will prevent converting fields with a "NULL_VALUE" string to null switch (stringValue) { case Constants.Documents.Indexing.Fields.NullValue: return(null); case Constants.Documents.Indexing.Fields.EmptyString: return(string.Empty); } } if (fieldType.IsJson == false) { return(stringValue); } return(context.ReadForMemory(stringValue, field.Name)); }
/// <summary>Processes all occurrences of a single field </summary> public abstract void ProcessFields(IFieldable[] fields, int count);
/// <summary> Return the offsetGap from the analyzer assigned to field </summary> public override int GetOffsetGap(IFieldable field) { Analyzer analyzer = analyzerMap[field.Name] ?? defaultAnalyzer; return analyzer.GetOffsetGap(field); }
public override void ProcessFields(IFieldable[] fields, int count) { fieldState.Reset(docState.doc.Boost); int maxFieldLength = docState.maxFieldLength; bool doInvert = consumer.Start(fields, count); for (int i = 0; i < count; i++) { IFieldable field = fields[i]; // TODO FI: this should be "genericized" to querying // consumer if it wants to see this particular field // tokenized. if (field.IsIndexed && doInvert) { bool anyToken; if (fieldState.length > 0) fieldState.position += docState.analyzer.GetPositionIncrementGap(fieldInfo.name); if (!field.IsTokenized) { // un-tokenized field System.String stringValue = field.StringValue; int valueLength = stringValue.Length; perThread.singleToken.Reinit(stringValue, 0, valueLength); fieldState.attributeSource = perThread.singleToken; consumer.Start(field); bool success = false; try { consumer.Add(); success = true; } finally { if (!success) docState.docWriter.SetAborting(); } fieldState.offset += valueLength; fieldState.length++; fieldState.position++; anyToken = valueLength > 0; } else { // tokenized field TokenStream stream; TokenStream streamValue = field.TokenStreamValue; if (streamValue != null) stream = streamValue; else { // the field does not have a TokenStream, // so we have to obtain one from the analyzer System.IO.TextReader reader; // find or make Reader System.IO.TextReader readerValue = field.ReaderValue; if (readerValue != null) reader = readerValue; else { System.String stringValue = field.StringValue; if (stringValue == null) throw new System.ArgumentException("field must have either TokenStream, String or Reader value"); perThread.stringReader.Init(stringValue); reader = perThread.stringReader; } // Tokenize field and add to postingTable stream = docState.analyzer.ReusableTokenStream(fieldInfo.name, reader); } // reset the TokenStream to the first token stream.Reset(); int startLength = fieldState.length; try { int offsetEnd = fieldState.offset - 1; bool hasMoreTokens = stream.IncrementToken(); fieldState.attributeSource = stream; IOffsetAttribute offsetAttribute = fieldState.attributeSource.AddAttribute<IOffsetAttribute>(); IPositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.AddAttribute<IPositionIncrementAttribute>(); consumer.Start(field); for (; ; ) { // If we hit an exception in stream.next below // (which is fairly common, eg if analyzer // chokes on a given document), then it's // non-aborting and (above) this one document // will be marked as deleted, but still // consume a docID if (!hasMoreTokens) break; int posIncr = posIncrAttribute.PositionIncrement; fieldState.position += posIncr; if (fieldState.position > 0) { fieldState.position--; } if (posIncr == 0) fieldState.numOverlap++; bool success = false; try { // If we hit an exception in here, we abort // all buffered documents since the last // flush, on the likelihood that the // internal state of the consumer is now // corrupt and should not be flushed to a // new segment: consumer.Add(); success = true; } finally { if (!success) docState.docWriter.SetAborting(); } fieldState.position++; offsetEnd = fieldState.offset + offsetAttribute.EndOffset; if (++fieldState.length >= maxFieldLength) { if (docState.infoStream != null) docState.infoStream.WriteLine("maxFieldLength " + maxFieldLength + " reached for field " + fieldInfo.name + ", ignoring following tokens"); break; } hasMoreTokens = stream.IncrementToken(); } // trigger streams to perform end-of-stream operations stream.End(); fieldState.offset += offsetAttribute.EndOffset; anyToken = fieldState.length > startLength; } finally { stream.Close(); } } if (anyToken) fieldState.offset += docState.analyzer.GetOffsetGap(field); fieldState.boost *= field.Boost; } // LUCENE-2387: don't hang onto the field, so GC can // reclaim fields[i] = null; } consumer.Finish(); endConsumer.Finish(); }
internal abstract void Start(IFieldable field);
internal override bool Start(IFieldable[] fields, int count) { doCall = consumer.Start(fields, count); if (nextPerField != null) doNextCall = nextPerField.Start(fields, count); return doCall || doNextCall; }
public override void ProcessFields(IFieldable[] fields, int count) { one.ProcessFields(fields, count); two.ProcessFields(fields, count); }
internal override void Start(IFieldable f) { if (doVectorOffsets) { offsetAttribute = fieldState.attributeSource.AddAttribute<IOffsetAttribute>(); } else { offsetAttribute = null; } }
/// <summary> <p/>Adds a field to a document. Several fields may be added with /// the same name. In this case, if the fields are indexed, their text is /// treated as though appended for the purposes of search.<p/> /// <p/> Note that add like the removeField(s) methods only makes sense /// prior to adding a document to an index. These methods cannot /// be used to change the content of an existing index! In order to achieve this, /// a document has to be deleted from an index and a new changed version of that /// document has to be added.<p/> /// </summary> public void Add(IFieldable field) { fields.Add(field); }
internal override void Start(IFieldable f) { termAtt = fieldState.attributeSource.AddAttribute<ITermAttribute>(); consumer.Start(f); if (nextPerField != null) { nextPerField.Start(f); } }
internal void WriteField(FieldInfo fi, IFieldable field) { fieldsStream.WriteVInt(fi.number); byte bits = 0; if (field.IsTokenized) bits |= FieldsWriter.FIELD_IS_TOKENIZED; if (field.IsBinary) bits |= FieldsWriter.FIELD_IS_BINARY; fieldsStream.WriteByte(bits); // compression is disabled for the current field if (field.IsBinary) { byte[] data = field.GetBinaryValue(); int len = field.BinaryLength; int offset = field.BinaryOffset; fieldsStream.WriteVInt(len); fieldsStream.WriteBytes(data, offset, len); } else { fieldsStream.WriteString(field.StringValue); } }
public override DocumentsWriter.DocWriter ProcessDocument() { consumer.StartDocument(); fieldsWriter.StartDocument(); Document doc = docState.doc; System.Diagnostics.Debug.Assert(docFieldProcessor.docWriter.writer.TestPoint("DocumentsWriter.ThreadState.init start")); fieldCount = 0; int thisFieldGen = fieldGen++; System.Collections.Generic.IList<IFieldable> docFields = doc.GetFields(); int numDocFields = docFields.Count; // Absorb any new fields first seen in this document. // Also absorb any changes to fields we had already // seen before (eg suddenly turning on norms or // vectors, etc.): for (int i = 0; i < numDocFields; i++) { IFieldable field = docFields[i]; System.String fieldName = field.Name; // Make sure we have a PerField allocated int hashPos = fieldName.GetHashCode() & hashMask; DocFieldProcessorPerField fp = fieldHash[hashPos]; while (fp != null && !fp.fieldInfo.name.Equals(fieldName)) fp = fp.next; if (fp == null) { // TODO FI: we need to genericize the "flags" that a // field holds, and, how these flags are merged; it // needs to be more "pluggable" such that if I want // to have a new "thing" my Fields can do, I can // easily add it FieldInfo fi = fieldInfos.Add(fieldName, field.IsIndexed, field.IsTermVectorStored, field.IsStorePositionWithTermVector, field.IsStoreOffsetWithTermVector, field.OmitNorms, false, field.OmitTermFreqAndPositions); fp = new DocFieldProcessorPerField(this, fi); fp.next = fieldHash[hashPos]; fieldHash[hashPos] = fp; totalFieldCount++; if (totalFieldCount >= fieldHash.Length / 2) Rehash(); } else { fp.fieldInfo.Update(field.IsIndexed, field.IsTermVectorStored, field.IsStorePositionWithTermVector, field.IsStoreOffsetWithTermVector, field.OmitNorms, false, field.OmitTermFreqAndPositions); } if (thisFieldGen != fp.lastGen) { // First time we're seeing this field for this doc fp.fieldCount = 0; if (fieldCount == fields.Length) { int newSize = fields.Length * 2; DocFieldProcessorPerField[] newArray = new DocFieldProcessorPerField[newSize]; Array.Copy(fields, 0, newArray, 0, fieldCount); fields = newArray; } fields[fieldCount++] = fp; fp.lastGen = thisFieldGen; } if (fp.fieldCount == fp.fields.Length) { IFieldable[] newArray = new IFieldable[fp.fields.Length * 2]; Array.Copy(fp.fields, 0, newArray, 0, fp.fieldCount); fp.fields = newArray; } fp.fields[fp.fieldCount++] = field; if (field.IsStored) { fieldsWriter.AddField(field, fp.fieldInfo); } } // If we are writing vectors then we must visit // fields in sorted order so they are written in // sorted order. TODO: we actually only need to // sort the subset of fields that have vectors // enabled; we could save [small amount of] CPU // here. QuickSort(fields, 0, fieldCount - 1); for (int i = 0; i < fieldCount; i++) fields[i].consumer.ProcessFields(fields[i].fields, fields[i].fieldCount); if (docState.maxTermPrefix != null && docState.infoStream != null) { docState.infoStream.WriteLine("WARNING: document contains at least one immense term (longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'"); docState.maxTermPrefix = null; } DocumentsWriter.DocWriter one = fieldsWriter.FinishDocument(); DocumentsWriter.DocWriter two = consumer.FinishDocument(); if (one == null) { return two; } else if (two == null) { return one; } else { PerDoc both = GetPerDoc(); both.docID = docState.docID; System.Diagnostics.Debug.Assert(one.docID == docState.docID); System.Diagnostics.Debug.Assert(two.docID == docState.docID); both.one = one; both.two = two; return both; } }
private static void Add(System.Collections.IDictionary map, IFieldable field) { map[field.Name] = field; }
internal abstract bool Start(IFieldable[] fields, int count);
public override int GetOffsetGap(IFieldable field) { return GetAnalyzer(field.Name).GetOffsetGap(field); }