private object ConvertType(Lucene.Net.Documents.Document indexDocument, IFieldable field, FieldType fieldType)
{
if (field.IsBinary)
{
throw new NotImplementedException("Support for binary values");
}
var stringValue = field.StringValue;
if (stringValue == Constants.NullValue || stringValue == null)
{
return(null);
}
if (stringValue == Constants.EmptyString || stringValue == string.Empty)
{
return(string.Empty);
}
if (fieldType.IsJson == false)
{
return(stringValue);
}
var bytes = _context.Encoding.GetBytes(stringValue);
var ms = new MemoryStream(bytes);
return(_context.ReadForMemory(ms, field.Name));
}
public virtual void TestLazyFieldsAfterClose()
{
Assert.IsTrue(dir != null);
Assert.IsTrue(fieldInfos != null);
FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null);
Assert.IsTrue(reader != null);
Assert.IsTrue(reader.Size() == 1);
ISet <string> loadFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>();
loadFieldNames.Add(DocHelper.TEXT_FIELD_1_KEY);
loadFieldNames.Add(DocHelper.TEXT_FIELD_UTF1_KEY);
ISet <string> lazyFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>();
lazyFieldNames.Add(DocHelper.LARGE_LAZY_FIELD_KEY);
lazyFieldNames.Add(DocHelper.LAZY_FIELD_KEY);
lazyFieldNames.Add(DocHelper.LAZY_FIELD_BINARY_KEY);
lazyFieldNames.Add(DocHelper.TEXT_FIELD_UTF2_KEY);
SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames);
Document doc = reader.Doc(0, fieldSelector, null);
Assert.IsTrue(doc != null, "doc is null and it shouldn't be");
IFieldable field = doc.GetFieldable(DocHelper.LAZY_FIELD_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy, "field is not lazy and it should be");
reader.Dispose();
Assert.Throws <AlreadyClosedException>(() => { var value = field.StringValue(null); },
"did not hit AlreadyClosedException as expected");
}
private static object ConvertType(JsonOperationContext context, IFieldable field, FieldType fieldType, IState state)
{
if (field.IsBinary)
{
ThrowBinaryValuesNotSupported();
}
var stringValue = field.StringValue(state);
if (stringValue == Constants.Documents.Indexing.Fields.NullValue || stringValue == null)
{
return(null);
}
if (stringValue == Constants.Documents.Indexing.Fields.EmptyString || stringValue == string.Empty)
{
return(string.Empty);
}
if (fieldType.IsJson == false)
{
return(stringValue);
}
var bytes = Encodings.Utf8.GetBytes(stringValue);
var ms = new MemoryStream(bytes);
return(context.ReadForMemory(ms, field.Name));
}
internal void WriteField(FieldInfo fi, IFieldable field, IState state)
{
fieldsStream.WriteVInt(fi.number);
byte bits = 0;
if (field.IsTokenized)
{
bits |= FieldsWriter.FIELD_IS_TOKENIZED;
}
if (field.IsBinary)
{
bits |= FieldsWriter.FIELD_IS_BINARY;
}
fieldsStream.WriteByte(bits);
// compression is disabled for the current field
if (field.IsBinary)
{
byte[] data = field.GetBinaryValue(state);
int len = field.BinaryLength;
int offset = field.BinaryOffset;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, offset, len);
}
else
{
fieldsStream.WriteString(field.StringValue(state));
}
}
public static void CheckNorms(IndexReader reader)
{
// test omit norms
for (int i = 0; i < DocHelper.fields.Length; i++)
{
IFieldable f = DocHelper.fields[i];
if (f.IsIndexed)
{
Assert.AreEqual(reader.HasNorms(f.Name, null), !f.OmitNorms);
Assert.AreEqual(reader.HasNorms(f.Name, null), !DocHelper.noNorms.Contains(f.Name));
if (!reader.HasNorms(f.Name, null))
{
// test for fake norms of 1.0 or null depending on the flag
byte[] norms = reader.Norms(f.Name, null);
byte norm1 = DefaultSimilarity.EncodeNorm(1.0f);
Assert.IsNull(norms);
norms = new byte[reader.MaxDoc];
reader.Norms(f.Name, norms, 0, null);
for (int j = 0; j < reader.MaxDoc; j++)
{
Assert.AreEqual(norms[j], norm1);
}
}
}
}
}
private static FieldType GetFieldType(IFieldable field, Lucene.Net.Documents.Document indexDocument)
{
return(new FieldType
{
IsArray = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.IsArrayFieldSuffix) != null,
IsJson = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.ConvertToJsonSuffix) != null,
});
}
internal virtual void ValidateField(IFieldable f)
{
System.String val = f.StringValue;
if (!val.StartsWith("^") || !val.EndsWith("$"))
{
throw new System.SystemException("Invalid field:" + f.ToString() + " val=" + val);
}
}
/// <summary>
/// Allows method chaining of add field
/// </summary>
/// <param name="document">Document to add field to</param>
/// <param name="field">Field to add</param>
/// <returns>Document</returns>
public static Document AddField(this Document document, IFieldable field)
{
if (document != null)
{
document.Add(field);
}
return(document);
}
internal override void Start(IFieldable f)
{
termAtt = fieldState.attributeSource.AddAttribute <ITermAttribute>();
consumer.Start(f);
if (nextPerField != null)
{
nextPerField.Start(f);
}
}
protected internal virtual object ConvertFieldValue(IFieldable field)
{
var fieldValue = (object)field.StringValue;
if (converter != null)
{
fieldValue = converter.ConvertFrom(fieldValue);
}
return(fieldValue);
}
public virtual void TestLazyFields()
{
Assert.IsTrue(dir != null);
Assert.IsTrue(fieldInfos != null);
FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null);
Assert.IsTrue(reader != null);
Assert.IsTrue(reader.Size() == 1);
ISet <string> loadFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>();
loadFieldNames.Add(DocHelper.TEXT_FIELD_1_KEY);
loadFieldNames.Add(DocHelper.TEXT_FIELD_UTF1_KEY);
ISet <string> lazyFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>();
//new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
lazyFieldNames.Add(DocHelper.LARGE_LAZY_FIELD_KEY);
lazyFieldNames.Add(DocHelper.LAZY_FIELD_KEY);
lazyFieldNames.Add(DocHelper.LAZY_FIELD_BINARY_KEY);
lazyFieldNames.Add(DocHelper.TEXT_FIELD_UTF2_KEY);
SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames);
Document doc = reader.Doc(0, fieldSelector, null);
Assert.IsTrue(doc != null, "doc is null and it shouldn't be");
IFieldable field = doc.GetFieldable(DocHelper.LAZY_FIELD_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy, "field is not lazy and it should be");
System.String value_Renamed = field.StringValue(null);
Assert.IsTrue(value_Renamed != null, "value is null and it shouldn't be");
Assert.IsTrue(value_Renamed.Equals(DocHelper.LAZY_FIELD_TEXT) == true, value_Renamed + " is not equal to " + DocHelper.LAZY_FIELD_TEXT);
field = doc.GetFieldable(DocHelper.TEXT_FIELD_1_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy == false, "Field is lazy and it should not be");
field = doc.GetFieldable(DocHelper.TEXT_FIELD_UTF1_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy == false, "Field is lazy and it should not be");
Assert.IsTrue(field.StringValue(null).Equals(DocHelper.FIELD_UTF1_TEXT) == true, field.StringValue(null) + " is not equal to " + DocHelper.FIELD_UTF1_TEXT);
field = doc.GetFieldable(DocHelper.TEXT_FIELD_UTF2_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy == true, "Field is lazy and it should not be");
Assert.IsTrue(field.StringValue(null).Equals(DocHelper.FIELD_UTF2_TEXT) == true, field.StringValue(null) + " is not equal to " + DocHelper.FIELD_UTF2_TEXT);
field = doc.GetFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.StringValue(null) == null, "stringValue isn't null for lazy binary field");
byte[] bytes = field.GetBinaryValue(null);
Assert.IsTrue(bytes != null, "bytes is null and it shouldn't be");
Assert.IsTrue(DocHelper.LAZY_FIELD_BINARY_BYTES.Length == bytes.Length, "");
for (int i = 0; i < bytes.Length; i++)
{
Assert.IsTrue(bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i], "byte[" + i + "] is mismatched");
}
}
protected internal override object ConvertFieldValue(IFieldable field)
{
var value = base.ConvertFieldValue(field);
if (typeToValueTypeConverter != null)
{
value = typeToValueTypeConverter.ConvertFrom(value);
}
return(value);
}
/// <summary> <p/>Removes all fields with the given name from the document.
/// If there is no field with the specified name, the document remains unchanged.<p/>
/// <p/> Note that the removeField(s) methods like the add method only make sense
/// prior to adding a document to an index. These methods cannot
/// be used to change the content of an existing index! In order to achieve this,
/// a document has to be deleted from an index and a new changed version of that
/// document has to be added.<p/>
/// </summary>
public void RemoveFields(System.String name)
{
for (int i = fields.Count - 1; i >= 0; i--)
{
IFieldable field = fields[i];
if (field.Name.Equals(name))
{
fields.RemoveAt(i);
}
}
}
internal override void Start(IFieldable f)
{
if (fieldState.attributeSource.HasAttribute <IPayloadAttribute>())
{
payloadAttribute = fieldState.attributeSource.GetAttribute <IPayloadAttribute>();
}
else
{
payloadAttribute = null;
}
}
internal override void Start(IFieldable f)
{
if (doVectorOffsets)
{
offsetAttribute = fieldState.attributeSource.AddAttribute <IOffsetAttribute>();
}
else
{
offsetAttribute = null;
}
}
private static string GetStringValue(IFieldable field)
{
switch (field.StringValue)
{
case Constants.NullValue:
return null;
case Constants.EmptyString:
return string.Empty;
default:
return field.StringValue;
}
}
void Select(IFieldable toSelect, bool state, bool additive)
{
//Clear by click on empty space
if (toSelect == null)
{
DeselectAll();
}
else
{
if (additive)
{
if (state)
{
ConsumateSelection();
}
else
{
ConsumateSelection();
}
}
else
{
//if(selected.Contains(toSelect))
if (selected.Count > 0 || !state)
{
DeselectAll(toSelect);
}
//else
//{
//state = !state;
//}
if (state)
{
ConsumateSelection();
}
}
}
void ConsumateSelection()
{
if (state)
{
selected.Add(toSelect);
}
else
{
selected.Remove(toSelect);
}
toSelect.ShowSelection(state);
_UImanager.selectionmetric.UpdateText();
}
}
internal virtual void loadDoc(IndexReader ir)
{
// beware of deleted docs in the future
Document doc = ir.Document(rand.Next(ir.MaxDoc), new AnonymousClassFieldSelector(this));
var fields = doc.GetFields();
for (int i = 0; i < fields.Count; i++)
{
IFieldable f = fields[i];
Enclosing_Instance.ValidateField(f);
}
}
/// <summary> <p/>Removes field with the specified name from the document.
/// If multiple fields exist with this name, this method removes the first field that has been added.
/// If there is no field with the specified name, the document remains unchanged.<p/>
/// <p/> Note that the removeField(s) methods like the add method only make sense
/// prior to adding a document to an index. These methods cannot
/// be used to change the content of an existing index! In order to achieve this,
/// a document has to be deleted from an index and a new changed version of that
/// document has to be added.<p/>
/// </summary>
public void RemoveField(System.String name)
{
System.Collections.Generic.IEnumerator <IFieldable> it = fields.GetEnumerator();
while (it.MoveNext())
{
IFieldable field = it.Current;
if (field.Name.Equals(name))
{
fields.Remove(field);
return;
}
}
}
public IFieldable CreateFieldItem(Sprite sprite, Vector3 scale, Vector3 position, Vector3 pixelSize, Color color)
{
IFieldable fieldable = _itemPool.Get();
_Container.Inject(fieldable);
fieldable.Initialize();
fieldable.SetParent(_UImanager.fieldPanel);
fieldable.SetScale(scale);
fieldable.SetPosition(position);
fieldable.SetSprite(sprite);
fieldable.SetPixelSize(pixelSize);
fieldable.SetColor(color);
all.Add(fieldable);
return(fieldable);
}
private static JToken GetInt(IndexSearcher searcher, int doc, string name)
{
IFieldable field = searcher.Doc(doc).GetFieldable(name);
if (field != null)
{
string s = field.StringValue;
int i = 0;
if (int.TryParse(s, out i))
{
return(new JValue(i));
}
}
return(null);
}
/// <summary>
/// 将T转换成doc
/// </summary>
/// <param name="model"></param>
/// <param name="fieldModelList"></param>
/// <returns></returns>
private Document ParseModeltoDoc(T model, IEnumerable <FieldDataModel> fieldModelList)
{
Document document = new Document();
Type type = model.GetType();
foreach (var item in fieldModelList)
{
PropertyInfo propertyInfo = type.GetProperty(item.PropertyName);
var propertyValue = propertyInfo.GetValue(model);
if (propertyValue != null)
{
string valueString = propertyValue.ToString();
IFieldable fieldable = null;
if (item.FieldType == TypeCode.String)
{
fieldable = new Field(item.FieldName, valueString, item.Store, item.Index, item.TermVector);
}
else
{
NumericField numericField = new NumericField(item.FieldName, item.Store, item.Index == Field.Index.ANALYZED_NO_NORMS);
switch (item.FieldType)
{
case TypeCode.Double:
numericField.SetDoubleValue(Convert.ToDouble(valueString));
break;
case TypeCode.Single:
numericField.SetFloatValue(Convert.ToSingle(valueString));
break;
case TypeCode.Int32:
numericField.SetIntValue(Convert.ToInt32(valueString));
break;
case TypeCode.Int64:
numericField.SetLongValue(Convert.ToInt64(valueString));
break;
default:
break;
}
fieldable = numericField;
}
document.Add(fieldable);
}
}
return(document);
}
public void AddField(IFieldable field, FieldInfo fieldInfo)
{
if (doc == null)
{
doc = storedFieldsWriter.GetPerDoc();
doc.docID = docState.docID;
localFieldsWriter.SetFieldsStream(doc.fdt);
System.Diagnostics.Debug.Assert(doc.numStoredFields == 0, "doc.numStoredFields=" + doc.numStoredFields);
System.Diagnostics.Debug.Assert(0 == doc.fdt.Length);
System.Diagnostics.Debug.Assert(0 == doc.fdt.FilePointer);
}
localFieldsWriter.WriteField(fieldInfo, field);
System.Diagnostics.Debug.Assert(docState.TestPoint("StoredFieldsWriterPerThread.processFields.writeField"));
doc.numStoredFields++;
}
public void DeselectAll(IFieldable except = null)
{
for (int i = selected.Count - 1; i > -1; i--)
{
if (except == null || selected[i] != except)
{
selected[i].ShowSelection(false);
selected.RemoveAt(i);
}
else
{
continue;
}
}
_UImanager.selectionmetric.UpdateText();
}
/// <summary>Prints the fields of a document for human consumption. </summary>
public override System.String ToString()
{
System.Text.StringBuilder buffer = new System.Text.StringBuilder();
buffer.Append("Document<");
for (int i = 0; i < fields.Count; i++)
{
IFieldable field = fields[i];
buffer.Append(field.ToString());
if (i != fields.Count - 1)
{
buffer.Append(" ");
}
}
buffer.Append(">");
return(buffer.ToString());
}
public void AddField(IFieldable field, FieldInfo fieldInfo)
{
if (doc == null)
{
doc = storedFieldsWriter.GetPerDoc();
doc.docID = docState.docID;
localFieldsWriter.SetFieldsStream(doc.fdt);
System.Diagnostics.Debug.Assert(doc.numStoredFields == 0, "doc.numStoredFields=" + doc.numStoredFields);
System.Diagnostics.Debug.Assert(0 == doc.fdt.Length);
System.Diagnostics.Debug.Assert(0 == doc.fdt.FilePointer);
}
localFieldsWriter.WriteField(fieldInfo, field);
System.Diagnostics.Debug.Assert(docState.TestPoint("StoredFieldsWriterPerThread.processFields.writeField"));
doc.numStoredFields++;
}
public static FieldIndex ToFieldIndex(this IFieldable fieldable)
{
if (!fieldable.IsIndexed)
{
return(FieldIndex.NO);
}
if (fieldable.IsTokenized)
{
return((fieldable.OmitNorms) ? FieldIndex.ANALYZED_NO_NORMS : FieldIndex.ANALYZED);
}
else
{
return((fieldable.OmitNorms) ? FieldIndex.NOT_ANALYZED_NO_NORMS : FieldIndex.NOT_ANALYZED);
}
}
internal override bool Start(IFieldable[] fields, int count)
{
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
for (int i = 0; i < count; i++)
{
IFieldable field = fields[i];
if (field.IsIndexed && field.IsTermVectorStored)
{
doVectors = true;
doVectorPositions |= field.IsStorePositionWithTermVector;
doVectorOffsets |= field.IsStoreOffsetWithTermVector;
}
}
if (doVectors)
{
if (perThread.doc == null)
{
perThread.doc = termsWriter.GetPerDoc();
perThread.doc.docID = docState.docID;
System.Diagnostics.Debug.Assert(perThread.doc.numVectorFields == 0);
System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.Length);
System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.FilePointer);
}
System.Diagnostics.Debug.Assert(perThread.doc.docID == docState.docID);
if (termsHashPerField.numPostings != 0)
{
// Only necessary if previous doc hit a
// non-aborting exception while writing vectors in
// this field:
termsHashPerField.Reset();
perThread.termsHashPerThread.Reset(false);
}
}
// TODO: only if needed for performance
//perThread.postingsCount = 0;
return(doVectors);
}
internal override bool Start(IFieldable[] fields, int count)
{
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
for (int i = 0; i < count; i++)
{
IFieldable field = fields[i];
if (field.IsIndexed && field.IsTermVectorStored)
{
doVectors = true;
doVectorPositions |= field.IsStorePositionWithTermVector;
doVectorOffsets |= field.IsStoreOffsetWithTermVector;
}
}
if (doVectors)
{
if (perThread.doc == null)
{
perThread.doc = termsWriter.GetPerDoc();
perThread.doc.docID = docState.docID;
System.Diagnostics.Debug.Assert(perThread.doc.numVectorFields == 0);
System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.Length);
System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.FilePointer);
}
System.Diagnostics.Debug.Assert(perThread.doc.docID == docState.docID);
if (termsHashPerField.numPostings != 0)
{
// Only necessary if previous doc hit a
// non-aborting exception while writing vectors in
// this field:
termsHashPerField.Reset();
perThread.termsHashPerThread.Reset(false);
}
}
// TODO: only if needed for performance
//perThread.postingsCount = 0;
return doVectors;
}
public virtual void TestLoadSize()
{
FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null);
Document doc;
doc = reader.Doc(0, new AnonymousClassFieldSelector(this), null);
IFieldable f1 = doc.GetFieldable(DocHelper.TEXT_FIELD_1_KEY);
IFieldable f3 = doc.GetFieldable(DocHelper.TEXT_FIELD_3_KEY);
IFieldable fb = doc.GetFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
Assert.IsTrue(f1.IsBinary);
Assert.IsTrue(!f3.IsBinary);
Assert.IsTrue(fb.IsBinary);
AssertSizeEquals(2 * DocHelper.FIELD_1_TEXT.Length, f1.GetBinaryValue(null));
Assert.AreEqual(DocHelper.FIELD_3_TEXT, f3.StringValue(null));
AssertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.Length, fb.GetBinaryValue(null));
reader.Dispose();
}
public virtual void Test()
{
Assert.IsTrue(dir != null);
Assert.IsTrue(fieldInfos != null);
FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null);
Assert.IsTrue(reader != null);
Assert.IsTrue(reader.Size() == 1);
Document doc = reader.Doc(0, null, null);
Assert.IsTrue(doc != null);
Assert.IsTrue(doc.GetField(DocHelper.TEXT_FIELD_1_KEY) != null);
IFieldable field = doc.GetField(DocHelper.TEXT_FIELD_2_KEY);
Assert.IsTrue(field != null);
Assert.IsTrue(field.IsTermVectorStored == true);
Assert.IsTrue(field.IsStoreOffsetWithTermVector == true);
Assert.IsTrue(field.IsStorePositionWithTermVector == true);
Assert.IsTrue(field.OmitNorms == false);
Assert.IsTrue(field.OmitTermFreqAndPositions == false);
field = doc.GetField(DocHelper.TEXT_FIELD_3_KEY);
Assert.IsTrue(field != null);
Assert.IsTrue(field.IsTermVectorStored == false);
Assert.IsTrue(field.IsStoreOffsetWithTermVector == false);
Assert.IsTrue(field.IsStorePositionWithTermVector == false);
Assert.IsTrue(field.OmitNorms == true);
Assert.IsTrue(field.OmitTermFreqAndPositions == false);
field = doc.GetField(DocHelper.NO_TF_KEY);
Assert.IsTrue(field != null);
Assert.IsTrue(field.IsTermVectorStored == false);
Assert.IsTrue(field.IsStoreOffsetWithTermVector == false);
Assert.IsTrue(field.IsStorePositionWithTermVector == false);
Assert.IsTrue(field.OmitNorms == false);
Assert.IsTrue(field.OmitTermFreqAndPositions == true);
reader.Dispose();
}
private static object ConvertType(JsonOperationContext context, IFieldable field, FieldType fieldType, IState state)
{
if (field.IsBinary)
{
ThrowBinaryValuesNotSupported();
}
var stringValue = field.StringValue(state);
if (stringValue == null)
{
return(null);
}
if (stringValue == string.Empty)
{
return(string.Empty);
}
if (field.IsTokenized == false)
{
// NULL_VALUE and EMPTY_STRING fields aren't tokenized
// this will prevent converting fields with a "NULL_VALUE" string to null
switch (stringValue)
{
case Constants.Documents.Indexing.Fields.NullValue:
return(null);
case Constants.Documents.Indexing.Fields.EmptyString:
return(string.Empty);
}
}
if (fieldType.IsJson == false)
{
return(stringValue);
}
return(context.ReadForMemory(stringValue, field.Name));
}
/// <summary>Processes all occurrences of a single field </summary> public abstract void ProcessFields(IFieldable[] fields, int count);
internal virtual void ValidateField(IFieldable f)
{
System.String val = f.StringValue;
if (!val.StartsWith("^") || !val.EndsWith("$"))
{
throw new System.SystemException("Invalid field:" + f.ToString() + " val=" + val);
}
}
/// <summary> Return the offsetGap from the analyzer assigned to field </summary>
public override int GetOffsetGap(IFieldable field)
{
Analyzer analyzer = analyzerMap[field.Name] ?? defaultAnalyzer;
return analyzer.GetOffsetGap(field);
}
public override void ProcessFields(IFieldable[] fields, int count)
{
fieldState.Reset(docState.doc.Boost);
int maxFieldLength = docState.maxFieldLength;
bool doInvert = consumer.Start(fields, count);
for (int i = 0; i < count; i++)
{
IFieldable field = fields[i];
// TODO FI: this should be "genericized" to querying
// consumer if it wants to see this particular field
// tokenized.
if (field.IsIndexed && doInvert)
{
bool anyToken;
if (fieldState.length > 0)
fieldState.position += docState.analyzer.GetPositionIncrementGap(fieldInfo.name);
if (!field.IsTokenized)
{
// un-tokenized field
System.String stringValue = field.StringValue;
int valueLength = stringValue.Length;
perThread.singleToken.Reinit(stringValue, 0, valueLength);
fieldState.attributeSource = perThread.singleToken;
consumer.Start(field);
bool success = false;
try
{
consumer.Add();
success = true;
}
finally
{
if (!success)
docState.docWriter.SetAborting();
}
fieldState.offset += valueLength;
fieldState.length++;
fieldState.position++;
anyToken = valueLength > 0;
}
else
{
// tokenized field
TokenStream stream;
TokenStream streamValue = field.TokenStreamValue;
if (streamValue != null)
stream = streamValue;
else
{
// the field does not have a TokenStream,
// so we have to obtain one from the analyzer
System.IO.TextReader reader; // find or make Reader
System.IO.TextReader readerValue = field.ReaderValue;
if (readerValue != null)
reader = readerValue;
else
{
System.String stringValue = field.StringValue;
if (stringValue == null)
throw new System.ArgumentException("field must have either TokenStream, String or Reader value");
perThread.stringReader.Init(stringValue);
reader = perThread.stringReader;
}
// Tokenize field and add to postingTable
stream = docState.analyzer.ReusableTokenStream(fieldInfo.name, reader);
}
// reset the TokenStream to the first token
stream.Reset();
int startLength = fieldState.length;
try
{
int offsetEnd = fieldState.offset - 1;
bool hasMoreTokens = stream.IncrementToken();
fieldState.attributeSource = stream;
IOffsetAttribute offsetAttribute = fieldState.attributeSource.AddAttribute<IOffsetAttribute>();
IPositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.AddAttribute<IPositionIncrementAttribute>();
consumer.Start(field);
for (; ; )
{
// If we hit an exception in stream.next below
// (which is fairly common, eg if analyzer
// chokes on a given document), then it's
// non-aborting and (above) this one document
// will be marked as deleted, but still
// consume a docID
if (!hasMoreTokens)
break;
int posIncr = posIncrAttribute.PositionIncrement;
fieldState.position += posIncr;
if (fieldState.position > 0)
{
fieldState.position--;
}
if (posIncr == 0)
fieldState.numOverlap++;
bool success = false;
try
{
// If we hit an exception in here, we abort
// all buffered documents since the last
// flush, on the likelihood that the
// internal state of the consumer is now
// corrupt and should not be flushed to a
// new segment:
consumer.Add();
success = true;
}
finally
{
if (!success)
docState.docWriter.SetAborting();
}
fieldState.position++;
offsetEnd = fieldState.offset + offsetAttribute.EndOffset;
if (++fieldState.length >= maxFieldLength)
{
if (docState.infoStream != null)
docState.infoStream.WriteLine("maxFieldLength " + maxFieldLength + " reached for field " + fieldInfo.name + ", ignoring following tokens");
break;
}
hasMoreTokens = stream.IncrementToken();
}
// trigger streams to perform end-of-stream operations
stream.End();
fieldState.offset += offsetAttribute.EndOffset;
anyToken = fieldState.length > startLength;
}
finally
{
stream.Close();
}
}
if (anyToken)
fieldState.offset += docState.analyzer.GetOffsetGap(field);
fieldState.boost *= field.Boost;
}
// LUCENE-2387: don't hang onto the field, so GC can
// reclaim
fields[i] = null;
}
consumer.Finish();
endConsumer.Finish();
}
private static string GetStringValue(IFieldable field)
{
switch (field.StringValue)
{
case Constants.NullValue:
return null;
case Constants.EmptyString:
return string.Empty;
default:
return field.StringValue;
}
}
internal abstract void Start(IFieldable field);
internal override bool Start(IFieldable[] fields, int count)
{
doCall = consumer.Start(fields, count);
if (nextPerField != null)
doNextCall = nextPerField.Start(fields, count);
return doCall || doNextCall;
}
public override void ProcessFields(IFieldable[] fields, int count)
{
one.ProcessFields(fields, count);
two.ProcessFields(fields, count);
}
internal override void Start(IFieldable f)
{
if (doVectorOffsets)
{
offsetAttribute = fieldState.attributeSource.AddAttribute<IOffsetAttribute>();
}
else
{
offsetAttribute = null;
}
}
/// <summary> <p/>Adds a field to a document. Several fields may be added with
/// the same name. In this case, if the fields are indexed, their text is
/// treated as though appended for the purposes of search.<p/>
/// <p/> Note that add like the removeField(s) methods only makes sense
/// prior to adding a document to an index. These methods cannot
/// be used to change the content of an existing index! In order to achieve this,
/// a document has to be deleted from an index and a new changed version of that
/// document has to be added.<p/>
/// </summary>
public void Add(IFieldable field)
{
fields.Add(field);
}
internal override void Start(IFieldable f)
{
termAtt = fieldState.attributeSource.AddAttribute<ITermAttribute>();
consumer.Start(f);
if (nextPerField != null)
{
nextPerField.Start(f);
}
}
internal void WriteField(FieldInfo fi, IFieldable field)
{
fieldsStream.WriteVInt(fi.number);
byte bits = 0;
if (field.IsTokenized)
bits |= FieldsWriter.FIELD_IS_TOKENIZED;
if (field.IsBinary)
bits |= FieldsWriter.FIELD_IS_BINARY;
fieldsStream.WriteByte(bits);
// compression is disabled for the current field
if (field.IsBinary)
{
byte[] data = field.GetBinaryValue();
int len = field.BinaryLength;
int offset = field.BinaryOffset;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, offset, len);
}
else
{
fieldsStream.WriteString(field.StringValue);
}
}
public override DocumentsWriter.DocWriter ProcessDocument()
{
consumer.StartDocument();
fieldsWriter.StartDocument();
Document doc = docState.doc;
System.Diagnostics.Debug.Assert(docFieldProcessor.docWriter.writer.TestPoint("DocumentsWriter.ThreadState.init start"));
fieldCount = 0;
int thisFieldGen = fieldGen++;
System.Collections.Generic.IList<IFieldable> docFields = doc.GetFields();
int numDocFields = docFields.Count;
// Absorb any new fields first seen in this document.
// Also absorb any changes to fields we had already
// seen before (eg suddenly turning on norms or
// vectors, etc.):
for (int i = 0; i < numDocFields; i++)
{
IFieldable field = docFields[i];
System.String fieldName = field.Name;
// Make sure we have a PerField allocated
int hashPos = fieldName.GetHashCode() & hashMask;
DocFieldProcessorPerField fp = fieldHash[hashPos];
while (fp != null && !fp.fieldInfo.name.Equals(fieldName))
fp = fp.next;
if (fp == null)
{
// TODO FI: we need to genericize the "flags" that a
// field holds, and, how these flags are merged; it
// needs to be more "pluggable" such that if I want
// to have a new "thing" my Fields can do, I can
// easily add it
FieldInfo fi = fieldInfos.Add(fieldName, field.IsIndexed, field.IsTermVectorStored,
field.IsStorePositionWithTermVector, field.IsStoreOffsetWithTermVector,
field.OmitNorms, false, field.OmitTermFreqAndPositions);
fp = new DocFieldProcessorPerField(this, fi);
fp.next = fieldHash[hashPos];
fieldHash[hashPos] = fp;
totalFieldCount++;
if (totalFieldCount >= fieldHash.Length / 2)
Rehash();
}
else
{
fp.fieldInfo.Update(field.IsIndexed, field.IsTermVectorStored,
field.IsStorePositionWithTermVector, field.IsStoreOffsetWithTermVector,
field.OmitNorms, false, field.OmitTermFreqAndPositions);
}
if (thisFieldGen != fp.lastGen)
{
// First time we're seeing this field for this doc
fp.fieldCount = 0;
if (fieldCount == fields.Length)
{
int newSize = fields.Length * 2;
DocFieldProcessorPerField[] newArray = new DocFieldProcessorPerField[newSize];
Array.Copy(fields, 0, newArray, 0, fieldCount);
fields = newArray;
}
fields[fieldCount++] = fp;
fp.lastGen = thisFieldGen;
}
if (fp.fieldCount == fp.fields.Length)
{
IFieldable[] newArray = new IFieldable[fp.fields.Length * 2];
Array.Copy(fp.fields, 0, newArray, 0, fp.fieldCount);
fp.fields = newArray;
}
fp.fields[fp.fieldCount++] = field;
if (field.IsStored)
{
fieldsWriter.AddField(field, fp.fieldInfo);
}
}
// If we are writing vectors then we must visit
// fields in sorted order so they are written in
// sorted order. TODO: we actually only need to
// sort the subset of fields that have vectors
// enabled; we could save [small amount of] CPU
// here.
QuickSort(fields, 0, fieldCount - 1);
for (int i = 0; i < fieldCount; i++)
fields[i].consumer.ProcessFields(fields[i].fields, fields[i].fieldCount);
if (docState.maxTermPrefix != null && docState.infoStream != null)
{
docState.infoStream.WriteLine("WARNING: document contains at least one immense term (longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
docState.maxTermPrefix = null;
}
DocumentsWriter.DocWriter one = fieldsWriter.FinishDocument();
DocumentsWriter.DocWriter two = consumer.FinishDocument();
if (one == null)
{
return two;
}
else if (two == null)
{
return one;
}
else
{
PerDoc both = GetPerDoc();
both.docID = docState.docID;
System.Diagnostics.Debug.Assert(one.docID == docState.docID);
System.Diagnostics.Debug.Assert(two.docID == docState.docID);
both.one = one;
both.two = two;
return both;
}
}
private static void Add(System.Collections.IDictionary map, IFieldable field)
{
map[field.Name] = field;
}
internal abstract bool Start(IFieldable[] fields, int count);
public override int GetOffsetGap(IFieldable field)
{
return GetAnalyzer(field.Name).GetOffsetGap(field);
}
