internal override bool Start(Fieldable[] fields, int count)
{
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
for (int i = 0; i < count; i++)
{
Fieldable field = fields[i];
if (field.IsIndexed() && field.IsTermVectorStored())
{
doVectors = true;
doVectorPositions |= field.IsStorePositionWithTermVector();
doVectorOffsets |= field.IsStoreOffsetWithTermVector();
}
}
if (doVectors)
{
if (perThread.doc == null)
{
perThread.doc = termsWriter.GetPerDoc();
perThread.doc.docID = docState.docID;
System.Diagnostics.Debug.Assert(perThread.doc.numVectorFields == 0);
System.Diagnostics.Debug.Assert(0 == perThread.doc.tvf.Length());
System.Diagnostics.Debug.Assert(0 == perThread.doc.tvf.GetFilePointer());
}
else
{
System.Diagnostics.Debug.Assert(perThread.doc.docID == docState.docID);
if (termsHashPerField.numPostings != 0)
{
// Only necessary if previous doc hit a
// non-aborting exception while writing vectors in
// this field:
termsHashPerField.Reset();
}
}
}
// TODO: only if needed for performance
//perThread.postingsCount = 0;
return(doVectors);
}
public void Reset()
{
if (!postingsCompacted)
{
CompactPostings();
}
System.Diagnostics.Debug.Assert(numPostings <= postingsHash.Length);
if (numPostings > 0)
{
perThread.termsHash.RecyclePostings(postingsHash, numPostings);
Array.Clear(postingsHash, 0, numPostings);
numPostings = 0;
}
postingsCompacted = false;
nextPerField?.Reset();
}
public override void Flush(IDictionary <string, TermsHashConsumerPerField> fieldsToFlush, SegmentWriteState state)
{
// Gather all FieldData's that have postings, across all
// ThreadStates
IList <FreqProxTermsWriterPerField> allFields = new List <FreqProxTermsWriterPerField>();
foreach (TermsHashConsumerPerField f in fieldsToFlush.Values)
{
FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField)f;
if (perField.termsHashPerField.bytesHash.Count > 0)
{
allFields.Add(perField);
}
}
int numAllFields = allFields.Count;
// Sort by field name
CollectionUtil.IntroSort(allFields);
FieldsConsumer consumer = state.SegmentInfo.Codec.PostingsFormat.FieldsConsumer(state);
bool success = false;
try
{
TermsHash termsHash = null;
/*
* Current writer chain:
* FieldsConsumer
* -> IMPL: FormatPostingsTermsDictWriter
* -> TermsConsumer
* -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
* -> DocsConsumer
* -> IMPL: FormatPostingsDocsWriter
* -> PositionsConsumer
* -> IMPL: FormatPostingsPositionsWriter
*/
for (int fieldNumber = 0; fieldNumber < numAllFields; fieldNumber++)
{
FieldInfo fieldInfo = allFields[fieldNumber].fieldInfo;
FreqProxTermsWriterPerField fieldWriter = allFields[fieldNumber];
// If this field has postings then add them to the
// segment
fieldWriter.Flush(fieldInfo.Name, consumer, state);
TermsHashPerField perField = fieldWriter.termsHashPerField;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(termsHash == null || termsHash == perField.termsHash);
}
termsHash = perField.termsHash;
int numPostings = perField.bytesHash.Count;
perField.Reset();
perField.ShrinkHash(numPostings);
fieldWriter.Reset();
}
if (termsHash != null)
{
termsHash.Reset();
}
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(consumer);
}
else
{
IOUtils.DisposeWhileHandlingException(consumer);
}
}
}
// TODO: would be nice to factor out more of this, eg the
// FreqProxFieldMergeState, and code to visit all Fields
// under the same FieldInfo together, up into TermsHash*.
// Other writers would presumably share alot of this...
public override void Flush(System.Collections.IDictionary threadsAndFields, SegmentWriteState state)
{
// Gather all FieldData's that have postings, across all
// ThreadStates
System.Collections.ArrayList allFields = new System.Collections.ArrayList();
System.Collections.IEnumerator it = new System.Collections.Hashtable(threadsAndFields).GetEnumerator();
while (it.MoveNext())
{
System.Collections.DictionaryEntry entry = (System.Collections.DictionaryEntry)it.Current;
System.Collections.ICollection fields = (System.Collections.ICollection)entry.Value;
System.Collections.IEnumerator fieldsIt = fields.GetEnumerator();
while (fieldsIt.MoveNext())
{
FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField)((System.Collections.DictionaryEntry)fieldsIt.Current).Key;
if (perField.termsHashPerField.numPostings > 0)
{
allFields.Add(perField);
}
}
}
// Sort by field name
allFields.Sort();
int numAllFields = allFields.Count;
// TODO: allow Lucene user to customize this consumer:
FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos);
/*
* Current writer chain:
* FormatPostingsFieldsConsumer
* -> IMPL: FormatPostingsFieldsWriter
* -> FormatPostingsTermsConsumer
* -> IMPL: FormatPostingsTermsWriter
* -> FormatPostingsDocConsumer
* -> IMPL: FormatPostingsDocWriter
* -> FormatPostingsPositionsConsumer
* -> IMPL: FormatPostingsPositionsWriter
*/
int start = 0;
while (start < numAllFields)
{
FieldInfo fieldInfo = ((FreqProxTermsWriterPerField)allFields[start]).fieldInfo;
System.String fieldName = fieldInfo.name;
int end = start + 1;
while (end < numAllFields && ((FreqProxTermsWriterPerField)allFields[end]).fieldInfo.name.Equals(fieldName))
{
end++;
}
FreqProxTermsWriterPerField[] fields = new FreqProxTermsWriterPerField[end - start];
for (int i = start; i < end; i++)
{
fields[i - start] = (FreqProxTermsWriterPerField)allFields[i];
// Aggregate the storePayload as seen by the same
// field across multiple threads
fieldInfo.storePayloads |= fields[i - start].hasPayloads;
}
// If this field has postings then add them to the
// segment
AppendPostings(fields, consumer);
for (int i = 0; i < fields.Length; i++)
{
TermsHashPerField perField = fields[i].termsHashPerField;
int numPostings = perField.numPostings;
perField.Reset();
perField.ShrinkHash(numPostings);
fields[i].Reset();
}
start = end;
}
it = new System.Collections.Hashtable(threadsAndFields).GetEnumerator();
while (it.MoveNext())
{
System.Collections.DictionaryEntry entry = (System.Collections.DictionaryEntry)it.Current;
FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread)entry.Key;
perThread.termsHashPerThread.Reset(true);
}
consumer.Finish();
}
// TODO: would be nice to factor out more of this, eg the
// FreqProxFieldMergeState, and code to visit all Fields
// under the same FieldInfo together, up into TermsHash*.
// Other writers would presumably share alot of this...
public override void Flush(Support.Dictionary <TermsHashConsumerPerThread, IList <TermsHashConsumerPerField> > threadsAndFields, SegmentWriteState state)
{
// Gather all FieldData's that have postings, across all
// ThreadStates
List <FreqProxTermsWriterPerField> allFields = new List <FreqProxTermsWriterPerField>();
foreach (KeyValuePair <TermsHashConsumerPerThread, IList <TermsHashConsumerPerField> > entry in threadsAndFields)
{
IList <TermsHashConsumerPerField> fields = entry.Value;
foreach (TermsHashConsumerPerField i in fields)
{
FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField)i;
if (perField.termsHashPerField.numPostings > 0)
{
allFields.Add(perField);
}
}
}
// Sort by field name
allFields.Sort();
int numAllFields = allFields.Count;
// TODO: allow Lucene user to customize this consumer:
FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos);
/*
* Current writer chain:
* FormatPostingsFieldsConsumer
* -> IMPL: FormatPostingsFieldsWriter
* -> FormatPostingsTermsConsumer
* -> IMPL: FormatPostingsTermsWriter
* -> FormatPostingsDocConsumer
* -> IMPL: FormatPostingsDocWriter
* -> FormatPostingsPositionsConsumer
* -> IMPL: FormatPostingsPositionsWriter
*/
int start = 0;
while (start < numAllFields)
{
FieldInfo fieldInfo = ((FreqProxTermsWriterPerField)allFields[start]).fieldInfo;
System.String fieldName = fieldInfo.name;
int end = start + 1;
while (end < numAllFields && ((FreqProxTermsWriterPerField)allFields[end]).fieldInfo.name.Equals(fieldName))
{
end++;
}
FreqProxTermsWriterPerField[] fields = new FreqProxTermsWriterPerField[end - start];
for (int i = start; i < end; i++)
{
fields[i - start] = (FreqProxTermsWriterPerField)allFields[i];
// Aggregate the storePayload as seen by the same
// field across multiple threads
fieldInfo.storePayloads |= fields[i - start].hasPayloads;
}
// If this field has postings then add them to the
// segment
AppendPostings(fields, consumer);
for (int i = 0; i < fields.Length; i++)
{
TermsHashPerField perField = fields[i].termsHashPerField;
int numPostings = perField.numPostings;
perField.Reset();
perField.ShrinkHash(numPostings);
fields[i].Reset();
}
start = end;
}
foreach (KeyValuePair <TermsHashConsumerPerThread, IList <TermsHashConsumerPerField> > entry in threadsAndFields)
{
FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread)entry.Key;
perThread.termsHashPerThread.Reset(true);
}
consumer.Finish();
}
