/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> IOException </throws>
private int MergeFields()
{
fieldInfos = new FieldInfos(); // merge field names
int docCount = 0;
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.INDEXED), false, false, false);
fieldInfos.Add(reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED), false);
}
fieldInfos.Write(directory, segment + ".fnm");
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
try
{
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
int maxDoc = reader.MaxDoc();
for (int j = 0; j < maxDoc; j++)
{
if (!reader.IsDeleted(j))
{
// skip deleted docs
fieldsWriter.AddDocument(reader.Document(j, fieldSelectorMerge));
docCount++;
}
}
}
}
finally
{
fieldsWriter.Close();
}
return(docCount);
}
/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int MergeFields()
{
if (!mergeDocStores)
{
// When we are not merging by doc stores, that means
// all segments were written as part of a single
// autoCommit=false IndexWriter session, so their field
// name -> number mapping are the same. So, we start
// with the fieldInfos of the last segment in this
// case, to keep that numbering.
SegmentReader sr = (SegmentReader) readers[readers.Count - 1];
fieldInfos = (FieldInfos) sr.core.fieldInfos.Clone();
}
else
{
fieldInfos = new FieldInfos(); // merge field names
}
for (System.Collections.IEnumerator iter = readers.GetEnumerator(); iter.MoveNext(); )
{
IndexReader reader = (IndexReader) iter.Current;
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader) reader;
FieldInfos readerFieldInfos = segmentReader.FieldInfos();
int numReaderFieldInfos = readerFieldInfos.Size();
for (int j = 0; j < numReaderFieldInfos; j++)
{
FieldInfo fi = readerFieldInfos.FieldInfo(j);
fieldInfos.Add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.HasNorms(fi.name), fi.storePayloads, fi.omitTermFreqAndPositions);
}
}
else
{
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR), true, false, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.OMIT_TERM_FREQ_AND_POSITIONS), false, false, false, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.INDEXED), false, false, false, false, false);
fieldInfos.Add(reader.GetFieldNames(FieldOption.UNINDEXED), false);
}
}
fieldInfos.Write(directory, segment + ".fnm");
int docCount = 0;
SetMatchingSegmentReaders();
if (mergeDocStores)
{
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
// merge field values
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
try
{
int idx = 0;
for (System.Collections.IEnumerator iter = readers.GetEnumerator(); iter.MoveNext(); )
{
IndexReader reader = (IndexReader) iter.Current;
SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++];
FieldsReader matchingFieldsReader = null;
if (matchingSegmentReader != null)
{
FieldsReader fieldsReader = matchingSegmentReader.GetFieldsReader();
if (fieldsReader != null && fieldsReader.CanReadRawDocs())
{
matchingFieldsReader = fieldsReader;
}
}
if (reader.HasDeletions())
{
docCount += CopyFieldsWithDeletions(fieldSelectorMerge, fieldsWriter, reader, matchingFieldsReader);
}
else
{
docCount += CopyFieldsNoDeletions(fieldSelectorMerge, fieldsWriter, reader, matchingFieldsReader);
}
}
}
finally
{
fieldsWriter.Close();
}
System.String fileName = segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION;
long fdxFileLength = directory.FileLength(fileName);
if (4 + ((long) docCount) * 8 != fdxFileLength)
// This is most likely a bug in Sun JRE 1.6.0_04/_05;
// we detect that the bug has struck, here, and
// throw an exception to prevent the corruption from
// entering the index. See LUCENE-1282 for
// details.
throw new System.SystemException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.FileExists(fileName) + "; now aborting this merge to prevent index corruption");
}
// If we are skipping the doc stores, that means there
// are no deletions in any of these segments, so we
// just sum numDocs() of each segment to get total docCount
else
{
for (System.Collections.IEnumerator iter = readers.GetEnumerator(); iter.MoveNext(); )
{
docCount += ((IndexReader) iter.Current).NumDocs();
}
}
return docCount;
}
/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int MergeFields()
{
if (!mergeDocStores)
{
// When we are not merging by doc stores, that means
// all segments were written as part of a single
// autoCommit=false IndexWriter session, so their field
// name -> number mapping are the same. So, we start
// with the fieldInfos of the last segment in this
// case, to keep that numbering.
SegmentReader sr = (SegmentReader)readers[readers.Count - 1];
fieldInfos = (FieldInfos)sr.fieldInfos.Clone();
}
else
{
fieldInfos = new FieldInfos(); // merge field names
}
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader)reader;
for (int j = 0; j < segmentReader.GetFieldInfos().Size(); j++)
{
FieldInfo fi = segmentReader.GetFieldInfos().FieldInfo(j);
fieldInfos.Add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.HasNorms(fi.name), fi.storePayloads);
}
}
else
{
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false);
fieldInfos.Add(reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED), false);
}
}
fieldInfos.Write(directory, segment + ".fnm");
int docCount = 0;
if (mergeDocStores)
{
// If the i'th reader is a SegmentReader and has
// identical fieldName -> number mapping, then this
// array will be non-null at position i:
SegmentReader[] matchingSegmentReaders = new SegmentReader[readers.Count];
// If this reader is a SegmentReader, and all of its
// field name -> number mappings match the "merged"
// FieldInfos, then we can do a bulk copy of the
// stored fields:
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader)reader;
bool same = true;
FieldInfos segmentFieldInfos = segmentReader.GetFieldInfos();
for (int j = 0; same && j < segmentFieldInfos.Size(); j++)
{
same = fieldInfos.FieldName(j).Equals(segmentFieldInfos.FieldName(j));
}
if (same)
{
matchingSegmentReaders[i] = segmentReader;
}
}
}
// Used for bulk-reading raw bytes for stored fields
int[] rawDocLengths = new int[MAX_RAW_MERGE_DOCS];
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
// merge field values
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
try
{
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
SegmentReader matchingSegmentReader = matchingSegmentReaders[i];
FieldsReader matchingFieldsReader;
if (matchingSegmentReader != null)
{
matchingFieldsReader = matchingSegmentReader.GetFieldsReader();
}
else
{
matchingFieldsReader = null;
}
int maxDoc = reader.MaxDoc();
for (int j = 0; j < maxDoc;)
{
if (!reader.IsDeleted(j))
{
// skip deleted docs
if (matchingSegmentReader != null)
{
// We can optimize this case (doing a bulk
// byte copy) since the field numbers are
// identical
int start = j;
int numDocs = 0;
do
{
j++;
numDocs++;
}while (j < maxDoc && !matchingSegmentReader.IsDeleted(j) && numDocs < MAX_RAW_MERGE_DOCS);
IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, start, numDocs);
fieldsWriter.AddRawDocuments(stream, rawDocLengths, numDocs);
docCount += numDocs;
if (checkAbort != null)
{
checkAbort.Work(300 * numDocs);
}
}
else
{
fieldsWriter.AddDocument(reader.Document(j, fieldSelectorMerge));
j++;
docCount++;
if (checkAbort != null)
{
checkAbort.Work(300);
}
}
}
else
{
j++;
}
}
}
}
finally
{
fieldsWriter.Close();
}
}
// If we are skipping the doc stores, that means there
// are no deletions in any of these segments, so we
// just sum numDocs() of each segment to get total docCount
else
{
for (int i = 0; i < readers.Count; i++)
{
docCount += ((IndexReader)readers[i]).NumDocs();
}
}
return(docCount);
}
/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int MergeFields()
{
if (!mergeDocStores)
{
// When we are not merging by doc stores, that means
// all segments were written as part of a single
// autoCommit=false IndexWriter session, so their field
// name -> number mapping are the same. So, we start
// with the fieldInfos of the last segment in this
// case, to keep that numbering.
SegmentReader sr = (SegmentReader)readers[readers.Count - 1];
fieldInfos = (FieldInfos)sr.core.fieldInfos.Clone();
}
else
{
fieldInfos = new FieldInfos(); // merge field names
}
for (System.Collections.IEnumerator iter = readers.GetEnumerator(); iter.MoveNext();)
{
IndexReader reader = (IndexReader)iter.Current;
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader)reader;
FieldInfos readerFieldInfos = segmentReader.FieldInfos();
int numReaderFieldInfos = readerFieldInfos.Size();
for (int j = 0; j < numReaderFieldInfos; j++)
{
FieldInfo fi = readerFieldInfos.FieldInfo(j);
fieldInfos.Add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.HasNorms(fi.name), fi.storePayloads, fi.omitTermFreqAndPositions);
}
}
else
{
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.TERMVECTOR), true, false, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.OMIT_TERM_FREQ_AND_POSITIONS), false, false, false, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(FieldOption.INDEXED), false, false, false, false, false);
fieldInfos.Add(reader.GetFieldNames(FieldOption.UNINDEXED), false);
}
}
fieldInfos.Write(directory, segment + ".fnm");
int docCount = 0;
SetMatchingSegmentReaders();
if (mergeDocStores)
{
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
// merge field values
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
try
{
int idx = 0;
for (System.Collections.IEnumerator iter = readers.GetEnumerator(); iter.MoveNext();)
{
IndexReader reader = (IndexReader)iter.Current;
SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++];
FieldsReader matchingFieldsReader = null;
if (matchingSegmentReader != null)
{
FieldsReader fieldsReader = matchingSegmentReader.GetFieldsReader();
if (fieldsReader != null && fieldsReader.CanReadRawDocs())
{
matchingFieldsReader = fieldsReader;
}
}
if (reader.HasDeletions())
{
docCount += CopyFieldsWithDeletions(fieldSelectorMerge, fieldsWriter, reader, matchingFieldsReader);
}
else
{
docCount += CopyFieldsNoDeletions(fieldSelectorMerge, fieldsWriter, reader, matchingFieldsReader);
}
}
}
finally
{
fieldsWriter.Close();
}
System.String fileName = segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION;
long fdxFileLength = directory.FileLength(fileName);
if (4 + ((long)docCount) * 8 != fdxFileLength)
{
// This is most likely a bug in Sun JRE 1.6.0_04/_05;
// we detect that the bug has struck, here, and
// throw an exception to prevent the corruption from
// entering the index. See LUCENE-1282 for
// details.
throw new System.SystemException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.FileExists(fileName) + "; now aborting this merge to prevent index corruption");
}
}
// If we are skipping the doc stores, that means there
// are no deletions in any of these segments, so we
// just sum numDocs() of each segment to get total docCount
else
{
for (System.Collections.IEnumerator iter = readers.GetEnumerator(); iter.MoveNext();)
{
docCount += ((IndexReader)iter.Current).NumDocs();
}
}
return(docCount);
}
/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int MergeFields()
{
if (!mergeDocStores)
{
// When we are not merging by doc stores, that means
// all segments were written as part of a single
// autoCommit=false IndexWriter session, so their field
// name -> number mapping are the same. So, we start
// with the fieldInfos of the last segment in this
// case, to keep that numbering.
SegmentReader sr = (SegmentReader) readers[readers.Count - 1];
fieldInfos = (FieldInfos) sr.fieldInfos.Clone();
}
else
{
fieldInfos = new FieldInfos(); // merge field names
}
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader) readers[i];
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader) reader;
for (int j = 0; j < segmentReader.GetFieldInfos().Size(); j++)
{
FieldInfo fi = segmentReader.GetFieldInfos().FieldInfo(j);
fieldInfos.Add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.HasNorms(fi.name), fi.storePayloads, fi.omitTf);
}
}
else
{
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.OMIT_TF), false, false, false, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false, false);
fieldInfos.Add(reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED), false);
}
}
fieldInfos.Write(directory, segment + ".fnm");
int docCount = 0;
SetMatchingSegmentReaders();
if (mergeDocStores)
{
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
// merge field values
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
try
{
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
SegmentReader matchingSegmentReader = matchingSegmentReaders[i];
FieldsReader matchingFieldsReader;
bool hasMatchingReader;
if (matchingSegmentReader != null)
{
FieldsReader fieldsReader = matchingSegmentReader.GetFieldsReader();
if (fieldsReader != null && !fieldsReader.CanReadRawDocs())
{
matchingFieldsReader = null;
hasMatchingReader = false;
}
else
{
matchingFieldsReader = fieldsReader;
hasMatchingReader = true;
}
}
else
{
hasMatchingReader = false;
matchingFieldsReader = null;
}
int maxDoc = reader.MaxDoc();
bool hasDeletions = reader.HasDeletions();
for (int j = 0; j < maxDoc; )
{
if (!hasDeletions || !reader.IsDeleted(j))
{ // skip deleted docs
if (hasMatchingReader)
{
// We can optimize this case (doing a bulk
// byte copy) since the field numbers are
// identical
int start = j;
int numDocs = 0;
do
{
j++;
numDocs++;
if (j >= maxDoc)
break;
if (hasDeletions && matchingSegmentReader.IsDeleted(j))
{
j++;
break;
}
} while (numDocs < MAX_RAW_MERGE_DOCS);
IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, start, numDocs);
fieldsWriter.AddRawDocuments(stream, rawDocLengths, numDocs);
docCount += numDocs;
if (checkAbort != null)
checkAbort.Work(300 * numDocs);
}
else
{
// NOTE: it's very important to first assign
// to doc then pass it to
// termVectorsWriter.addAllDocVectors; see
// LUCENE-1282
Document doc = reader.Document(j, fieldSelectorMerge);
fieldsWriter.AddDocument(doc);
j++;
docCount++;
if (checkAbort != null)
checkAbort.Work(300);
}
}
else
j++;
}
}
}
finally
{
fieldsWriter.Close();
}
long fdxFileLength = directory.FileLength(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION);
// {{dougsale-2.4.0}
// this shouldn't be a problem for us - if it is,
// then it's not a JRE bug...
//if (4+docCount*8 != fdxFileLength)
// // This is most likely a bug in Sun JRE 1.6.0_04/_05;
// // we detect that the bug has struck, here, and
// // throw an exception to prevent the corruption from
// // entering the index. See LUCENE-1282 for
// // details.
// throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + "; now aborting this merge to prevent index corruption");
}
else
// If we are skipping the doc stores, that means there
// are no deletions in any of these segments, so we
// just sum numDocs() of each segment to get total docCount
for (int i = 0; i < readers.Count; i++)
docCount += ((IndexReader)readers[i]).NumDocs();
return docCount;
}
/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int MergeFields()
{
if (!mergeDocStores)
{
// When we are not merging by doc stores, that means
// all segments were written as part of a single
// autoCommit=false IndexWriter session, so their field
// name -> number mapping are the same. So, we start
// with the fieldInfos of the last segment in this
// case, to keep that numbering.
SegmentReader sr = (SegmentReader)readers[readers.Count - 1];
fieldInfos = (FieldInfos)sr.fieldInfos.Clone();
}
else
{
fieldInfos = new FieldInfos(); // merge field names
}
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader)reader;
for (int j = 0; j < segmentReader.GetFieldInfos().Size(); j++)
{
FieldInfo fi = segmentReader.GetFieldInfos().FieldInfo(j);
fieldInfos.Add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.HasNorms(fi.name), fi.storePayloads, fi.omitTf);
}
}
else
{
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.OMIT_TF), false, false, false, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false, false);
fieldInfos.Add(reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED), false);
}
}
fieldInfos.Write(directory, segment + ".fnm");
int docCount = 0;
SetMatchingSegmentReaders();
if (mergeDocStores)
{
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
// merge field values
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
try
{
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader)readers[i];
SegmentReader matchingSegmentReader = matchingSegmentReaders[i];
FieldsReader matchingFieldsReader;
bool hasMatchingReader;
if (matchingSegmentReader != null)
{
FieldsReader fieldsReader = matchingSegmentReader.GetFieldsReader();
if (fieldsReader != null && !fieldsReader.CanReadRawDocs())
{
matchingFieldsReader = null;
hasMatchingReader = false;
}
else
{
matchingFieldsReader = fieldsReader;
hasMatchingReader = true;
}
}
else
{
hasMatchingReader = false;
matchingFieldsReader = null;
}
int maxDoc = reader.MaxDoc();
bool hasDeletions = reader.HasDeletions();
for (int j = 0; j < maxDoc;)
{
if (!hasDeletions || !reader.IsDeleted(j))
{ // skip deleted docs
if (hasMatchingReader)
{
// We can optimize this case (doing a bulk
// byte copy) since the field numbers are
// identical
int start = j;
int numDocs = 0;
do
{
j++;
numDocs++;
if (j >= maxDoc)
{
break;
}
if (hasDeletions && matchingSegmentReader.IsDeleted(j))
{
j++;
break;
}
} while (numDocs < MAX_RAW_MERGE_DOCS);
IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, start, numDocs);
fieldsWriter.AddRawDocuments(stream, rawDocLengths, numDocs);
docCount += numDocs;
if (checkAbort != null)
{
checkAbort.Work(300 * numDocs);
}
}
else
{
// NOTE: it's very important to first assign
// to doc then pass it to
// termVectorsWriter.addAllDocVectors; see
// LUCENE-1282
Document doc = reader.Document(j, fieldSelectorMerge);
fieldsWriter.AddDocument(doc);
j++;
docCount++;
if (checkAbort != null)
{
checkAbort.Work(300);
}
}
}
else
{
j++;
}
}
}
}
finally
{
fieldsWriter.Close();
}
long fdxFileLength = directory.FileLength(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION);
// {{dougsale-2.4.0}
// this shouldn't be a problem for us - if it is,
// then it's not a JRE bug...
//if (4+docCount*8 != fdxFileLength)
// // This is most likely a bug in Sun JRE 1.6.0_04/_05;
// // we detect that the bug has struck, here, and
// // throw an exception to prevent the corruption from
// // entering the index. See LUCENE-1282 for
// // details.
// throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + "; now aborting this merge to prevent index corruption");
}
else
{
// If we are skipping the doc stores, that means there
// are no deletions in any of these segments, so we
// just sum numDocs() of each segment to get total docCount
for (int i = 0; i < readers.Count; i++)
{
docCount += ((IndexReader)readers[i]).NumDocs();
}
}
return(docCount);
}
/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int MergeFields()
{
if (!mergeDocStores)
{
// When we are not merging by doc stores, that means
// all segments were written as part of a single
// autoCommit=false IndexWriter session, so their field
// name -> number mapping are the same. So, we start
// with the fieldInfos of the last segment in this
// case, to keep that numbering.
SegmentReader sr = (SegmentReader) readers[readers.Count - 1];
fieldInfos = (FieldInfos) sr.fieldInfos.Clone();
}
else
{
fieldInfos = new FieldInfos(); // merge field names
}
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader) readers[i];
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader) reader;
for (int j = 0; j < segmentReader.GetFieldInfos().Size(); j++)
{
FieldInfo fi = segmentReader.GetFieldInfos().FieldInfo(j);
fieldInfos.Add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.HasNorms(fi.name), fi.storePayloads);
}
}
else
{
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false);
fieldInfos.Add(reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED), false);
}
}
fieldInfos.Write(directory, segment + ".fnm");
int docCount = 0;
if (mergeDocStores)
{
// If the i'th reader is a SegmentReader and has
// identical fieldName -> number mapping, then this
// array will be non-null at position i:
SegmentReader[] matchingSegmentReaders = new SegmentReader[readers.Count];
// If this reader is a SegmentReader, and all of its
// field name -> number mappings match the "merged"
// FieldInfos, then we can do a bulk copy of the
// stored fields:
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader) readers[i];
if (reader is SegmentReader)
{
SegmentReader segmentReader = (SegmentReader) reader;
bool same = true;
FieldInfos segmentFieldInfos = segmentReader.GetFieldInfos();
for (int j = 0; same && j < segmentFieldInfos.Size(); j++)
same = fieldInfos.FieldName(j).Equals(segmentFieldInfos.FieldName(j));
if (same)
{
matchingSegmentReaders[i] = segmentReader;
}
}
}
// Used for bulk-reading raw bytes for stored fields
int[] rawDocLengths = new int[MAX_RAW_MERGE_DOCS];
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
// merge field values
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
try
{
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader) readers[i];
SegmentReader matchingSegmentReader = matchingSegmentReaders[i];
FieldsReader matchingFieldsReader;
if (matchingSegmentReader != null)
matchingFieldsReader = matchingSegmentReader.GetFieldsReader();
else
matchingFieldsReader = null;
int maxDoc = reader.MaxDoc();
for (int j = 0; j < maxDoc; )
{
if (!reader.IsDeleted(j))
{
// skip deleted docs
if (matchingSegmentReader != null)
{
// We can optimize this case (doing a bulk
// byte copy) since the field numbers are
// identical
int start = j;
int numDocs = 0;
do
{
j++;
numDocs++;
}
while (j < maxDoc && !matchingSegmentReader.IsDeleted(j) && numDocs < MAX_RAW_MERGE_DOCS);
IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, start, numDocs);
fieldsWriter.AddRawDocuments(stream, rawDocLengths, numDocs);
docCount += numDocs;
if (checkAbort != null)
checkAbort.Work(300 * numDocs);
}
else
{
fieldsWriter.AddDocument(reader.Document(j, fieldSelectorMerge));
j++;
docCount++;
if (checkAbort != null)
checkAbort.Work(300);
}
}
else
j++;
}
}
}
finally
{
fieldsWriter.Close();
}
System.Diagnostics.Debug.Assert(docCount*8 == directory.FileLength(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION),
"after MergeFields: fdx size mismatch: " + docCount + " docs vs " +
directory.FileLength(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION) +
" length in bytes of " + segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION);
}
// If we are skipping the doc stores, that means there
// are no deletions in any of these segments, so we
// just sum numDocs() of each segment to get total docCount
else
for (int i = 0; i < readers.Count; i++)
docCount += ((IndexReader) readers[i]).NumDocs();
return docCount;
}
/// <summary> </summary>
/// <returns> The number of documents in all of the readers
/// </returns>
/// <throws> IOException </throws>
private int MergeFields()
{
fieldInfos = new FieldInfos(); // merge field names
int docCount = 0;
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader) readers[i];
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false);
AddIndexed(reader, fieldInfos, reader.GetFieldNames(IndexReader.FieldOption.INDEXED), false, false, false);
fieldInfos.Add(reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED), false);
}
fieldInfos.Write(directory, segment + ".fnm");
FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
// in merge mode, we use this FieldSelector
FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
try
{
for (int i = 0; i < readers.Count; i++)
{
IndexReader reader = (IndexReader) readers[i];
int maxDoc = reader.MaxDoc();
for (int j = 0; j < maxDoc; j++)
if (!reader.IsDeleted(j))
{
// skip deleted docs
fieldsWriter.AddDocument(reader.Document(j, fieldSelectorMerge));
docCount++;
}
}
}
finally
{
fieldsWriter.Close();
}
return docCount;
}