public override int Merge(MergeState mergeState)
{
int docCount = 0;
// Used for bulk-reading raw bytes for stored fields
int[] rawDocLengths = new int[MAX_RAW_MERGE_DOCS];
int idx = 0;
foreach (AtomicReader reader in mergeState.Readers)
{
SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++];
Lucene40StoredFieldsReader matchingFieldsReader = null;
if (matchingSegmentReader != null)
{
StoredFieldsReader fieldsReader = matchingSegmentReader.FieldsReader;
// we can only bulk-copy if the matching reader is also a Lucene40FieldsReader
if (fieldsReader != null && fieldsReader is Lucene40StoredFieldsReader)
{
matchingFieldsReader = (Lucene40StoredFieldsReader)fieldsReader;
}
}
if (reader.LiveDocs != null)
{
docCount += CopyFieldsWithDeletions(mergeState, reader, matchingFieldsReader, rawDocLengths);
}
else
{
docCount += CopyFieldsNoDeletions(mergeState, reader, matchingFieldsReader, rawDocLengths);
}
}
Finish(mergeState.FieldInfos, docCount);
return(docCount);
}
public override int Merge(MergeState mergeState)
{
int docCount = 0;
int idx = 0;
foreach (AtomicReader reader in mergeState.Readers)
{
SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++];
CompressingStoredFieldsReader matchingFieldsReader = null;
if (matchingSegmentReader != null)
{
StoredFieldsReader fieldsReader = matchingSegmentReader.FieldsReader;
// we can only bulk-copy if the matching reader is also a CompressingStoredFieldsReader
if (fieldsReader != null && fieldsReader is CompressingStoredFieldsReader compressingStoredFieldsReader)
{
matchingFieldsReader = compressingStoredFieldsReader;
}
}
int maxDoc = reader.MaxDoc;
IBits liveDocs = reader.LiveDocs;
if (matchingFieldsReader is null || matchingFieldsReader.Version != VERSION_CURRENT || matchingFieldsReader.CompressionMode != compressionMode || matchingFieldsReader.ChunkSize != chunkSize) // the way data is decompressed depends on the chunk size - means reader version is not the same as the writer version
{
// naive merge...
for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = NextLiveDoc(i + 1, liveDocs, maxDoc))
{
Document doc = reader.Document(i);
AddDocument(doc, mergeState.FieldInfos);
++docCount;
mergeState.CheckAbort.Work(300);
}
}
public override int Merge(MergeState mergeState)
{
int docCount = 0;
int idx = 0;
foreach (AtomicReader reader in mergeState.Readers)
{
SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++];
CompressingStoredFieldsReader matchingFieldsReader = null;
if (matchingSegmentReader != null)
{
StoredFieldsReader fieldsReader = matchingSegmentReader.FieldsReader;
// we can only bulk-copy if the matching reader is also a CompressingStoredFieldsReader
if (fieldsReader != null && fieldsReader is CompressingStoredFieldsReader compressingStoredFieldsReader)
{
matchingFieldsReader = compressingStoredFieldsReader;
}
}
int maxDoc = reader.MaxDoc;
IBits liveDocs = reader.LiveDocs;
if (matchingFieldsReader == null || matchingFieldsReader.Version != VERSION_CURRENT || matchingFieldsReader.CompressionMode != compressionMode || matchingFieldsReader.ChunkSize != chunkSize) // the way data is decompressed depends on the chunk size - means reader version is not the same as the writer version
{
// naive merge...
for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = NextLiveDoc(i + 1, liveDocs, maxDoc))
{
Document doc = reader.Document(i);
AddDocument(doc, mergeState.FieldInfos);
++docCount;
mergeState.CheckAbort.Work(300);
}
}
else
{
int docID = NextLiveDoc(0, liveDocs, maxDoc);
if (docID < maxDoc)
{
// not all docs were deleted
CompressingStoredFieldsReader.ChunkIterator it = matchingFieldsReader.GetChunkIterator(docID);
int[] startOffsets = Arrays.Empty <int>();
do
{
// go to the next chunk that contains docID
it.Next(docID);
// transform lengths into offsets
if (startOffsets.Length < it.chunkDocs)
{
startOffsets = new int[ArrayUtil.Oversize(it.chunkDocs, 4)];
}
for (int i = 1; i < it.chunkDocs; ++i)
{
startOffsets[i] = startOffsets[i - 1] + it.lengths[i - 1];
}
if (numBufferedDocs == 0 && startOffsets[it.chunkDocs - 1] < chunkSize && startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize && NextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) // no deletion in the chunk - chunk is large enough - chunk is small enough - starting a new chunk
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(docID == it.docBase);
}
// no need to decompress, just copy data
indexWriter.WriteIndex(it.chunkDocs, fieldsStream.Position); // LUCENENET specific: Renamed from getFilePointer() to match FileStream
WriteHeader(this.docBase, it.chunkDocs, it.numStoredFields, it.lengths);
it.CopyCompressedData(fieldsStream);
this.docBase += it.chunkDocs;
docID = NextLiveDoc(it.docBase + it.chunkDocs, liveDocs, maxDoc);
docCount += it.chunkDocs;
mergeState.CheckAbort.Work(300 * it.chunkDocs);
}
else
{
// decompress
it.Decompress();
if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.Length)
{
throw new CorruptIndexException("Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.Length);
}
// copy non-deleted docs
for (; docID < it.docBase + it.chunkDocs; docID = NextLiveDoc(docID + 1, liveDocs, maxDoc))
{
int diff = docID - it.docBase;
StartDocument(it.numStoredFields[diff]);
bufferedDocs.WriteBytes(it.bytes.Bytes, it.bytes.Offset + startOffsets[diff], it.lengths[diff]);
FinishDocument();
++docCount;
mergeState.CheckAbort.Work(300);
}
}
} while (docID < maxDoc);
it.CheckIntegrity();
}
}
}
Finish(mergeState.FieldInfos, docCount);
return(docCount);
}
internal AssertingStoredFieldsReader(StoredFieldsReader @in, int maxDoc)
{
this.@in = @in;
this.MaxDoc = maxDoc;
}
internal SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentCommitInfo si, IOContext context, int termsIndexDivisor)
{
fieldsReaderLocal = new AnonymousFieldsReaderLocal(this);
termVectorsLocal = new AnonymousTermVectorsLocal(this);
if (termsIndexDivisor == 0)
{
throw new ArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)");
}
Codec codec = si.Info.Codec;
Directory cfsDir; // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory.
bool success = false;
try
{
if (si.Info.UseCompoundFile)
{
cfsDir = cfsReader = new CompoundFileDirectory(dir, IndexFileNames.SegmentFileName(si.Info.Name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
}
else
{
cfsReader = null;
cfsDir = dir;
}
FieldInfos fieldInfos = owner.FieldInfos;
this.termsIndexDivisor = termsIndexDivisor;
PostingsFormat format = codec.PostingsFormat;
SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si.Info, fieldInfos, context, termsIndexDivisor);
// Ask codec for its Fields
fields = format.FieldsProducer(segmentReadState);
Debug.Assert(fields != null);
// ask codec for its Norms:
// TODO: since we don't write any norms file if there are no norms,
// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
if (fieldInfos.HasNorms)
{
normsProducer = codec.NormsFormat.NormsProducer(segmentReadState);
Debug.Assert(normsProducer != null);
}
else
{
normsProducer = null;
}
fieldsReaderOrig = si.Info.Codec.StoredFieldsFormat.FieldsReader(cfsDir, si.Info, fieldInfos, context);
if (fieldInfos.HasVectors) // open term vector files only as needed
{
termVectorsReaderOrig = si.Info.Codec.TermVectorsFormat.VectorsReader(cfsDir, si.Info, fieldInfos, context);
}
else
{
termVectorsReaderOrig = null;
}
success = true;
}
finally
{
if (!success)
{
DecRef();
}
}
}
internal AssertingStoredFieldsReader(StoredFieldsReader @in, int maxDoc)
{
this.@in = @in;
this.maxDoc = maxDoc;
}
internal SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentCommitInfo si, IOContext context, int termsIndexDivisor)
{
fieldsReaderLocal = new AnonymousFieldsReaderLocal(this);
termVectorsLocal = new AnonymousTermVectorsLocal(this);
if (termsIndexDivisor == 0)
{
throw new System.ArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)");
}
Codec codec = si.Info.Codec;
Directory cfsDir; // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory.
bool success = false;
try
{
if (si.Info.UseCompoundFile)
{
cfsDir = cfsReader = new CompoundFileDirectory(dir, IndexFileNames.SegmentFileName(si.Info.Name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
}
else
{
cfsReader = null;
cfsDir = dir;
}
FieldInfos fieldInfos = owner.FieldInfos;
this.termsIndexDivisor = termsIndexDivisor;
PostingsFormat format = codec.PostingsFormat;
SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si.Info, fieldInfos, context, termsIndexDivisor);
// Ask codec for its Fields
fields = format.FieldsProducer(segmentReadState);
Debug.Assert(fields != null);
// ask codec for its Norms:
// TODO: since we don't write any norms file if there are no norms,
// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
if (fieldInfos.HasNorms)
{
normsProducer = codec.NormsFormat.NormsProducer(segmentReadState);
Debug.Assert(normsProducer != null);
}
else
{
normsProducer = null;
}
// LUCENENET TODO: EXCEPTIONS Not sure why this catch block is swallowing AccessViolationException,
// because it didn't exist in Lucene. Is it really needed? AVE is for protected memory...could
// this be needed because we are using unchecked??
#if !NETSTANDARD
try
{
#endif
fieldsReaderOrig = si.Info.Codec.StoredFieldsFormat.FieldsReader(cfsDir, si.Info, fieldInfos, context);
#if !NETSTANDARD
}
#pragma warning disable 168
catch (System.AccessViolationException ave)
#pragma warning restore 168
{
}
#endif
if (fieldInfos.HasVectors) // open term vector files only as needed
{
termVectorsReaderOrig = si.Info.Codec.TermVectorsFormat.VectorsReader(cfsDir, si.Info, fieldInfos, context);
}
else
{
termVectorsReaderOrig = null;
}
success = true;
}
finally
{
if (!success)
{
DecRef();
}
}
}
