public override int Merge(MergeState mergeState) { int docCount = 0; int idx = 0; foreach (AtomicReader reader in mergeState.Readers) { SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++]; CompressingTermVectorsReader matchingVectorsReader = null; if (matchingSegmentReader != null) { TermVectorsReader vectorsReader = matchingSegmentReader.TermVectorsReader; // we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader if (vectorsReader != null && vectorsReader is CompressingTermVectorsReader compressingTermVectorsReader) { matchingVectorsReader = compressingTermVectorsReader; } } int maxDoc = reader.MaxDoc; IBits liveDocs = reader.LiveDocs; if (matchingVectorsReader is null || matchingVectorsReader.Version != VERSION_CURRENT || matchingVectorsReader.CompressionMode != compressionMode || matchingVectorsReader.ChunkSize != chunkSize || matchingVectorsReader.PackedInt32sVersion != PackedInt32s.VERSION_CURRENT) { // naive merge... for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = NextLiveDoc(i + 1, liveDocs, maxDoc)) { Fields vectors = reader.GetTermVectors(i); AddAllDocVectors(vectors, mergeState); ++docCount; mergeState.CheckAbort.Work(300); } }
// used by clone private CompressingTermVectorsReader(CompressingTermVectorsReader reader) { this.fieldInfos = reader.fieldInfos; this.vectorsStream = (IndexInput)reader.vectorsStream.Clone(); this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone(); this.packedIntsVersion = reader.packedIntsVersion; this.compressionMode = reader.compressionMode; this.decompressor = (Decompressor)reader.decompressor.Clone(); this.chunkSize = reader.chunkSize; this.numDocs = reader.numDocs; this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0); this.version = reader.version; this.closed = false; }
// used by clone private CompressingTermVectorsReader(CompressingTermVectorsReader reader) { this.fieldInfos = reader.fieldInfos; this.vectorsStream = (IndexInput)reader.vectorsStream.Clone(); this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone(); this.packedIntsVersion = reader.packedIntsVersion; this.compressionMode = reader.compressionMode; this.decompressor = (Decompressor)reader.decompressor.Clone(); this.chunkSize = reader.chunkSize; this.numDocs = reader.numDocs; this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0); this.version = reader.version; this.closed = false; }
internal TVTerms(CompressingTermVectorsReader outerInstance, int numTerms, int flags, int[] prefixLengths, int[] suffixLengths, int[] termFreqs, int[] positionIndex, int[] positions, int[] startOffsets, int[] lengths, int[] payloadIndex, BytesRef payloadBytes, BytesRef termBytes) { this.outerInstance = outerInstance; this.numTerms = numTerms; this.flags = flags; this.prefixLengths = prefixLengths; this.suffixLengths = suffixLengths; this.termFreqs = termFreqs; this.positionIndex = positionIndex; this.positions = positions; this.startOffsets = startOffsets; this.lengths = lengths; this.payloadIndex = payloadIndex; this.payloadBytes = payloadBytes; this.termBytes = termBytes; }
public TVFields(CompressingTermVectorsReader outerInstance, int[] fieldNums, int[] fieldFlags, int[] fieldNumOffs, int[] numTerms, int[] fieldLengths, int[][] prefixLengths, int[][] suffixLengths, int[][] termFreqs, int[][] positionIndex, int[][] positions, int[][] startOffsets, int[][] lengths, BytesRef payloadBytes, int[][] payloadIndex, BytesRef suffixBytes) { this.outerInstance = outerInstance; this.fieldNums = fieldNums; this.fieldFlags = fieldFlags; this.fieldNumOffs = fieldNumOffs; this.numTerms = numTerms; this.fieldLengths = fieldLengths; this.prefixLengths = prefixLengths; this.suffixLengths = suffixLengths; this.termFreqs = termFreqs; this.positionIndex = positionIndex; this.positions = positions; this.startOffsets = startOffsets; this.lengths = lengths; this.payloadBytes = payloadBytes; this.payloadIndex = payloadIndex; this.suffixBytes = suffixBytes; }
public override int Merge(MergeState mergeState) { int docCount = 0; int idx = 0; foreach (AtomicReader reader in mergeState.Readers) { SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++]; CompressingTermVectorsReader matchingVectorsReader = null; if (matchingSegmentReader != null) { TermVectorsReader vectorsReader = matchingSegmentReader.TermVectorsReader; // we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader if (vectorsReader != null && vectorsReader is CompressingTermVectorsReader) { matchingVectorsReader = (CompressingTermVectorsReader)vectorsReader; } } int maxDoc = reader.MaxDoc; Bits liveDocs = reader.LiveDocs; if (matchingVectorsReader == null || matchingVectorsReader.Version != VERSION_CURRENT || matchingVectorsReader.CompressionMode != CompressionMode || matchingVectorsReader.ChunkSize != ChunkSize || matchingVectorsReader.PackedIntsVersion != PackedInts.VERSION_CURRENT) { // naive merge... for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = NextLiveDoc(i + 1, liveDocs, maxDoc)) { Fields vectors = reader.GetTermVectors(i); AddAllDocVectors(vectors, mergeState); ++docCount; mergeState.checkAbort.Work(300); } } else { CompressingStoredFieldsIndexReader index = matchingVectorsReader.Index; IndexInput vectorsStreamOrig = matchingVectorsReader.VectorsStream; vectorsStreamOrig.Seek(0); ChecksumIndexInput vectorsStream = new BufferedChecksumIndexInput((IndexInput)vectorsStreamOrig.Clone()); for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc;) { // We make sure to move the checksum input in any case, otherwise the final // integrity check might need to read the whole file a second time long startPointer = index.GetStartPointer(i); if (startPointer > vectorsStream.FilePointer) { vectorsStream.Seek(startPointer); } if ((PendingDocs.Count == 0) && (i == 0 || index.GetStartPointer(i - 1) < startPointer)) // start of a chunk { int docBase = vectorsStream.ReadVInt(); int chunkDocs = vectorsStream.ReadVInt(); Debug.Assert(docBase + chunkDocs <= matchingSegmentReader.MaxDoc); if (docBase + chunkDocs < matchingSegmentReader.MaxDoc && NextDeletedDoc(docBase, liveDocs, docBase + chunkDocs) == docBase + chunkDocs) { long chunkEnd = index.GetStartPointer(docBase + chunkDocs); long chunkLength = chunkEnd - vectorsStream.FilePointer; IndexWriter.WriteIndex(chunkDocs, this.VectorsStream.FilePointer); this.VectorsStream.WriteVInt(docCount); this.VectorsStream.WriteVInt(chunkDocs); this.VectorsStream.CopyBytes(vectorsStream, chunkLength); docCount += chunkDocs; this.NumDocs += chunkDocs; mergeState.checkAbort.Work(300 * chunkDocs); i = NextLiveDoc(docBase + chunkDocs, liveDocs, maxDoc); } else { for (; i < docBase + chunkDocs; i = NextLiveDoc(i + 1, liveDocs, maxDoc)) { Fields vectors = reader.GetTermVectors(i); AddAllDocVectors(vectors, mergeState); ++docCount; mergeState.checkAbort.Work(300); } } } else { Fields vectors = reader.GetTermVectors(i); AddAllDocVectors(vectors, mergeState); ++docCount; mergeState.checkAbort.Work(300); i = NextLiveDoc(i + 1, liveDocs, maxDoc); } } vectorsStream.Seek(vectorsStream.Length() - CodecUtil.FooterLength()); CodecUtil.CheckFooter(vectorsStream); } } Finish(mergeState.FieldInfos, docCount); return(docCount); }
internal TVTerms(CompressingTermVectorsReader outerInstance, int numTerms, int flags, int[] prefixLengths, int[] suffixLengths, int[] termFreqs, int[] positionIndex, int[] positions, int[] startOffsets, int[] lengths, int[] payloadIndex, BytesRef payloadBytes, BytesRef termBytes) { this.OuterInstance = outerInstance; this.NumTerms = numTerms; this.Flags = flags; this.PrefixLengths = prefixLengths; this.SuffixLengths = suffixLengths; this.TermFreqs = termFreqs; this.PositionIndex = positionIndex; this.Positions = positions; this.StartOffsets = startOffsets; this.Lengths = lengths; this.PayloadIndex = payloadIndex; this.PayloadBytes = payloadBytes; this.TermBytes = termBytes; }
public TVFields(CompressingTermVectorsReader outerInstance, int[] fieldNums, int[] fieldFlags, int[] fieldNumOffs, int[] numTerms, int[] fieldLengths, int[][] prefixLengths, int[][] suffixLengths, int[][] termFreqs, int[][] positionIndex, int[][] positions, int[][] startOffsets, int[][] lengths, BytesRef payloadBytes, int[][] payloadIndex, BytesRef suffixBytes) { this.OuterInstance = outerInstance; this.FieldNums = fieldNums; this.FieldFlags = fieldFlags; this.FieldNumOffs = fieldNumOffs; this.NumTerms = numTerms; this.FieldLengths = fieldLengths; this.PrefixLengths = prefixLengths; this.SuffixLengths = suffixLengths; this.TermFreqs = termFreqs; this.PositionIndex = positionIndex; this.Positions = positions; this.StartOffsets = startOffsets; this.Lengths = lengths; this.PayloadBytes = payloadBytes; this.PayloadIndex = payloadIndex; this.SuffixBytes = suffixBytes; }