/// <summary>
/// Sole constructor. </summary>
public CompressingTermVectorsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.compressionMode = compressionMode;
string segment = si.Name;
bool success = false;
fieldInfos = fn;
numDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
// Load the index into memory
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION);
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingTermVectorsWriter.CODEC_SFX_IDX;
version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.GetFilePointer());
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
if (version >= CompressingTermVectorsWriter.VERSION_CHECKSUM)
{
indexStream.ReadVInt64(); // the end of the data file
CodecUtil.CheckFooter(indexStream);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(indexStream);
#pragma warning restore 612, 618
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
string vectorsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_EXTENSION);
vectorsStream = d.OpenInput(vectorsStreamFN, context);
string codecNameDat = formatName + CompressingTermVectorsWriter.CODEC_SFX_DAT;
int version2 = CodecUtil.CheckHeader(vectorsStream, codecNameDat, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
if (version != version2)
{
throw new Exception("Version mismatch between stored fields index and data: " + version + " != " + version2);
}
Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == vectorsStream.GetFilePointer());
packedIntsVersion = vectorsStream.ReadVInt32();
chunkSize = vectorsStream.ReadVInt32();
decompressor = compressionMode.NewDecompressor();
this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this, indexStream);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public CompressingTermVectorsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.compressionMode = compressionMode;
string segment = si.Name;
bool success = false;
fieldInfos = fn;
numDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
// Load the index into memory
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION);
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingTermVectorsWriter.CODEC_SFX_IDX;
version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.FilePointer);
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
if (version >= CompressingTermVectorsWriter.VERSION_CHECKSUM)
{
indexStream.ReadVLong(); // the end of the data file
CodecUtil.CheckFooter(indexStream);
}
else
{
CodecUtil.CheckEOF(indexStream);
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
string vectorsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_EXTENSION);
vectorsStream = d.OpenInput(vectorsStreamFN, context);
string codecNameDat = formatName + CompressingTermVectorsWriter.CODEC_SFX_DAT;
int version2 = CodecUtil.CheckHeader(vectorsStream, codecNameDat, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
if (version != version2)
{
throw new Exception("Version mismatch between stored fields index and data: " + version + " != " + version2);
}
Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == vectorsStream.FilePointer);
packedIntsVersion = vectorsStream.ReadVInt();
chunkSize = vectorsStream.ReadVInt();
decompressor = compressionMode.NewDecompressor();
this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(this, indexStream);
}
}
}
// used by clone
private CompressingTermVectorsReader(CompressingTermVectorsReader reader)
{
this.fieldInfos = reader.fieldInfos;
this.vectorsStream = (IndexInput)reader.vectorsStream.Clone();
this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone();
this.packedIntsVersion = reader.packedIntsVersion;
this.compressionMode = reader.compressionMode;
this.decompressor = (Decompressor)reader.decompressor.Clone();
this.chunkSize = reader.chunkSize;
this.numDocs = reader.numDocs;
this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
this.version = reader.version;
this.closed = false;
}
// used by clone
private CompressingTermVectorsReader(CompressingTermVectorsReader reader)
{
this.fieldInfos = reader.fieldInfos;
this.vectorsStream = (IndexInput)reader.vectorsStream.Clone();
this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone();
this.packedIntsVersion = reader.packedIntsVersion;
this.compressionMode = reader.compressionMode;
this.decompressor = (Decompressor)reader.decompressor.Clone();
this.chunkSize = reader.chunkSize;
this.numDocs = reader.numDocs;
this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
this.version = reader.version;
this.closed = false;
}
// used by clone
private CompressingStoredFieldsReader(CompressingStoredFieldsReader reader)
{
this.version = reader.version;
this.fieldInfos = reader.fieldInfos;
this.fieldsStream = (IndexInput)reader.fieldsStream.Clone();
this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone();
this.maxPointer = reader.maxPointer;
this.chunkSize = reader.chunkSize;
this.packedIntsVersion = reader.packedIntsVersion;
this.compressionMode = reader.compressionMode;
this.decompressor = (Decompressor)reader.decompressor.Clone();
this.numDocs = reader.numDocs;
this.bytes = new BytesRef(reader.bytes.Bytes.Length);
this.closed = false;
}
// used by clone
private CompressingStoredFieldsReader(CompressingStoredFieldsReader reader)
{
this.Version_Renamed = reader.Version_Renamed;
this.FieldInfos = reader.FieldInfos;
this.FieldsStream = (IndexInput)reader.FieldsStream.Clone();
this.IndexReader = (CompressingStoredFieldsIndexReader)reader.IndexReader.Clone();
this.MaxPointer = reader.MaxPointer;
this.ChunkSize_Renamed = reader.ChunkSize_Renamed;
this.PackedIntsVersion = reader.PackedIntsVersion;
this.CompressionMode_Renamed = reader.CompressionMode_Renamed;
this.Decompressor = (Decompressor)reader.Decompressor.Clone();
this.NumDocs = reader.NumDocs;
this.Bytes = new BytesRef(reader.Bytes.Bytes.Length);
this.Closed = false;
}
// used by clone
private CompressingStoredFieldsReader(CompressingStoredFieldsReader reader)
{
this.Version_Renamed = reader.Version_Renamed;
this.FieldInfos = reader.FieldInfos;
this.FieldsStream = (IndexInput)reader.FieldsStream.Clone();
this.IndexReader = (CompressingStoredFieldsIndexReader)reader.IndexReader.Clone();
this.MaxPointer = reader.MaxPointer;
this.ChunkSize_Renamed = reader.ChunkSize_Renamed;
this.PackedIntsVersion = reader.PackedIntsVersion;
this.CompressionMode_Renamed = reader.CompressionMode_Renamed;
this.Decompressor = (Decompressor)reader.Decompressor.Clone();
this.NumDocs = reader.NumDocs;
this.Bytes = new BytesRef(reader.Bytes.Bytes.Length);
this.Closed = false;
}
public override int Merge(MergeState mergeState)
{
int docCount = 0;
int idx = 0;
foreach (AtomicReader reader in mergeState.Readers)
{
SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++];
CompressingTermVectorsReader matchingVectorsReader = null;
if (matchingSegmentReader != null)
{
TermVectorsReader vectorsReader = matchingSegmentReader.TermVectorsReader;
// we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader
if (vectorsReader != null && vectorsReader is CompressingTermVectorsReader)
{
matchingVectorsReader = (CompressingTermVectorsReader)vectorsReader;
}
}
int maxDoc = reader.MaxDoc;
Bits liveDocs = reader.LiveDocs;
if (matchingVectorsReader == null || matchingVectorsReader.Version != VERSION_CURRENT || matchingVectorsReader.CompressionMode != CompressionMode || matchingVectorsReader.ChunkSize != ChunkSize || matchingVectorsReader.PackedIntsVersion != PackedInts.VERSION_CURRENT)
{
// naive merge...
for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = NextLiveDoc(i + 1, liveDocs, maxDoc))
{
Fields vectors = reader.GetTermVectors(i);
AddAllDocVectors(vectors, mergeState);
++docCount;
mergeState.checkAbort.Work(300);
}
}
else
{
CompressingStoredFieldsIndexReader index = matchingVectorsReader.Index;
IndexInput vectorsStreamOrig = matchingVectorsReader.VectorsStream;
vectorsStreamOrig.Seek(0);
ChecksumIndexInput vectorsStream = new BufferedChecksumIndexInput((IndexInput)vectorsStreamOrig.Clone());
for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc;)
{
// We make sure to move the checksum input in any case, otherwise the final
// integrity check might need to read the whole file a second time
long startPointer = index.GetStartPointer(i);
if (startPointer > vectorsStream.FilePointer)
{
vectorsStream.Seek(startPointer);
}
if ((PendingDocs.Count == 0) && (i == 0 || index.GetStartPointer(i - 1) < startPointer)) // start of a chunk
{
int docBase = vectorsStream.ReadVInt();
int chunkDocs = vectorsStream.ReadVInt();
Debug.Assert(docBase + chunkDocs <= matchingSegmentReader.MaxDoc);
if (docBase + chunkDocs < matchingSegmentReader.MaxDoc && NextDeletedDoc(docBase, liveDocs, docBase + chunkDocs) == docBase + chunkDocs)
{
long chunkEnd = index.GetStartPointer(docBase + chunkDocs);
long chunkLength = chunkEnd - vectorsStream.FilePointer;
IndexWriter.WriteIndex(chunkDocs, this.VectorsStream.FilePointer);
this.VectorsStream.WriteVInt(docCount);
this.VectorsStream.WriteVInt(chunkDocs);
this.VectorsStream.CopyBytes(vectorsStream, chunkLength);
docCount += chunkDocs;
this.NumDocs += chunkDocs;
mergeState.checkAbort.Work(300 * chunkDocs);
i = NextLiveDoc(docBase + chunkDocs, liveDocs, maxDoc);
}
else
{
for (; i < docBase + chunkDocs; i = NextLiveDoc(i + 1, liveDocs, maxDoc))
{
Fields vectors = reader.GetTermVectors(i);
AddAllDocVectors(vectors, mergeState);
++docCount;
mergeState.checkAbort.Work(300);
}
}
}
else
{
Fields vectors = reader.GetTermVectors(i);
AddAllDocVectors(vectors, mergeState);
++docCount;
mergeState.checkAbort.Work(300);
i = NextLiveDoc(i + 1, liveDocs, maxDoc);
}
}
vectorsStream.Seek(vectorsStream.Length() - CodecUtil.FooterLength());
CodecUtil.CheckFooter(vectorsStream);
}
}
Finish(mergeState.FieldInfos, docCount);
return(docCount);
}
/// <summary>
/// Sole constructor. </summary>
public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.compressionMode = compressionMode;
string segment = si.Name;
bool success = false;
fieldInfos = fn;
numDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
string fieldsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION);
// Load the index into memory
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingStoredFieldsWriter.CODEC_SFX_IDX;
version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.Position); // LUCENENET specific: Renamed from getFilePointer() to match FileStream
}
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
long maxPointer = -1;
if (version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
maxPointer = indexStream.ReadVInt64();
CodecUtil.CheckFooter(indexStream);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(indexStream);
#pragma warning restore 612, 618
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
fieldsStream = d.OpenInput(fieldsStreamFN, context);
if (version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
if (maxPointer + CodecUtil.FooterLength() != fieldsStream.Length)
{
throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.Length);
}
}
else
{
maxPointer = fieldsStream.Length;
}
this.maxPointer = maxPointer;
string codecNameDat = formatName + CompressingStoredFieldsWriter.CODEC_SFX_DAT;
int fieldsVersion = CodecUtil.CheckHeader(fieldsStream, codecNameDat, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
if (version != fieldsVersion)
{
throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + fieldsVersion);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameDat) == fieldsStream.Position); // LUCENENET specific: Renamed from getFilePointer() to match FileStream
}
if (version >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS)
{
chunkSize = fieldsStream.ReadVInt32();
}
else
{
chunkSize = -1;
}
packedIntsVersion = fieldsStream.ReadVInt32();
decompressor = compressionMode.NewDecompressor();
this.bytes = new BytesRef();
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this, indexStream);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.CompressionMode_Renamed = compressionMode;
string segment = si.Name;
bool success = false;
FieldInfos = fn;
NumDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
string fieldsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION);
// Load the index into memory
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingStoredFieldsWriter.CODEC_SFX_IDX;
Version_Renamed = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.FilePointer);
IndexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
long maxPointer = -1;
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
maxPointer = indexStream.ReadVLong();
CodecUtil.CheckFooter(indexStream);
}
else
{
CodecUtil.CheckEOF(indexStream);
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
FieldsStream = d.OpenInput(fieldsStreamFN, context);
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
if (maxPointer + CodecUtil.FooterLength() != FieldsStream.Length())
{
throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + FieldsStream.Length());
}
}
else
{
maxPointer = FieldsStream.Length();
}
this.MaxPointer = maxPointer;
string codecNameDat = formatName + CompressingStoredFieldsWriter.CODEC_SFX_DAT;
int fieldsVersion = CodecUtil.CheckHeader(FieldsStream, codecNameDat, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
if (Version_Renamed != fieldsVersion)
{
throw new CorruptIndexException("Version mismatch between stored fields index and data: " + Version_Renamed + " != " + fieldsVersion);
}
Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == FieldsStream.FilePointer);
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS)
{
ChunkSize_Renamed = FieldsStream.ReadVInt();
}
else
{
ChunkSize_Renamed = -1;
}
PackedIntsVersion = FieldsStream.ReadVInt();
Decompressor = compressionMode.NewDecompressor();
this.Bytes = new BytesRef();
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(this, indexStream);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.CompressionMode_Renamed = compressionMode;
string segment = si.Name;
bool success = false;
FieldInfos = fn;
NumDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
string fieldsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION);
// Load the index into memory
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingStoredFieldsWriter.CODEC_SFX_IDX;
Version_Renamed = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.FilePointer);
IndexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
long maxPointer = -1;
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
maxPointer = indexStream.ReadVLong();
CodecUtil.CheckFooter(indexStream);
}
else
{
CodecUtil.CheckEOF(indexStream);
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
FieldsStream = d.OpenInput(fieldsStreamFN, context);
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
if (maxPointer + CodecUtil.FooterLength() != FieldsStream.Length())
{
throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + FieldsStream.Length());
}
}
else
{
maxPointer = FieldsStream.Length();
}
this.MaxPointer = maxPointer;
string codecNameDat = formatName + CompressingStoredFieldsWriter.CODEC_SFX_DAT;
int fieldsVersion = CodecUtil.CheckHeader(FieldsStream, codecNameDat, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
if (Version_Renamed != fieldsVersion)
{
throw new CorruptIndexException("Version mismatch between stored fields index and data: " + Version_Renamed + " != " + fieldsVersion);
}
Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == FieldsStream.FilePointer);
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS)
{
ChunkSize_Renamed = FieldsStream.ReadVInt();
}
else
{
ChunkSize_Renamed = -1;
}
PackedIntsVersion = FieldsStream.ReadVInt();
Decompressor = compressionMode.NewDecompressor();
this.Bytes = new BytesRef();
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(this, indexStream);
}
}
}
