public PreFlexRWStoredFieldsWriter(Directory directory, string segment, IOContext context)
{
Debug.Assert(directory != null);
this.Directory = directory;
this.Segment = segment;
bool success = false;
try
{
FieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION), context);
IndexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION), context);
FieldsStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT);
IndexStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
Abort();
}
}
}
/// <summary>
/// Save a single segment's info. </summary>
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
{
string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
si.AddFile(fileName);
IndexOutput output = dir.CreateOutput(fileName, ioContext);
bool success = false;
try
{
CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT);
// Write the Lucene version that created this segment, since 3.1
output.WriteString(si.Version);
output.WriteInt(si.DocCount);
output.WriteByte((byte)(sbyte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO));
output.WriteStringStringMap(si.Diagnostics);
output.WriteStringSet(si.Files);
CodecUtil.WriteFooter(output);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(output);
si.Dir.DeleteFile(fileName);
}
else
{
output.Dispose();
}
}
}
public IndexInputSlicerAnonymousInnerClassHelper(NIOFSDirectory outerInstance, Lucene.Net.Store.IOContext context, FileInfo path, FileStream descriptor)
: base(outerInstance)
{
this.OuterInstance = outerInstance;
this.Context = context;
this.Path = path;
this.Descriptor = descriptor;
}
public override IndexInput OpenInput(string name, IOContext context)
{
if (sleepMillis != -1)
{
return new SlowIndexInput(this, base.OpenInput(name, context));
}
return base.OpenInput(name, context);
}
internal TermInfosReader(Directory dir, string seg, FieldInfos fis, IOContext context, int indexDivisor)
{
bool success = false;
if (indexDivisor < 1 && indexDivisor != -1)
{
throw new System.ArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + indexDivisor);
}
try
{
Directory = dir;
Segment = seg;
FieldInfos = fis;
OrigEnum = new SegmentTermEnum(Directory.OpenInput(IndexFileNames.SegmentFileName(Segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), context), FieldInfos, false);
Size_Renamed = OrigEnum.Size;
if (indexDivisor != -1)
{
// Load terms index
TotalIndexInterval = OrigEnum.IndexInterval * indexDivisor;
string indexFileName = IndexFileNames.SegmentFileName(Segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION);
SegmentTermEnum indexEnum = new SegmentTermEnum(Directory.OpenInput(indexFileName, context), FieldInfos, true);
try
{
Index = new TermInfosReaderIndex(indexEnum, indexDivisor, dir.FileLength(indexFileName), TotalIndexInterval);
IndexLength = Index.Length();
}
finally
{
indexEnum.Dispose();
}
}
else
{
// Do not load terms index:
TotalIndexInterval = -1;
Index = null;
IndexLength = -1;
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
Dispose();
}
}
}
public override IndexOutput CreateOutput(string name, IOContext context)
{
if (sleepMillis != -1)
{
return new SlowIndexOutput(this, base.CreateOutput(name, context));
}
return base.CreateOutput(name, context);
}
/// <summary>
/// Create a {@code SegmentReadState}. </summary>
public SegmentReadState(SegmentReadState other, string newSegmentSuffix)
{
this.Directory = other.Directory;
this.SegmentInfo = other.SegmentInfo;
this.FieldInfos = other.FieldInfos;
this.Context = other.Context;
this.TermsIndexDivisor = other.TermsIndexDivisor;
this.SegmentSuffix = newSegmentSuffix;
}
/// <summary>
/// Create a {@code SegmentReadState}. </summary>
public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, string segmentSuffix)
{
this.Directory = dir;
this.SegmentInfo = info;
this.FieldInfos = fieldInfos;
this.Context = context;
this.TermsIndexDivisor = termsIndexDivisor;
this.SegmentSuffix = segmentSuffix;
}
/// <summary>
/// Create a {@code SegmentReadState}. </summary>
public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, string segmentSuffix)
{
this.Directory = dir;
this.SegmentInfo = info;
this.FieldInfos = fieldInfos;
this.Context = context;
this.TermsIndexDivisor = termsIndexDivisor;
this.SegmentSuffix = segmentSuffix;
}
public override IndexInput OpenInput(string name, IOContext context)
{
IndexInput ii = base.OpenInput(name, context);
if (name.EndsWith(".prx") || name.EndsWith(".pos"))
{
// we decorate the proxStream with a wrapper class that allows to count the number of calls of seek()
ii = new SeeksCountingStream(OuterInstance, ii);
}
return(ii);
}
/// <summary>
/// Constructor which takes segment suffix.
/// </summary>
/// <seealso cref="SegmentWriteState(InfoStream, Directory, SegmentInfo, FieldInfos, int,
/// BufferedUpdates, IOContext)"/>
public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context, string segmentSuffix)
{
this.InfoStream = infoStream;
this.SegUpdates = segUpdates;
this.Directory = directory;
this.SegmentInfo = segmentInfo;
this.FieldInfos = fieldInfos;
this.TermIndexInterval = termIndexInterval;
this.SegmentSuffix = segmentSuffix;
this.Context = context;
}
/// <summary>
/// Create a shallow copy of <see cref="SegmentWriteState"/> with a new segment suffix. </summary>
public SegmentWriteState(SegmentWriteState state, string segmentSuffix)
{
InfoStream = state.InfoStream;
Directory = state.Directory;
SegmentInfo = state.SegmentInfo;
FieldInfos = state.FieldInfos;
TermIndexInterval = state.TermIndexInterval;
Context = state.Context;
this.SegmentSuffix = segmentSuffix;
SegUpdates = state.SegUpdates;
DelCountOnFlush = state.DelCountOnFlush;
}
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION);
// Unfortunately, for 3.x indices, each segment's
// FieldInfos can lie about hasVectors (claim it's true
// when really it's false).... so we have to carefully
// check if the files really exist before trying to open
// them (4.x has fixed this):
bool exists;
if (Lucene3xSegmentInfoFormat.GetDocStoreOffset(segmentInfo) != -1 && Lucene3xSegmentInfoFormat.GetDocStoreIsCompoundFile(segmentInfo))
{
string cfxFileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION);
if (segmentInfo.Dir.FileExists(cfxFileName))
{
Directory cfsDir = new CompoundFileDirectory(segmentInfo.Dir, cfxFileName, context, false);
try
{
exists = cfsDir.FileExists(fileName);
}
finally
{
cfsDir.Dispose();
}
}
else
{
exists = false;
}
}
else
{
exists = directory.FileExists(fileName);
}
if (!exists)
{
// 3x's FieldInfos sometimes lies and claims a segment
// has vectors when it doesn't:
return null;
}
else
{
return new Lucene3xTermVectorsReader(directory, segmentInfo, fieldInfos, context);
}
}
public override SegmentInfo Read(Directory dir, string segment, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
ChecksumIndexInput input = dir.OpenChecksumInput(fileName, context);
bool success = false;
try
{
int codecVersion = CodecUtil.CheckHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_START, Lucene46SegmentInfoFormat.VERSION_CURRENT);
string version = input.ReadString();
int docCount = input.ReadInt();
if (docCount < 0)
{
throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")");
}
bool isCompoundFile = input.ReadByte() == SegmentInfo.YES;
IDictionary<string, string> diagnostics = input.ReadStringStringMap();
ISet<string> files = input.ReadStringSet();
if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(input);
}
else
{
CodecUtil.CheckEOF(input);
}
SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
si.Files = files;
success = true;
return si;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(input);
}
else
{
input.Dispose();
}
}
}
// note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!!
public SegmentMerger(IList<AtomicReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir, int termIndexInterval, MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context, bool validate)
{
// validate incoming readers
if (validate)
{
foreach (AtomicReader reader in readers)
{
reader.CheckIntegrity();
}
}
MergeState = new MergeState(readers, segmentInfo, infoStream, checkAbort);
Directory = dir;
this.TermIndexInterval = termIndexInterval;
this.Codec = segmentInfo.Codec;
this.Context = context;
this.FieldInfosBuilder = new FieldInfos.Builder(fieldNumbers);
MergeState.SegmentInfo.DocCount = SetDocMaps();
}
/// <summary>
/// Sole constructor. </summary>
public Lucene40StoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context)
{
string segment = si.Name;
bool success = false;
FieldInfos = fn;
try
{
FieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context);
string indexStreamFN = IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
IndexStream = d.OpenInput(indexStreamFN, context);
CodecUtil.CheckHeader(IndexStream, Lucene40StoredFieldsWriter.CODEC_NAME_IDX, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
CodecUtil.CheckHeader(FieldsStream, Lucene40StoredFieldsWriter.CODEC_NAME_DAT, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_DAT == FieldsStream.FilePointer);
Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX == IndexStream.FilePointer);
long indexSize = IndexStream.Length() - Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX;
this.Size_Renamed = (int)(indexSize >> 3);
// Verify two sources of "maxDoc" agree:
if (this.Size_Renamed != si.DocCount)
{
throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.Size_Renamed + " but segmentInfo shows " + si.DocCount);
}
NumTotalDocs = (int)(indexSize >> 3);
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
try
{
Dispose();
} // ensure we throw our original exception
catch (Exception)
{
}
}
}
}
private int LastFieldNumber = -1; // only for assert
#endregion Fields
#region Constructors
public PreFlexRWNormsConsumer(Directory directory, string segment, IOContext context)
{
string normsFileName = IndexFileNames.SegmentFileName(segment, "", NORMS_EXTENSION);
bool success = false;
IndexOutput output = null;
try
{
output = directory.CreateOutput(normsFileName, context);
// output.WriteBytes(NORMS_HEADER, 0, NORMS_HEADER.Length);
foreach (var @sbyte in NORMS_HEADER)
{
output.WriteByte((byte)@sbyte);
}
@out = output;
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
/// <summary>
/// Save a single segment's info. </summary>
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
{
string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
si.AddFile(fileName);
IndexOutput output = dir.CreateOutput(fileName, ioContext);
bool success = false;
try
{
CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT);
// Write the Lucene version that created this segment, since 3.1
output.WriteString(si.Version);
output.WriteInt32(si.DocCount);
output.WriteByte((byte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO));
output.WriteStringStringMap(si.Diagnostics);
output.WriteStringSet(si.GetFiles());
CodecUtil.WriteFooter(output);
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(output);
si.Dir.DeleteFile(fileName);
}
else
{
output.Dispose();
}
}
}
public PreFlexRWTermVectorsWriter(Directory directory, string segment, IOContext context)
{
this.Directory = directory;
this.Segment = segment;
bool success = false;
try
{
// Open files for TermVector storage
Tvx = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION), context);
Tvx.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT);
Tvd = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION), context);
Tvd.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT);
Tvf = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION), context);
Tvf.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
Abort();
}
}
}
/// <summary> /// Returns a <seealso cref="StoredFieldsReader"/> to load stored /// fields. /// </summary> public abstract StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context);
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
IndexOutput output = directory.CreateOutput(fileName, context);
bool success = false;
try
{
output.WriteVInt(FORMAT_PREFLEX_RW);
output.WriteVInt(infos.Size());
foreach (FieldInfo fi in infos)
{
sbyte bits = 0x0;
if (fi.HasVectors())
{
bits |= STORE_TERMVECTOR;
}
if (fi.OmitsNorms())
{
bits |= OMIT_NORMS;
}
if (fi.HasPayloads())
{
bits |= STORE_PAYLOADS;
}
if (fi.Indexed)
{
bits |= IS_INDEXED;
Debug.Assert(fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads());
if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_ONLY)
{
bits |= OMIT_TERM_FREQ_AND_POSITIONS;
}
else if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS)
{
bits |= OMIT_POSITIONS;
}
}
output.WriteString(fi.Name);
/*
* we need to write the field number since IW tries
* to stabelize the field numbers across segments so the
* FI ordinal is not necessarily equivalent to the field number
*/
output.WriteInt(fi.Number);
output.WriteByte((byte)bits);
if (fi.Indexed && !fi.OmitsNorms())
{
// to allow null norm types we need to indicate if norms are written
// only in RW case
output.WriteByte((byte)(sbyte)(fi.NormType == null ? 0 : 1));
}
Debug.Assert(fi.Attributes() == null); // not used or supported
}
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
{
return new PreFlexRWTermVectorsWriter(directory, segmentInfo.Name, context);
}
/// <summary> /// Writes the provided <see cref="FieldInfos"/> to the /// directory. /// </summary> public abstract void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context);
public override IndexInput OpenInput(string fileName, IOContext context)
{
IndexInput @in = base.OpenInput(fileName, context);
if (fileName.EndsWith(".frq"))
{
@in = new CountingStream(OuterInstance, @in);
}
return @in;
}
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
{
return new CompressingStoredFieldsWriter(directory, si, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize);
}
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
{
return new AssertingTermVectorsWriter(@in.VectorsWriter(directory, segmentInfo, context));
}
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
{
return new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context);
}
/// <summary> /// Read <see cref="SegmentInfo"/> data from a directory. </summary> /// <param name="directory"> Directory to read from. </param> /// <param name="segmentName"> Name of the segment to read. </param> /// <param name="context"> IO context. </param> /// <returns> Infos instance to be populated with data. </returns> /// <exception cref="System.IO.IOException"> If an I/O error occurs. </exception> public abstract SegmentInfo Read(Directory directory, string segmentName, IOContext context);
/// <summary> /// Read the <see cref="FieldInfos"/> previously written with /// <see cref="FieldInfosWriter"/>. /// </summary> public abstract FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext iocontext);
public override sealed TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
{
return(new CompressingTermVectorsWriter(directory, segmentInfo, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize));
}
public override sealed TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
{
return(new CompressingTermVectorsReader(directory, segmentInfo, SegmentSuffix, fieldInfos, context, FormatName, CompressionMode));
}
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
{
return(new Lucene40TermVectorsReader(directory, segmentInfo, fieldInfos, context));
}
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
{
return new AssertingTermVectorsReader(@in.VectorsReader(directory, segmentInfo, fieldInfos, context));
}
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
{
return(new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, context, formatName, compressionMode));
}
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
{
return new CompressingStoredFieldsReader(directory, si, SegmentSuffix, fn, context, FormatName, CompressionMode);
}
/// <summary> /// Returns a <seealso cref="StoredFieldsWriter"/> to write stored /// fields. /// </summary> public abstract StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context);
/// <summary>
/// Sole constructor. </summary>
public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context)
: this(infoStream, directory, segmentInfo, fieldInfos, termIndexInterval, segUpdates, context, "")
{
}
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
{
return new AssertingStoredFieldsWriter(@in.FieldsWriter(directory, si, context));
}
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
IndexOutput output = directory.CreateOutput(fileName, context);
bool success = false;
try
{
CodecUtil.WriteHeader(output, Lucene42FieldInfosFormat.CODEC_NAME, Lucene42FieldInfosFormat.FORMAT_CURRENT);
output.WriteVInt(infos.Size());
foreach (FieldInfo fi in infos)
{
FieldInfo.IndexOptions? indexOptions = fi.FieldIndexOptions;
sbyte bits = 0x0;
if (fi.HasVectors())
{
bits |= Lucene42FieldInfosFormat.STORE_TERMVECTOR;
}
if (fi.OmitsNorms())
{
bits |= Lucene42FieldInfosFormat.OMIT_NORMS;
}
if (fi.HasPayloads())
{
bits |= Lucene42FieldInfosFormat.STORE_PAYLOADS;
}
if (fi.Indexed)
{
bits |= Lucene42FieldInfosFormat.IS_INDEXED;
Debug.Assert(indexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads());
if (indexOptions == FieldInfo.IndexOptions.DOCS_ONLY)
{
bits |= Lucene42FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS;
}
else if (indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
{
bits |= Lucene42FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS;
}
else if (indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS)
{
bits |= Lucene42FieldInfosFormat.OMIT_POSITIONS;
}
}
output.WriteString(fi.Name);
output.WriteVInt(fi.Number);
output.WriteByte((byte)bits);
// pack the DV types in one byte
var dv = DocValuesByte(fi.DocValuesType);
var nrm = DocValuesByte(fi.NormType);
Debug.Assert((dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0);
var val = unchecked((sbyte)(0xff & ((nrm << 4) | dv)));
output.WriteByte((byte)val);
output.WriteStringStringMap(fi.Attributes());
}
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
public override FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext iocontext)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION);
IndexInput input = directory.OpenInput(fileName, iocontext);
try
{
int format = input.ReadVInt();
if (format > FORMAT_MINIMUM)
{
throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT);
}
if (format < PreFlexRWFieldInfosWriter.FORMAT_CURRENT && format != PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW)
{
throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT);
}
int size = input.ReadVInt(); //read in the size
FieldInfo[] infos = new FieldInfo[size];
for (int i = 0; i < size; i++)
{
string name = input.ReadString();
int fieldNumber = format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW ? input.ReadInt() : i;
byte bits = input.ReadByte();
bool isIndexed = (bits & PreFlexRWFieldInfosWriter.IS_INDEXED) != 0;
bool storeTermVector = (bits & PreFlexRWFieldInfosWriter.STORE_TERMVECTOR) != 0;
bool omitNorms = (bits & PreFlexRWFieldInfosWriter.OMIT_NORMS) != 0;
bool storePayloads = (bits & PreFlexRWFieldInfosWriter.STORE_PAYLOADS) != 0;
FieldInfo.IndexOptions?indexOptions;
if (!isIndexed)
{
indexOptions = null;
}
else if ((bits & PreFlexRWFieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0)
{
indexOptions = FieldInfo.IndexOptions.DOCS_ONLY;
}
else if ((bits & PreFlexRWFieldInfosWriter.OMIT_POSITIONS) != 0)
{
if (format <= PreFlexRWFieldInfosWriter.FORMAT_OMIT_POSITIONS)
{
indexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS;
}
else
{
throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")");
}
}
else
{
indexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
}
// LUCENE-3027: past indices were able to write
// storePayloads=true when omitTFAP is also true,
// which is invalid. We correct that, here:
if (indexOptions != FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
{
storePayloads = false;
}
DocValuesType_e?normType = isIndexed && !omitNorms ? (DocValuesType_e?)DocValuesType_e.NUMERIC : null;
if (format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW && normType != null)
{
// RW can have norms but doesn't write them
normType = input.ReadByte() != 0 ? (DocValuesType_e?)DocValuesType_e.NUMERIC : null;
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, null, normType, null);
}
if (input.FilePointer != input.Length())
{
throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.FilePointer + " vs size " + input.Length() + " (resource: " + input + ")");
}
return(new FieldInfos(infos));
}
finally
{
input.Dispose();
}
}
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
{
return new Lucene3xTermVectorsReaderAnonymousInnerClassHelper(this, directory, segmentInfo, fieldInfos, context);
}
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
{
return(new Lucene40StoredFieldsWriter(directory, si.Name, context));
}
public Lucene3xTermVectorsReaderAnonymousInnerClassHelper(PreFlexRWTermVectorsFormat outerInstance, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
: base(directory, segmentInfo, fieldInfos, context)
{
this.OuterInstance = outerInstance;
}
/// <summary> /// Returns a <see cref="TermVectorsReader"/> to read term /// vectors. /// </summary> public abstract TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context);
// NOTE: this is not "really" 3.x format, because we are
// writing each SI to its own file, vs 3.x where the list
// of segments and SI for each segment is written into a
// single segments_N file
/// <summary>
/// Save a single segment's info. </summary>
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
{
SegmentInfos.Write3xInfo(dir, si, ioContext);
}
/// <summary> /// Returns a <see cref="TermVectorsWriter"/> to write term /// vectors. /// </summary> public abstract TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context);
/// <summary> /// Returns a <seealso cref="StoredFieldsWriter"/> to write stored /// fields. /// </summary> public abstract StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context);
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
{
return(new Lucene40TermVectorsWriter(directory, segmentInfo.Name, context));
}
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
{
return(new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context, formatName, compressionMode, chunkSize));
}
/// <summary>
/// Seals the <seealso cref="SegmentInfo"/> for the new flushed segment and persists
/// the deleted documents <seealso cref="MutableBits"/>.
/// </summary>
internal virtual void SealFlushedSegment(FlushedSegment flushedSegment)
{
Debug.Assert(flushedSegment != null);
SegmentCommitInfo newSegment = flushedSegment.SegmentInfo;
IndexWriter.SetDiagnostics(newSegment.Info, IndexWriter.SOURCE_FLUSH);
IOContext context = new IOContext(new FlushInfo(newSegment.Info.DocCount, newSegment.SizeInBytes()));
bool success = false;
try
{
if (IndexWriterConfig.UseCompoundFile)
{
CollectionsHelper.AddAll(FilesToDelete, IndexWriter.CreateCompoundFile(InfoStream, Directory, MergeState.CheckAbort.NONE, newSegment.Info, context));
newSegment.Info.UseCompoundFile = true;
}
// Have codec write SegmentInfo. Must do this after
// creating CFS so that 1) .si isn't slurped into CFS,
// and 2) .si reflects useCompoundFile=true change
// above:
Codec.SegmentInfoFormat().SegmentInfoWriter.Write(Directory, newSegment.Info, flushedSegment.FieldInfos, context);
// TODO: ideally we would freeze newSegment here!!
// because any changes after writing the .si will be
// lost...
// Must write deleted docs after the CFS so we don't
// slurp the del file into CFS:
if (flushedSegment.LiveDocs != null)
{
int delCount = flushedSegment.DelCount;
Debug.Assert(delCount > 0);
if (InfoStream.IsEnabled("DWPT"))
{
InfoStream.Message("DWPT", "flush: write " + delCount + " deletes gen=" + flushedSegment.SegmentInfo.DelGen);
}
// TODO: we should prune the segment if it's 100%
// deleted... but merge will also catch it.
// TODO: in the NRT case it'd be better to hand
// this del vector over to the
// shortly-to-be-opened SegmentReader and let it
// carry the changes; there's no reason to use
// filesystem as intermediary here.
SegmentCommitInfo info = flushedSegment.SegmentInfo;
Codec codec = info.Info.Codec;
codec.LiveDocsFormat().WriteLiveDocs(flushedSegment.LiveDocs, Directory, info, delCount, context);
newSegment.DelCount = delCount;
newSegment.AdvanceDelGen();
}
success = true;
}
finally
{
if (!success)
{
if (InfoStream.IsEnabled("DWPT"))
{
InfoStream.Message("DWPT", "hit exception creating compound file for newly flushed segment " + newSegment.Info.Name);
}
}
}
}
/// <summary>
/// Create a <see cref="SegmentReadState"/>. </summary>
public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor)
: this(dir, info, fieldInfos, context, termsIndexDivisor, "")
{
}
/// <summary> /// Write <see cref="SegmentInfo"/> data. </summary> /// <exception cref="System.IO.IOException"> If an I/O error occurs. </exception> public abstract void Write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext);
/// <summary> /// Returns a <seealso cref="StoredFieldsReader"/> to load stored /// fields. /// </summary> public abstract StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context);
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
{
return(new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context));
}
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
{
return new AssertingStoredFieldsReader(@in.FieldsReader(directory, si, fn, context), si.DocCount);
}
// NOTE: this is not "really" 3.x format, because we are
// writing each SI to its own file, vs 3.x where the list
// of segments and SI for each segment is written into a
// single segments_N file
/// <summary>
/// Save a single segment's info. </summary>
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
{
SegmentInfos.Write3xInfo(dir, si, ioContext);
}
/// <summary>
/// TODO: javadoc </summary>
public static IOContext NewIOContext(Random random, IOContext oldContext)
{
int randomNumDocs = random.Next(4192);
int size = random.Next(512) * randomNumDocs;
if (oldContext.FlushInfo != null)
{
// Always return at least the estimatedSegmentSize of
// the incoming IOContext:
return new IOContext(new FlushInfo(randomNumDocs, (long)Math.Max(oldContext.FlushInfo.EstimatedSegmentSize, size)));
}
else if (oldContext.MergeInfo != null)
{
// Always return at least the estimatedMergeBytes of
// the incoming IOContext:
return new IOContext(new MergeInfo(randomNumDocs, Math.Max(oldContext.MergeInfo.EstimatedMergeBytes, size), random.NextBoolean(), TestUtil.NextInt(random, 1, 100)));
}
else
{
// Make a totally random IOContext:
IOContext context;
switch (random.Next(5))
{
case 0:
context = IOContext.DEFAULT;
break;
case 1:
context = IOContext.READ;
break;
case 2:
context = IOContext.READONCE;
break;
case 3:
context = new IOContext(new MergeInfo(randomNumDocs, size, true, -1));
break;
case 4:
context = new IOContext(new FlushInfo(randomNumDocs, size));
break;
default:
context = IOContext.DEFAULT;
break;
}
return context;
}
}
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
{
return(new Lucene40StoredFieldsReader(directory, si, fn, context));
}
