public PreFlexRWStoredFieldsWriter(Directory directory, string segment, IOContext context) { Debug.Assert(directory != null); this.Directory = directory; this.Segment = segment; bool success = false; try { FieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION), context); IndexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION), context); FieldsStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT); IndexStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT); success = true; } finally { if (!success) { Abort(); } } }
/// <summary> /// Save a single segment's info. </summary> public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION); si.AddFile(fileName); IndexOutput output = dir.CreateOutput(fileName, ioContext); bool success = false; try { CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT); // Write the Lucene version that created this segment, since 3.1 output.WriteString(si.Version); output.WriteInt(si.DocCount); output.WriteByte((byte)(sbyte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO)); output.WriteStringStringMap(si.Diagnostics); output.WriteStringSet(si.Files); CodecUtil.WriteFooter(output); success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(output); si.Dir.DeleteFile(fileName); } else { output.Dispose(); } } }
public IndexInputSlicerAnonymousInnerClassHelper(NIOFSDirectory outerInstance, Lucene.Net.Store.IOContext context, FileInfo path, FileStream descriptor) : base(outerInstance) { this.OuterInstance = outerInstance; this.Context = context; this.Path = path; this.Descriptor = descriptor; }
public override IndexInput OpenInput(string name, IOContext context) { if (sleepMillis != -1) { return new SlowIndexInput(this, base.OpenInput(name, context)); } return base.OpenInput(name, context); }
internal TermInfosReader(Directory dir, string seg, FieldInfos fis, IOContext context, int indexDivisor) { bool success = false; if (indexDivisor < 1 && indexDivisor != -1) { throw new System.ArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + indexDivisor); } try { Directory = dir; Segment = seg; FieldInfos = fis; OrigEnum = new SegmentTermEnum(Directory.OpenInput(IndexFileNames.SegmentFileName(Segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), context), FieldInfos, false); Size_Renamed = OrigEnum.Size; if (indexDivisor != -1) { // Load terms index TotalIndexInterval = OrigEnum.IndexInterval * indexDivisor; string indexFileName = IndexFileNames.SegmentFileName(Segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); SegmentTermEnum indexEnum = new SegmentTermEnum(Directory.OpenInput(indexFileName, context), FieldInfos, true); try { Index = new TermInfosReaderIndex(indexEnum, indexDivisor, dir.FileLength(indexFileName), TotalIndexInterval); IndexLength = Index.Length(); } finally { indexEnum.Dispose(); } } else { // Do not load terms index: TotalIndexInterval = -1; Index = null; IndexLength = -1; } success = true; } finally { // With lock-less commits, it's entirely possible (and // fine) to hit a FileNotFound exception above. In // this case, we want to explicitly close any subset // of things that were opened so that we don't have to // wait for a GC to do so. if (!success) { Dispose(); } } }
public override IndexOutput CreateOutput(string name, IOContext context) { if (sleepMillis != -1) { return new SlowIndexOutput(this, base.CreateOutput(name, context)); } return base.CreateOutput(name, context); }
/// <summary> /// Create a {@code SegmentReadState}. </summary> public SegmentReadState(SegmentReadState other, string newSegmentSuffix) { this.Directory = other.Directory; this.SegmentInfo = other.SegmentInfo; this.FieldInfos = other.FieldInfos; this.Context = other.Context; this.TermsIndexDivisor = other.TermsIndexDivisor; this.SegmentSuffix = newSegmentSuffix; }
/// <summary> /// Create a {@code SegmentReadState}. </summary> public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, string segmentSuffix) { this.Directory = dir; this.SegmentInfo = info; this.FieldInfos = fieldInfos; this.Context = context; this.TermsIndexDivisor = termsIndexDivisor; this.SegmentSuffix = segmentSuffix; }
/// <summary> /// Create a {@code SegmentReadState}. </summary> public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, string segmentSuffix) { this.Directory = dir; this.SegmentInfo = info; this.FieldInfos = fieldInfos; this.Context = context; this.TermsIndexDivisor = termsIndexDivisor; this.SegmentSuffix = segmentSuffix; }
public override IndexInput OpenInput(string name, IOContext context) { IndexInput ii = base.OpenInput(name, context); if (name.EndsWith(".prx") || name.EndsWith(".pos")) { // we decorate the proxStream with a wrapper class that allows to count the number of calls of seek() ii = new SeeksCountingStream(OuterInstance, ii); } return(ii); }
/// <summary> /// Constructor which takes segment suffix. /// </summary> /// <seealso cref="SegmentWriteState(InfoStream, Directory, SegmentInfo, FieldInfos, int, /// BufferedUpdates, IOContext)"/> public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context, string segmentSuffix) { this.InfoStream = infoStream; this.SegUpdates = segUpdates; this.Directory = directory; this.SegmentInfo = segmentInfo; this.FieldInfos = fieldInfos; this.TermIndexInterval = termIndexInterval; this.SegmentSuffix = segmentSuffix; this.Context = context; }
/// <summary> /// Create a shallow copy of <see cref="SegmentWriteState"/> with a new segment suffix. </summary> public SegmentWriteState(SegmentWriteState state, string segmentSuffix) { InfoStream = state.InfoStream; Directory = state.Directory; SegmentInfo = state.SegmentInfo; FieldInfos = state.FieldInfos; TermIndexInterval = state.TermIndexInterval; Context = state.Context; this.SegmentSuffix = segmentSuffix; SegUpdates = state.SegUpdates; DelCountOnFlush = state.DelCountOnFlush; }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { string fileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION); // Unfortunately, for 3.x indices, each segment's // FieldInfos can lie about hasVectors (claim it's true // when really it's false).... so we have to carefully // check if the files really exist before trying to open // them (4.x has fixed this): bool exists; if (Lucene3xSegmentInfoFormat.GetDocStoreOffset(segmentInfo) != -1 && Lucene3xSegmentInfoFormat.GetDocStoreIsCompoundFile(segmentInfo)) { string cfxFileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION); if (segmentInfo.Dir.FileExists(cfxFileName)) { Directory cfsDir = new CompoundFileDirectory(segmentInfo.Dir, cfxFileName, context, false); try { exists = cfsDir.FileExists(fileName); } finally { cfsDir.Dispose(); } } else { exists = false; } } else { exists = directory.FileExists(fileName); } if (!exists) { // 3x's FieldInfos sometimes lies and claims a segment // has vectors when it doesn't: return null; } else { return new Lucene3xTermVectorsReader(directory, segmentInfo, fieldInfos, context); } }
public override SegmentInfo Read(Directory dir, string segment, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION); ChecksumIndexInput input = dir.OpenChecksumInput(fileName, context); bool success = false; try { int codecVersion = CodecUtil.CheckHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_START, Lucene46SegmentInfoFormat.VERSION_CURRENT); string version = input.ReadString(); int docCount = input.ReadInt(); if (docCount < 0) { throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")"); } bool isCompoundFile = input.ReadByte() == SegmentInfo.YES; IDictionary<string, string> diagnostics = input.ReadStringStringMap(); ISet<string> files = input.ReadStringSet(); if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM) { CodecUtil.CheckFooter(input); } else { CodecUtil.CheckEOF(input); } SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics); si.Files = files; success = true; return si; } finally { if (!success) { IOUtils.CloseWhileHandlingException(input); } else { input.Dispose(); } } }
// note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!! public SegmentMerger(IList<AtomicReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir, int termIndexInterval, MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context, bool validate) { // validate incoming readers if (validate) { foreach (AtomicReader reader in readers) { reader.CheckIntegrity(); } } MergeState = new MergeState(readers, segmentInfo, infoStream, checkAbort); Directory = dir; this.TermIndexInterval = termIndexInterval; this.Codec = segmentInfo.Codec; this.Context = context; this.FieldInfosBuilder = new FieldInfos.Builder(fieldNumbers); MergeState.SegmentInfo.DocCount = SetDocMaps(); }
/// <summary> /// Sole constructor. </summary> public Lucene40StoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) { string segment = si.Name; bool success = false; FieldInfos = fn; try { FieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context); string indexStreamFN = IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION); IndexStream = d.OpenInput(indexStreamFN, context); CodecUtil.CheckHeader(IndexStream, Lucene40StoredFieldsWriter.CODEC_NAME_IDX, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT); CodecUtil.CheckHeader(FieldsStream, Lucene40StoredFieldsWriter.CODEC_NAME_DAT, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT); Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_DAT == FieldsStream.FilePointer); Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX == IndexStream.FilePointer); long indexSize = IndexStream.Length() - Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX; this.Size_Renamed = (int)(indexSize >> 3); // Verify two sources of "maxDoc" agree: if (this.Size_Renamed != si.DocCount) { throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.Size_Renamed + " but segmentInfo shows " + si.DocCount); } NumTotalDocs = (int)(indexSize >> 3); success = true; } finally { // With lock-less commits, it's entirely possible (and // fine) to hit a FileNotFound exception above. In // this case, we want to explicitly close any subset // of things that were opened so that we don't have to // wait for a GC to do so. if (!success) { try { Dispose(); } // ensure we throw our original exception catch (Exception) { } } } }
private int LastFieldNumber = -1; // only for assert #endregion Fields #region Constructors public PreFlexRWNormsConsumer(Directory directory, string segment, IOContext context) { string normsFileName = IndexFileNames.SegmentFileName(segment, "", NORMS_EXTENSION); bool success = false; IndexOutput output = null; try { output = directory.CreateOutput(normsFileName, context); // output.WriteBytes(NORMS_HEADER, 0, NORMS_HEADER.Length); foreach (var @sbyte in NORMS_HEADER) { output.WriteByte((byte)@sbyte); } @out = output; success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(output); } } }
/// <summary> /// Save a single segment's info. </summary> public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION); si.AddFile(fileName); IndexOutput output = dir.CreateOutput(fileName, ioContext); bool success = false; try { CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT); // Write the Lucene version that created this segment, since 3.1 output.WriteString(si.Version); output.WriteInt32(si.DocCount); output.WriteByte((byte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO)); output.WriteStringStringMap(si.Diagnostics); output.WriteStringSet(si.GetFiles()); CodecUtil.WriteFooter(output); success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(output); si.Dir.DeleteFile(fileName); } else { output.Dispose(); } } }
public PreFlexRWTermVectorsWriter(Directory directory, string segment, IOContext context) { this.Directory = directory; this.Segment = segment; bool success = false; try { // Open files for TermVector storage Tvx = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION), context); Tvx.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT); Tvd = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION), context); Tvd.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT); Tvf = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION), context); Tvf.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT); success = true; } finally { if (!success) { Abort(); } } }
/// <summary> /// Returns a <seealso cref="StoredFieldsReader"/> to load stored /// fields. /// </summary> public abstract StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context);
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segmentName, "", FIELD_INFOS_EXTENSION); IndexOutput output = directory.CreateOutput(fileName, context); bool success = false; try { output.WriteVInt(FORMAT_PREFLEX_RW); output.WriteVInt(infos.Size()); foreach (FieldInfo fi in infos) { sbyte bits = 0x0; if (fi.HasVectors()) { bits |= STORE_TERMVECTOR; } if (fi.OmitsNorms()) { bits |= OMIT_NORMS; } if (fi.HasPayloads()) { bits |= STORE_PAYLOADS; } if (fi.Indexed) { bits |= IS_INDEXED; Debug.Assert(fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads()); if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_ONLY) { bits |= OMIT_TERM_FREQ_AND_POSITIONS; } else if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS) { bits |= OMIT_POSITIONS; } } output.WriteString(fi.Name); /* * we need to write the field number since IW tries * to stabelize the field numbers across segments so the * FI ordinal is not necessarily equivalent to the field number */ output.WriteInt(fi.Number); output.WriteByte((byte)bits); if (fi.Indexed && !fi.OmitsNorms()) { // to allow null norm types we need to indicate if norms are written // only in RW case output.WriteByte((byte)(sbyte)(fi.NormType == null ? 0 : 1)); } Debug.Assert(fi.Attributes() == null); // not used or supported } success = true; } finally { if (success) { output.Dispose(); } else { IOUtils.CloseWhileHandlingException(output); } } }
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return new PreFlexRWTermVectorsWriter(directory, segmentInfo.Name, context); }
/// <summary> /// Writes the provided <see cref="FieldInfos"/> to the /// directory. /// </summary> public abstract void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context);
public override IndexInput OpenInput(string fileName, IOContext context) { IndexInput @in = base.OpenInput(fileName, context); if (fileName.EndsWith(".frq")) { @in = new CountingStream(OuterInstance, @in); } return @in; }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return new CompressingStoredFieldsWriter(directory, si, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize); }
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return new AssertingTermVectorsWriter(@in.VectorsWriter(directory, segmentInfo, context)); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context); }
/// <summary> /// Read <see cref="SegmentInfo"/> data from a directory. </summary> /// <param name="directory"> Directory to read from. </param> /// <param name="segmentName"> Name of the segment to read. </param> /// <param name="context"> IO context. </param> /// <returns> Infos instance to be populated with data. </returns> /// <exception cref="System.IO.IOException"> If an I/O error occurs. </exception> public abstract SegmentInfo Read(Directory directory, string segmentName, IOContext context);
/// <summary> /// Read the <see cref="FieldInfos"/> previously written with /// <see cref="FieldInfosWriter"/>. /// </summary> public abstract FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext iocontext);
public override sealed TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return(new CompressingTermVectorsWriter(directory, segmentInfo, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize)); }
public override sealed TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return(new CompressingTermVectorsReader(directory, segmentInfo, SegmentSuffix, fieldInfos, context, FormatName, CompressionMode)); }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return(new Lucene40TermVectorsReader(directory, segmentInfo, fieldInfos, context)); }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return new AssertingTermVectorsReader(@in.VectorsReader(directory, segmentInfo, fieldInfos, context)); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return(new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, context, formatName, compressionMode)); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return new CompressingStoredFieldsReader(directory, si, SegmentSuffix, fn, context, FormatName, CompressionMode); }
/// <summary> /// Returns a <seealso cref="StoredFieldsWriter"/> to write stored /// fields. /// </summary> public abstract StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context);
/// <summary> /// Sole constructor. </summary> public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context) : this(infoStream, directory, segmentInfo, fieldInfos, termIndexInterval, segUpdates, context, "") { }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return new AssertingStoredFieldsWriter(@in.FieldsWriter(directory, si, context)); }
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION); IndexOutput output = directory.CreateOutput(fileName, context); bool success = false; try { CodecUtil.WriteHeader(output, Lucene42FieldInfosFormat.CODEC_NAME, Lucene42FieldInfosFormat.FORMAT_CURRENT); output.WriteVInt(infos.Size()); foreach (FieldInfo fi in infos) { FieldInfo.IndexOptions? indexOptions = fi.FieldIndexOptions; sbyte bits = 0x0; if (fi.HasVectors()) { bits |= Lucene42FieldInfosFormat.STORE_TERMVECTOR; } if (fi.OmitsNorms()) { bits |= Lucene42FieldInfosFormat.OMIT_NORMS; } if (fi.HasPayloads()) { bits |= Lucene42FieldInfosFormat.STORE_PAYLOADS; } if (fi.Indexed) { bits |= Lucene42FieldInfosFormat.IS_INDEXED; Debug.Assert(indexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads()); if (indexOptions == FieldInfo.IndexOptions.DOCS_ONLY) { bits |= Lucene42FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS; } else if (indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { bits |= Lucene42FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS; } else if (indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS) { bits |= Lucene42FieldInfosFormat.OMIT_POSITIONS; } } output.WriteString(fi.Name); output.WriteVInt(fi.Number); output.WriteByte((byte)bits); // pack the DV types in one byte var dv = DocValuesByte(fi.DocValuesType); var nrm = DocValuesByte(fi.NormType); Debug.Assert((dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0); var val = unchecked((sbyte)(0xff & ((nrm << 4) | dv))); output.WriteByte((byte)val); output.WriteStringStringMap(fi.Attributes()); } success = true; } finally { if (success) { output.Dispose(); } else { IOUtils.CloseWhileHandlingException(output); } } }
public override FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext iocontext) { string fileName = IndexFileNames.SegmentFileName(segmentName, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION); IndexInput input = directory.OpenInput(fileName, iocontext); try { int format = input.ReadVInt(); if (format > FORMAT_MINIMUM) { throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT); } if (format < PreFlexRWFieldInfosWriter.FORMAT_CURRENT && format != PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW) { throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT); } int size = input.ReadVInt(); //read in the size FieldInfo[] infos = new FieldInfo[size]; for (int i = 0; i < size; i++) { string name = input.ReadString(); int fieldNumber = format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW ? input.ReadInt() : i; byte bits = input.ReadByte(); bool isIndexed = (bits & PreFlexRWFieldInfosWriter.IS_INDEXED) != 0; bool storeTermVector = (bits & PreFlexRWFieldInfosWriter.STORE_TERMVECTOR) != 0; bool omitNorms = (bits & PreFlexRWFieldInfosWriter.OMIT_NORMS) != 0; bool storePayloads = (bits & PreFlexRWFieldInfosWriter.STORE_PAYLOADS) != 0; FieldInfo.IndexOptions?indexOptions; if (!isIndexed) { indexOptions = null; } else if ((bits & PreFlexRWFieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) { indexOptions = FieldInfo.IndexOptions.DOCS_ONLY; } else if ((bits & PreFlexRWFieldInfosWriter.OMIT_POSITIONS) != 0) { if (format <= PreFlexRWFieldInfosWriter.FORMAT_OMIT_POSITIONS) { indexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS; } else { throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")"); } } else { indexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; } // LUCENE-3027: past indices were able to write // storePayloads=true when omitTFAP is also true, // which is invalid. We correct that, here: if (indexOptions != FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { storePayloads = false; } DocValuesType_e?normType = isIndexed && !omitNorms ? (DocValuesType_e?)DocValuesType_e.NUMERIC : null; if (format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW && normType != null) { // RW can have norms but doesn't write them normType = input.ReadByte() != 0 ? (DocValuesType_e?)DocValuesType_e.NUMERIC : null; } infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, null, normType, null); } if (input.FilePointer != input.Length()) { throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.FilePointer + " vs size " + input.Length() + " (resource: " + input + ")"); } return(new FieldInfos(infos)); } finally { input.Dispose(); } }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return new Lucene3xTermVectorsReaderAnonymousInnerClassHelper(this, directory, segmentInfo, fieldInfos, context); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return(new Lucene40StoredFieldsWriter(directory, si.Name, context)); }
public Lucene3xTermVectorsReaderAnonymousInnerClassHelper(PreFlexRWTermVectorsFormat outerInstance, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) : base(directory, segmentInfo, fieldInfos, context) { this.OuterInstance = outerInstance; }
/// <summary> /// Returns a <see cref="TermVectorsReader"/> to read term /// vectors. /// </summary> public abstract TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context);
// NOTE: this is not "really" 3.x format, because we are // writing each SI to its own file, vs 3.x where the list // of segments and SI for each segment is written into a // single segments_N file /// <summary> /// Save a single segment's info. </summary> public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { SegmentInfos.Write3xInfo(dir, si, ioContext); }
/// <summary> /// Returns a <see cref="TermVectorsWriter"/> to write term /// vectors. /// </summary> public abstract TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context);
/// <summary> /// Returns a <seealso cref="StoredFieldsWriter"/> to write stored /// fields. /// </summary> public abstract StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context);
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return(new Lucene40TermVectorsWriter(directory, segmentInfo.Name, context)); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return(new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context, formatName, compressionMode, chunkSize)); }
/// <summary> /// Seals the <seealso cref="SegmentInfo"/> for the new flushed segment and persists /// the deleted documents <seealso cref="MutableBits"/>. /// </summary> internal virtual void SealFlushedSegment(FlushedSegment flushedSegment) { Debug.Assert(flushedSegment != null); SegmentCommitInfo newSegment = flushedSegment.SegmentInfo; IndexWriter.SetDiagnostics(newSegment.Info, IndexWriter.SOURCE_FLUSH); IOContext context = new IOContext(new FlushInfo(newSegment.Info.DocCount, newSegment.SizeInBytes())); bool success = false; try { if (IndexWriterConfig.UseCompoundFile) { CollectionsHelper.AddAll(FilesToDelete, IndexWriter.CreateCompoundFile(InfoStream, Directory, MergeState.CheckAbort.NONE, newSegment.Info, context)); newSegment.Info.UseCompoundFile = true; } // Have codec write SegmentInfo. Must do this after // creating CFS so that 1) .si isn't slurped into CFS, // and 2) .si reflects useCompoundFile=true change // above: Codec.SegmentInfoFormat().SegmentInfoWriter.Write(Directory, newSegment.Info, flushedSegment.FieldInfos, context); // TODO: ideally we would freeze newSegment here!! // because any changes after writing the .si will be // lost... // Must write deleted docs after the CFS so we don't // slurp the del file into CFS: if (flushedSegment.LiveDocs != null) { int delCount = flushedSegment.DelCount; Debug.Assert(delCount > 0); if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "flush: write " + delCount + " deletes gen=" + flushedSegment.SegmentInfo.DelGen); } // TODO: we should prune the segment if it's 100% // deleted... but merge will also catch it. // TODO: in the NRT case it'd be better to hand // this del vector over to the // shortly-to-be-opened SegmentReader and let it // carry the changes; there's no reason to use // filesystem as intermediary here. SegmentCommitInfo info = flushedSegment.SegmentInfo; Codec codec = info.Info.Codec; codec.LiveDocsFormat().WriteLiveDocs(flushedSegment.LiveDocs, Directory, info, delCount, context); newSegment.DelCount = delCount; newSegment.AdvanceDelGen(); } success = true; } finally { if (!success) { if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "hit exception creating compound file for newly flushed segment " + newSegment.Info.Name); } } } }
/// <summary> /// Create a <see cref="SegmentReadState"/>. </summary> public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor) : this(dir, info, fieldInfos, context, termsIndexDivisor, "") { }
/// <summary> /// Write <see cref="SegmentInfo"/> data. </summary> /// <exception cref="System.IO.IOException"> If an I/O error occurs. </exception> public abstract void Write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext);
/// <summary> /// Returns a <seealso cref="StoredFieldsReader"/> to load stored /// fields. /// </summary> public abstract StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context);
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return(new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context)); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return new AssertingStoredFieldsReader(@in.FieldsReader(directory, si, fn, context), si.DocCount); }
// NOTE: this is not "really" 3.x format, because we are // writing each SI to its own file, vs 3.x where the list // of segments and SI for each segment is written into a // single segments_N file /// <summary> /// Save a single segment's info. </summary> public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { SegmentInfos.Write3xInfo(dir, si, ioContext); }
/// <summary> /// TODO: javadoc </summary> public static IOContext NewIOContext(Random random, IOContext oldContext) { int randomNumDocs = random.Next(4192); int size = random.Next(512) * randomNumDocs; if (oldContext.FlushInfo != null) { // Always return at least the estimatedSegmentSize of // the incoming IOContext: return new IOContext(new FlushInfo(randomNumDocs, (long)Math.Max(oldContext.FlushInfo.EstimatedSegmentSize, size))); } else if (oldContext.MergeInfo != null) { // Always return at least the estimatedMergeBytes of // the incoming IOContext: return new IOContext(new MergeInfo(randomNumDocs, Math.Max(oldContext.MergeInfo.EstimatedMergeBytes, size), random.NextBoolean(), TestUtil.NextInt(random, 1, 100))); } else { // Make a totally random IOContext: IOContext context; switch (random.Next(5)) { case 0: context = IOContext.DEFAULT; break; case 1: context = IOContext.READ; break; case 2: context = IOContext.READONCE; break; case 3: context = new IOContext(new MergeInfo(randomNumDocs, size, true, -1)); break; case 4: context = new IOContext(new FlushInfo(randomNumDocs, size)); break; default: context = IOContext.DEFAULT; break; } return context; } }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return(new Lucene40StoredFieldsReader(directory, si, fn, context)); }