/// <summary> /// Save a single segment's info. </summary> public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION); si.AddFile(fileName); IndexOutput output = dir.CreateOutput(fileName, ioContext); bool success = false; try { CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT); // Write the Lucene version that created this segment, since 3.1 output.WriteString(si.Version); output.WriteInt(si.DocCount); output.WriteByte((byte)(sbyte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO)); output.WriteStringStringMap(si.Diagnostics); output.WriteStringSet(si.Files); CodecUtil.WriteFooter(output); success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(output); si.Dir.DeleteFile(fileName); } else { output.Dispose(); } } }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { string fileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION); // Unfortunately, for 3.x indices, each segment's // FieldInfos can lie about hasVectors (claim it's true // when really it's false).... so we have to carefully // check if the files really exist before trying to open // them (4.x has fixed this): bool exists; if (Lucene3xSegmentInfoFormat.GetDocStoreOffset(segmentInfo) != -1 && Lucene3xSegmentInfoFormat.GetDocStoreIsCompoundFile(segmentInfo)) { string cfxFileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION); if (segmentInfo.Dir.FileExists(cfxFileName)) { Directory cfsDir = new CompoundFileDirectory(segmentInfo.Dir, cfxFileName, context, false); try { exists = cfsDir.FileExists(fileName); } finally { cfsDir.Dispose(); } } else { exists = false; } } else { exists = directory.FileExists(fileName); } if (!exists) { // 3x's FieldInfos sometimes lies and claims a segment // has vectors when it doesn't: return null; } else { return new Lucene3xTermVectorsReader(directory, segmentInfo, fieldInfos, context); } }
public override SegmentInfo Read(Directory dir, string segment, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION); ChecksumIndexInput input = dir.OpenChecksumInput(fileName, context); bool success = false; try { int codecVersion = CodecUtil.CheckHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_START, Lucene46SegmentInfoFormat.VERSION_CURRENT); string version = input.ReadString(); int docCount = input.ReadInt(); if (docCount < 0) { throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")"); } bool isCompoundFile = input.ReadByte() == SegmentInfo.YES; IDictionary<string, string> diagnostics = input.ReadStringStringMap(); ISet<string> files = input.ReadStringSet(); if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM) { CodecUtil.CheckFooter(input); } else { CodecUtil.CheckEOF(input); } SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics); si.Files = files; success = true; return si; } finally { if (!success) { IOUtils.CloseWhileHandlingException(input); } else { input.Dispose(); } } }
/// <summary> /// Sole constructor. </summary> public Lucene40StoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) { string segment = si.Name; bool success = false; FieldInfos = fn; try { FieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context); string indexStreamFN = IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION); IndexStream = d.OpenInput(indexStreamFN, context); CodecUtil.CheckHeader(IndexStream, Lucene40StoredFieldsWriter.CODEC_NAME_IDX, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT); CodecUtil.CheckHeader(FieldsStream, Lucene40StoredFieldsWriter.CODEC_NAME_DAT, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT); Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_DAT == FieldsStream.FilePointer); Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX == IndexStream.FilePointer); long indexSize = IndexStream.Length() - Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX; this.Size_Renamed = (int)(indexSize >> 3); // Verify two sources of "maxDoc" agree: if (this.Size_Renamed != si.DocCount) { throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.Size_Renamed + " but segmentInfo shows " + si.DocCount); } NumTotalDocs = (int)(indexSize >> 3); success = true; } finally { // With lock-less commits, it's entirely possible (and // fine) to hit a FileNotFound exception above. In // this case, we want to explicitly close any subset // of things that were opened so that we don't have to // wait for a GC to do so. if (!success) { try { Dispose(); } // ensure we throw our original exception catch (Exception) { } } } }
/// <summary> /// Save a single segment's info. </summary> public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION); si.AddFile(fileName); IndexOutput output = dir.CreateOutput(fileName, ioContext); bool success = false; try { CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT); // Write the Lucene version that created this segment, since 3.1 output.WriteString(si.Version); output.WriteInt32(si.DocCount); output.WriteByte((byte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO)); output.WriteStringStringMap(si.Diagnostics); output.WriteStringSet(si.GetFiles()); CodecUtil.WriteFooter(output); success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(output); si.Dir.DeleteFile(fileName); } else { output.Dispose(); } } }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return(new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context, formatName, compressionMode, chunkSize)); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { throw new System.NotSupportedException("this codec can only be used for reading"); }
/// <summary> /// Returns a <seealso cref="StoredFieldsWriter"/> to write stored /// fields. /// </summary> public abstract StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context);
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return new AssertingTermVectorsWriter(@in.VectorsWriter(directory, segmentInfo, context)); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return new CompressingStoredFieldsWriter(directory, si, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize); }
/// <summary> /// Returns a <seealso cref="TermVectorsWriter"/> to write term /// vectors. /// </summary> public abstract TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context);
internal readonly PackedInts.Reader[] StartPointersDeltas; // delta from the avg #endregion Fields #region Constructors // It is the responsibility of the caller to close fieldsIndexIn after this constructor // has been called internal CompressingStoredFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) { MaxDoc = si.DocCount; int[] docBases = new int[16]; long[] startPointers = new long[16]; int[] avgChunkDocs = new int[16]; long[] avgChunkSizes = new long[16]; PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16]; PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16]; int packedIntsVersion = fieldsIndexIn.ReadVInt(); int blockCount = 0; for (; ; ) { int numChunks = fieldsIndexIn.ReadVInt(); if (numChunks == 0) { break; } if (blockCount == docBases.Length) { int newSize = ArrayUtil.Oversize(blockCount + 1, 8); docBases = Arrays.CopyOf(docBases, newSize); startPointers = Arrays.CopyOf(startPointers, newSize); avgChunkDocs = Arrays.CopyOf(avgChunkDocs, newSize); avgChunkSizes = Arrays.CopyOf(avgChunkSizes, newSize); docBasesDeltas = Arrays.CopyOf(docBasesDeltas, newSize); startPointersDeltas = Arrays.CopyOf(startPointersDeltas, newSize); } // doc bases docBases[blockCount] = fieldsIndexIn.ReadVInt(); avgChunkDocs[blockCount] = fieldsIndexIn.ReadVInt(); int bitsPerDocBase = fieldsIndexIn.ReadVInt(); if (bitsPerDocBase > 32) { throw new CorruptIndexException("Corrupted bitsPerDocBase (resource=" + fieldsIndexIn + ")"); } docBasesDeltas[blockCount] = PackedInts.GetReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerDocBase); // start pointers startPointers[blockCount] = fieldsIndexIn.ReadVLong(); avgChunkSizes[blockCount] = fieldsIndexIn.ReadVLong(); int bitsPerStartPointer = fieldsIndexIn.ReadVInt(); if (bitsPerStartPointer > 64) { throw new CorruptIndexException("Corrupted bitsPerStartPointer (resource=" + fieldsIndexIn + ")"); } startPointersDeltas[blockCount] = PackedInts.GetReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerStartPointer); ++blockCount; } this.DocBases = Arrays.CopyOf(docBases, blockCount); this.StartPointers = Arrays.CopyOf(startPointers, blockCount); this.AvgChunkDocs = Arrays.CopyOf(avgChunkDocs, blockCount); this.AvgChunkSizes = Arrays.CopyOf(avgChunkSizes, blockCount); this.DocBasesDeltas = Arrays.CopyOf(docBasesDeltas, blockCount); this.StartPointersDeltas = Arrays.CopyOf(startPointersDeltas, blockCount); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return new Lucene40StoredFieldsReader(directory, si, fn, context); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return new Lucene40StoredFieldsWriter(directory, si.Name, context); }
/// <summary> /// Write <seealso cref="SegmentInfo"/> data. </summary> /// <exception cref="IOException"> If an I/O error occurs </exception> public abstract void Write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext);
public override sealed TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return new CompressingTermVectorsWriter(directory, segmentInfo, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize); }
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return(new Lucene40TermVectorsWriter(directory, segmentInfo.Name, context)); }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return(new Lucene40TermVectorsReader(directory, segmentInfo, fieldInfos, context)); }
public static void Files(Directory dir, SegmentInfo info, ISet <string> files) { files.Add(IndexFileNames.SegmentFileName(info.Name, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION)); }
/// <summary> /// Returns a <seealso cref="StoredFieldsReader"/> to load stored /// fields. /// </summary> public abstract StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context);
public override sealed TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return new CompressingTermVectorsReader(directory, segmentInfo, SegmentSuffix, fieldInfos, context, FormatName, CompressionMode); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return new AssertingStoredFieldsReader(@in.FieldsReader(directory, si, fn, context), si.DocCount); }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return new Lucene3xTermVectorsReaderAnonymousInnerClassHelper(this, directory, segmentInfo, fieldInfos, context); }
/// <summary> /// Returns a <see cref="TermVectorsWriter"/> to write term /// vectors. /// </summary> public abstract TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context);
public Lucene3xTermVectorsReaderAnonymousInnerClassHelper(PreFlexRWTermVectorsFormat outerInstance, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) : base(directory, segmentInfo, fieldInfos, context) { this.OuterInstance = outerInstance; }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return new AssertingTermVectorsReader(@in.VectorsReader(directory, segmentInfo, fieldInfos, context)); }
/// <returns> whether doc store files are stored in compound file (*.cfx) </returns> public static bool GetDocStoreIsCompoundFile(SegmentInfo si) { string v = si.GetAttribute(DS_COMPOUND_KEY); return v == null ? false : Convert.ToBoolean(v); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return new CompressingStoredFieldsReader(directory, si, SegmentSuffix, fn, context, FormatName, CompressionMode); }
// NOTE: this is not "really" 3.x format, because we are // writing each SI to its own file, vs 3.x where the list // of segments and SI for each segment is written into a // single segments_N file /// <summary> /// Save a single segment's info. </summary> public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { SegmentInfos.Write3xInfo(dir, si, ioContext); }
public override sealed TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return(new CompressingTermVectorsReader(directory, segmentInfo, SegmentSuffix, fieldInfos, context, FormatName, CompressionMode)); }
public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) { return new Lucene40TermVectorsReader(directory, segmentInfo, fieldInfos, context); }
public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return new PreFlexRWTermVectorsWriter(directory, segmentInfo.Name, context); }
public override sealed TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return(new CompressingTermVectorsWriter(directory, segmentInfo, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize)); }
/// <returns> name used to derive fields/vectors file we share with other segments </returns> public static string GetDocStoreSegment(SegmentInfo si) { string v = si.GetAttribute(DS_NAME_KEY); return v == null ? si.Name : v; }
/// <summary> /// Write <see cref="SegmentInfo"/> data. </summary> /// <exception cref="System.IO.IOException"> If an I/O error occurs. </exception> public abstract void Write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext);
/// <returns> if this segment shares stored fields & vectors, this /// offset is where in that file this segment's docs begin </returns> public static int GetDocStoreOffset(SegmentInfo si) { string v = si.GetAttribute(DS_OFFSET_KEY); return v == null ? -1 : Convert.ToInt32(v); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return(new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context)); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) { return new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return(new Lucene40StoredFieldsReader(directory, si, fn, context)); }
public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { return(new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, context, formatName, compressionMode)); }
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return(new Lucene40StoredFieldsWriter(directory, si.Name, context)); }
/// <summary> /// Returns a <see cref="TermVectorsReader"/> to read term /// vectors. /// </summary> public abstract TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context);
public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context) { return new AssertingStoredFieldsWriter(@in.FieldsWriter(directory, si, context)); }
/// <summary> /// Returns a <seealso cref="TermVectorsReader"/> to read term /// vectors. /// </summary> public abstract TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context);