/// <summary>
        /// Save a single segment's info. </summary>
        public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
        {
            string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
            si.AddFile(fileName);

            IndexOutput output = dir.CreateOutput(fileName, ioContext);

            bool success = false;
            try
            {
                CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT);
                // Write the Lucene version that created this segment, since 3.1
                output.WriteString(si.Version);
                output.WriteInt(si.DocCount);

                output.WriteByte((byte)(sbyte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO));
                output.WriteStringStringMap(si.Diagnostics);
                output.WriteStringSet(si.Files);
                CodecUtil.WriteFooter(output);
                success = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(output);
                    si.Dir.DeleteFile(fileName);
                }
                else
                {
                    output.Dispose();
                }
            }
        }
        public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
        {
            string fileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION);

            // Unfortunately, for 3.x indices, each segment's
            // FieldInfos can lie about hasVectors (claim it's true
            // when really it's false).... so we have to carefully
            // check if the files really exist before trying to open
            // them (4.x has fixed this):
            bool exists;
            if (Lucene3xSegmentInfoFormat.GetDocStoreOffset(segmentInfo) != -1 && Lucene3xSegmentInfoFormat.GetDocStoreIsCompoundFile(segmentInfo))
            {
                string cfxFileName = IndexFileNames.SegmentFileName(Lucene3xSegmentInfoFormat.GetDocStoreSegment(segmentInfo), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION);
                if (segmentInfo.Dir.FileExists(cfxFileName))
                {
                    Directory cfsDir = new CompoundFileDirectory(segmentInfo.Dir, cfxFileName, context, false);
                    try
                    {
                        exists = cfsDir.FileExists(fileName);
                    }
                    finally
                    {
                        cfsDir.Dispose();
                    }
                }
                else
                {
                    exists = false;
                }
            }
            else
            {
                exists = directory.FileExists(fileName);
            }

            if (!exists)
            {
                // 3x's FieldInfos sometimes lies and claims a segment
                // has vectors when it doesn't:
                return null;
            }
            else
            {
                return new Lucene3xTermVectorsReader(directory, segmentInfo, fieldInfos, context);
            }
        }
        public override SegmentInfo Read(Directory dir, string segment, IOContext context)
        {
            string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
            ChecksumIndexInput input = dir.OpenChecksumInput(fileName, context);
            bool success = false;
            try
            {
                int codecVersion = CodecUtil.CheckHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_START, Lucene46SegmentInfoFormat.VERSION_CURRENT);
                string version = input.ReadString();
                int docCount = input.ReadInt();
                if (docCount < 0)
                {
                    throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")");
                }
                bool isCompoundFile = input.ReadByte() == SegmentInfo.YES;
                IDictionary<string, string> diagnostics = input.ReadStringStringMap();
                ISet<string> files = input.ReadStringSet();

                if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM)
                {
                    CodecUtil.CheckFooter(input);
                }
                else
                {
                    CodecUtil.CheckEOF(input);
                }

                SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
                si.Files = files;

                success = true;

                return si;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(input);
                }
                else
                {
                    input.Dispose();
                }
            }
        }
        /// <summary>
        /// Sole constructor. </summary>
        public Lucene40StoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context)
        {
            string segment = si.Name;
            bool success = false;
            FieldInfos = fn;
            try
            {
                FieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context);
                string indexStreamFN = IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
                IndexStream = d.OpenInput(indexStreamFN, context);

                CodecUtil.CheckHeader(IndexStream, Lucene40StoredFieldsWriter.CODEC_NAME_IDX, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
                CodecUtil.CheckHeader(FieldsStream, Lucene40StoredFieldsWriter.CODEC_NAME_DAT, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
                Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_DAT == FieldsStream.FilePointer);
                Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX == IndexStream.FilePointer);
                long indexSize = IndexStream.Length() - Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX;
                this.Size_Renamed = (int)(indexSize >> 3);
                // Verify two sources of "maxDoc" agree:
                if (this.Size_Renamed != si.DocCount)
                {
                    throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.Size_Renamed + " but segmentInfo shows " + si.DocCount);
                }
                NumTotalDocs = (int)(indexSize >> 3);
                success = true;
            }
            finally
            {
                // With lock-less commits, it's entirely possible (and
                // fine) to hit a FileNotFound exception above. In
                // this case, we want to explicitly close any subset
                // of things that were opened so that we don't have to
                // wait for a GC to do so.
                if (!success)
                {
                    try
                    {
                        Dispose();
                    } // ensure we throw our original exception
                    catch (Exception)
                    {
                    }
                }
            }
        }
Esempio n. 5
0
        /// <summary>
        /// Save a single segment's info. </summary>
        public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
        {
            string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION);

            si.AddFile(fileName);

            IndexOutput output = dir.CreateOutput(fileName, ioContext);

            bool success = false;

            try
            {
                CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT);
                // Write the Lucene version that created this segment, since 3.1
                output.WriteString(si.Version);
                output.WriteInt32(si.DocCount);

                output.WriteByte((byte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO));
                output.WriteStringStringMap(si.Diagnostics);
                output.WriteStringSet(si.GetFiles());
                CodecUtil.WriteFooter(output);
                success = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.DisposeWhileHandlingException(output);
                    si.Dir.DeleteFile(fileName);
                }
                else
                {
                    output.Dispose();
                }
            }
        }
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
 {
     return(new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context, formatName, compressionMode, chunkSize));
 }
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
 {
     throw new System.NotSupportedException("this codec can only be used for reading");
 }
Esempio n. 8
0
 /// <summary>
 /// Returns a <seealso cref="StoredFieldsWriter"/> to write stored
 ///  fields.
 /// </summary>
 public abstract StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context);
 public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
 {
     return new AssertingTermVectorsWriter(@in.VectorsWriter(directory, segmentInfo, context));
 }
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
 {
     return new CompressingStoredFieldsWriter(directory, si, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize);
 }
Esempio n. 11
0
 /// <summary>
 /// Returns a <seealso cref="TermVectorsWriter"/> to write term
 ///  vectors.
 /// </summary>
 public abstract TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context);
        internal readonly PackedInts.Reader[] StartPointersDeltas; // delta from the avg

        #endregion Fields

        #region Constructors

        // It is the responsibility of the caller to close fieldsIndexIn after this constructor
        // has been called
        internal CompressingStoredFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si)
        {
            MaxDoc = si.DocCount;
            int[] docBases = new int[16];
            long[] startPointers = new long[16];
            int[] avgChunkDocs = new int[16];
            long[] avgChunkSizes = new long[16];
            PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16];
            PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16];

            int packedIntsVersion = fieldsIndexIn.ReadVInt();

            int blockCount = 0;

            for (; ; )
            {
                int numChunks = fieldsIndexIn.ReadVInt();
                if (numChunks == 0)
                {
                    break;
                }
                if (blockCount == docBases.Length)
                {
                    int newSize = ArrayUtil.Oversize(blockCount + 1, 8);
                    docBases = Arrays.CopyOf(docBases, newSize);
                    startPointers = Arrays.CopyOf(startPointers, newSize);
                    avgChunkDocs = Arrays.CopyOf(avgChunkDocs, newSize);
                    avgChunkSizes = Arrays.CopyOf(avgChunkSizes, newSize);
                    docBasesDeltas = Arrays.CopyOf(docBasesDeltas, newSize);
                    startPointersDeltas = Arrays.CopyOf(startPointersDeltas, newSize);
                }

                // doc bases
                docBases[blockCount] = fieldsIndexIn.ReadVInt();
                avgChunkDocs[blockCount] = fieldsIndexIn.ReadVInt();
                int bitsPerDocBase = fieldsIndexIn.ReadVInt();
                if (bitsPerDocBase > 32)
                {
                    throw new CorruptIndexException("Corrupted bitsPerDocBase (resource=" + fieldsIndexIn + ")");
                }
                docBasesDeltas[blockCount] = PackedInts.GetReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerDocBase);

                // start pointers
                startPointers[blockCount] = fieldsIndexIn.ReadVLong();
                avgChunkSizes[blockCount] = fieldsIndexIn.ReadVLong();
                int bitsPerStartPointer = fieldsIndexIn.ReadVInt();
                if (bitsPerStartPointer > 64)
                {
                    throw new CorruptIndexException("Corrupted bitsPerStartPointer (resource=" + fieldsIndexIn + ")");
                }
                startPointersDeltas[blockCount] = PackedInts.GetReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerStartPointer);

                ++blockCount;
            }

            this.DocBases = Arrays.CopyOf(docBases, blockCount);
            this.StartPointers = Arrays.CopyOf(startPointers, blockCount);
            this.AvgChunkDocs = Arrays.CopyOf(avgChunkDocs, blockCount);
            this.AvgChunkSizes = Arrays.CopyOf(avgChunkSizes, blockCount);
            this.DocBasesDeltas = Arrays.CopyOf(docBasesDeltas, blockCount);
            this.StartPointersDeltas = Arrays.CopyOf(startPointersDeltas, blockCount);
        }
 public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
 {
     return new Lucene40StoredFieldsReader(directory, si, fn, context);
 }
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
 {
     return new Lucene40StoredFieldsWriter(directory, si.Name, context);
 }
Esempio n. 15
0
 /// <summary>
 /// Write <seealso cref="SegmentInfo"/> data. </summary>
 /// <exception cref="IOException"> If an I/O error occurs </exception>
 public abstract void Write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext);
 public override sealed TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
 {
     return new CompressingTermVectorsWriter(directory, segmentInfo, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize);
 }
Esempio n. 17
0
 public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
 {
     return(new Lucene40TermVectorsWriter(directory, segmentInfo.Name, context));
 }
Esempio n. 18
0
 public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
 {
     return(new Lucene40TermVectorsReader(directory, segmentInfo, fieldInfos, context));
 }
Esempio n. 19
0
 public static void Files(Directory dir, SegmentInfo info, ISet <string> files)
 {
     files.Add(IndexFileNames.SegmentFileName(info.Name, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION));
 }
Esempio n. 20
0
 /// <summary>
 /// Returns a <seealso cref="StoredFieldsReader"/> to load stored
 ///  fields.
 /// </summary>
 public abstract StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context);
 public override sealed TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
 {
     return new CompressingTermVectorsReader(directory, segmentInfo, SegmentSuffix, fieldInfos, context, FormatName, CompressionMode);
 }
 public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
 {
     return new AssertingStoredFieldsReader(@in.FieldsReader(directory, si, fn, context), si.DocCount);
 }
 public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
 {
     return new Lucene3xTermVectorsReaderAnonymousInnerClassHelper(this, directory, segmentInfo, fieldInfos, context);
 }
Esempio n. 24
0
 /// <summary>
 /// Returns a <see cref="TermVectorsWriter"/> to write term
 /// vectors.
 /// </summary>
 public abstract TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context);
 public Lucene3xTermVectorsReaderAnonymousInnerClassHelper(PreFlexRWTermVectorsFormat outerInstance, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
     : base(directory, segmentInfo, fieldInfos, context)
 {
     this.OuterInstance = outerInstance;
 }
 public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
 {
     return new AssertingTermVectorsReader(@in.VectorsReader(directory, segmentInfo, fieldInfos, context));
 }
 /// <returns> whether doc store files are stored in compound file (*.cfx) </returns>
 public static bool GetDocStoreIsCompoundFile(SegmentInfo si)
 {
     string v = si.GetAttribute(DS_COMPOUND_KEY);
     return v == null ? false : Convert.ToBoolean(v);
 }
 public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
 {
     return new CompressingStoredFieldsReader(directory, si, SegmentSuffix, fn, context, FormatName, CompressionMode);
 }
 // NOTE: this is not "really" 3.x format, because we are
 // writing each SI to its own file, vs 3.x where the list
 // of segments and SI for each segment is written into a
 // single segments_N file
 /// <summary>
 /// Save a single segment's info. </summary>
 public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
 {
     SegmentInfos.Write3xInfo(dir, si, ioContext);
 }
Esempio n. 30
0
 public override sealed TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
 {
     return(new CompressingTermVectorsReader(directory, segmentInfo, SegmentSuffix, fieldInfos, context, FormatName, CompressionMode));
 }
 public override TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
 {
     return new Lucene40TermVectorsReader(directory, segmentInfo, fieldInfos, context);
 }
 public override TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
 {
     return new PreFlexRWTermVectorsWriter(directory, segmentInfo.Name, context);
 }
Esempio n. 33
0
 public override sealed TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
 {
     return(new CompressingTermVectorsWriter(directory, segmentInfo, SegmentSuffix, context, FormatName, CompressionMode, ChunkSize));
 }
 /// <returns> name used to derive fields/vectors file we share with other segments </returns>
 public static string GetDocStoreSegment(SegmentInfo si)
 {
     string v = si.GetAttribute(DS_NAME_KEY);
     return v == null ? si.Name : v;
 }
Esempio n. 35
0
 /// <summary>
 /// Write <see cref="SegmentInfo"/> data. </summary>
 /// <exception cref="System.IO.IOException"> If an I/O error occurs. </exception>
 public abstract void Write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext);
 /// <returns> if this segment shares stored fields & vectors, this
 ///         offset is where in that file this segment's docs begin  </returns>
 public static int GetDocStoreOffset(SegmentInfo si)
 {
     string v = si.GetAttribute(DS_OFFSET_KEY);
     return v == null ? -1 : Convert.ToInt32(v);
 }
Esempio n. 37
0
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
 {
     return(new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context));
 }
Esempio n. 38
0
 /// <summary>
 /// Returns a <seealso cref="StoredFieldsReader"/> to load stored
 ///  fields.
 /// </summary>
 public abstract StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context);
Esempio n. 39
0
        // NOTE: this is not "really" 3.x format, because we are
        // writing each SI to its own file, vs 3.x where the list
        // of segments and SI for each segment is written into a
        // single segments_N file

        /// <summary>
        /// Save a single segment's info. </summary>
        public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
        {
            SegmentInfos.Write3xInfo(dir, si, ioContext);
        }
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
 {
     return new PreFlexRWStoredFieldsWriter(directory, segmentInfo.Name, context);
 }
Esempio n. 41
0
 public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
 {
     return(new Lucene40StoredFieldsReader(directory, si, fn, context));
 }
 public override StoredFieldsReader FieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
 {
     return(new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, context, formatName, compressionMode));
 }
Esempio n. 43
0
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
 {
     return(new Lucene40StoredFieldsWriter(directory, si.Name, context));
 }
Esempio n. 44
0
 /// <summary>
 /// Returns a <seealso cref="StoredFieldsWriter"/> to write stored
 ///  fields.
 /// </summary>
 public abstract StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context);
Esempio n. 45
0
 /// <summary>
 /// Returns a <see cref="TermVectorsReader"/> to read term
 /// vectors.
 /// </summary>
 public abstract TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context);
 public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
 {
     return new AssertingStoredFieldsWriter(@in.FieldsWriter(directory, si, context));
 }
Esempio n. 47
0
 /// <summary>
 /// Returns a <seealso cref="TermVectorsReader"/> to read term
 ///  vectors.
 /// </summary>
 public abstract TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context);