public BloomFilteredFieldsProducer(BloomFilteringPostingsFormat outerInstance, SegmentReadState state)
            {
                this.outerInstance = outerInstance;
                var bloomFileName = IndexFileNames.SegmentFileName(
                    state.SegmentInfo.Name, state.SegmentSuffix, BLOOM_EXTENSION);
                ChecksumIndexInput bloomIn = null;
                var success = false;

                try
                {
                    bloomIn = state.Directory.OpenChecksumInput(bloomFileName, state.Context);
                    var version = CodecUtil.CheckHeader(bloomIn, /*BLOOM_CODEC_NAME*/ outerInstance.Name, VERSION_START, VERSION_CURRENT);
                    // Load the hash function used in the BloomFilter
                    // hashFunction = HashFunction.forName(bloomIn.readString());
                    // Load the delegate postings format
                    var delegatePostingsFormat = ForName(bloomIn.ReadString());

                    _delegateFieldsProducer = delegatePostingsFormat
                                              .FieldsProducer(state);
                    var numBlooms = bloomIn.ReadInt32();
                    for (var i = 0; i < numBlooms; i++)
                    {
                        var fieldNum  = bloomIn.ReadInt32();
                        var bloom     = FuzzySet.Deserialize(bloomIn);
                        var fieldInfo = state.FieldInfos.FieldInfo(fieldNum);
                        _bloomsByFieldName.Add(fieldInfo.Name, bloom);
                    }

                    if (version >= VERSION_CHECKSUM)
                    {
                        CodecUtil.CheckFooter(bloomIn);
                    }
                    else
                    {
#pragma warning disable 612, 618
                        CodecUtil.CheckEOF(bloomIn);
#pragma warning restore 612, 618
                    }

                    IOUtils.Dispose(bloomIn);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        IOUtils.DisposeWhileHandlingException(bloomIn, _delegateFieldsProducer);
                    }
                }
            }
Пример #2
0
        public override SegmentInfo Read(Directory dir, string segment, IOContext context)
        {
            string             fileName = IndexFileNames.SegmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
            ChecksumIndexInput input    = dir.OpenChecksumInput(fileName, context);
            bool success = false;

            try
            {
                int    codecVersion = CodecUtil.CheckHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_START, Lucene46SegmentInfoFormat.VERSION_CURRENT);
                string version      = input.ReadString();
                int    docCount     = input.ReadInt32();
                if (docCount < 0)
                {
                    throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")");
                }
                bool isCompoundFile = input.ReadByte() == SegmentInfo.YES;
                IDictionary <string, string> diagnostics = input.ReadStringStringMap();
                ISet <string> files = input.ReadStringSet();

                if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM)
                {
                    CodecUtil.CheckFooter(input);
                }
                else
                {
#pragma warning disable 612, 618
                    CodecUtil.CheckEOF(input);
#pragma warning restore 612, 618
                }

                SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
                si.SetFiles(files);

                success = true;

                return(si);
            }
            finally
            {
                if (!success)
                {
                    IOUtils.DisposeWhileHandlingException(input);
                }
                else
                {
                    input.Dispose();
                }
            }
        }
Пример #3
0
        public override FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext context)
        {
            string             fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
            ChecksumIndexInput input    = directory.OpenChecksumInput(fileName, context);

            bool success = false;

            try
            {
                int codecVersion = CodecUtil.CheckHeader(input, Lucene46FieldInfosFormat.CODEC_NAME, Lucene46FieldInfosFormat.FORMAT_START, Lucene46FieldInfosFormat.FORMAT_CURRENT);

                int         size  = input.ReadVInt32(); //read in the size
                FieldInfo[] infos = new FieldInfo[size];

                for (int i = 0; i < size; i++)
                {
                    string       name            = input.ReadString();
                    int          fieldNumber     = input.ReadVInt32();
                    byte         bits            = input.ReadByte();
                    bool         isIndexed       = (bits & Lucene46FieldInfosFormat.IS_INDEXED) != 0;
                    bool         storeTermVector = (bits & Lucene46FieldInfosFormat.STORE_TERMVECTOR) != 0;
                    bool         omitNorms       = (bits & Lucene46FieldInfosFormat.OMIT_NORMS) != 0;
                    bool         storePayloads   = (bits & Lucene46FieldInfosFormat.STORE_PAYLOADS) != 0;
                    IndexOptions indexOptions;
                    if (!isIndexed)
                    {
                        indexOptions = IndexOptions.NONE;
                    }
                    else if ((bits & Lucene46FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0)
                    {
                        indexOptions = IndexOptions.DOCS_ONLY;
                    }
                    else if ((bits & Lucene46FieldInfosFormat.OMIT_POSITIONS) != 0)
                    {
                        indexOptions = IndexOptions.DOCS_AND_FREQS;
                    }
                    else if ((bits & Lucene46FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0)
                    {
                        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
                    }
                    else
                    {
                        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
                    }

                    // DV Types are packed in one byte
                    byte          val                       = input.ReadByte();
                    DocValuesType docValuesType             = GetDocValuesType(input, (byte)(val & 0x0F));
                    DocValuesType normsType                 = GetDocValuesType(input, (byte)((val.TripleShift(4)) & 0x0F));
                    long          dvGen                     = input.ReadInt64();
                    IDictionary <string, string> attributes = input.ReadStringStringMap();
                    infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normsType, attributes.AsReadOnly());
                    infos[i].DocValuesGen = dvGen;
                }

                if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM)
                {
                    CodecUtil.CheckFooter(input);
                }
                else
                {
#pragma warning disable 612, 618
                    CodecUtil.CheckEOF(input);
#pragma warning restore 612, 618
                }
                FieldInfos fieldInfos = new FieldInfos(infos);
                success = true;
                return(fieldInfos);
            }
            finally
            {
                if (success)
                {
                    input.Dispose();
                }
                else
                {
                    IOUtils.DisposeWhileHandlingException(input);
                }
            }
        }
Пример #4
0
        /// <summary> Read a particular segmentFileName.  Note that this may
        /// throw an IOException if a commit is in process.
        ///
        /// </summary>
        /// <param name="directory">-- directory containing the segments file
        /// </param>
        /// <param name="segmentFileName">-- segment file to load
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public void  Read(Directory directory, System.String segmentFileName)
        {
            bool success = false;

            // Clear any previous segments:
            Clear();

            var input = new ChecksumIndexInput(directory.OpenInput(segmentFileName));

            generation = GenerationFromSegmentsFileName(segmentFileName);

            lastGeneration = generation;

            try
            {
                int format = input.ReadInt();
                if (format < 0)
                {
                    // file contains explicit format info
                    // check that it is a format we can understand
                    if (format < CURRENT_FORMAT)
                    {
                        throw new CorruptIndexException("Unknown format version: " + format);
                    }
                    version = input.ReadLong();                    // read version
                    counter = input.ReadInt();                     // read counter
                }
                else
                {
                    // file is in old format without explicit format info
                    counter = format;
                }

                for (int i = input.ReadInt(); i > 0; i--)
                {
                    // read segmentInfos
                    Add(new SegmentInfo(directory, format, input));
                }

                if (format >= 0)
                {
                    // in old format the version number may be at the end of the file
                    if (input.FilePointer >= input.Length())
                    {
                        version = (DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond);
                    }
                    // old file format without version number
                    else
                    {
                        version = input.ReadLong();                         // read version
                    }
                }

                if (format <= FORMAT_USER_DATA)
                {
                    if (format <= FORMAT_DIAGNOSTICS)
                    {
                        userData = input.ReadStringStringMap();
                    }
                    else if (0 != input.ReadByte())
                    {
                        // TODO: Should be read-only map
                        userData = new HashMap <string, string> {
                            { "userData", input.ReadString() }
                        };
                    }
                    else
                    {
                        // TODO: Should be empty read-only map
                        userData = new HashMap <string, string>();
                    }
                }
                else
                {
                    // TODO: Should be empty read-only map
                    userData = new HashMap <string, string>();
                }

                if (format <= FORMAT_CHECKSUM)
                {
                    long checksumNow  = input.Checksum;
                    long checksumThen = input.ReadLong();
                    if (checksumNow != checksumThen)
                    {
                        throw new CorruptIndexException("checksum mismatch in segments file");
                    }
                }
                success = true;
            }
            finally
            {
                input.Close();
                if (!success)
                {
                    // Clear any segment infos we had loaded so we
                    // have a clean slate on retry:
                    Clear();
                }
            }
        }