Abstract base class for input from a file in a {@link Directory}. A random-access input stream. Used for all Lucene index input operations.
Inheritance: System.ICloneable
Esempio n. 1
0
        public void  Read(IndexInput input, FieldInfos fieldInfos, IState state)
        {
            this.term = null; // invalidate cache
            int start       = input.ReadVInt(state);
            int length      = input.ReadVInt(state);
            int totalLength = start + length;

            if (preUTF8Strings)
            {
                text.SetLength(totalLength);
                input.ReadChars(text.result, start, length, state);
            }
            else
            {
                if (dirty)
                {
                    // Fully convert all bytes since bytes is dirty
                    UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes);
                    bytes.SetLength(totalLength);
                    input.ReadBytes(bytes.result, start, length, state);
                    UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text);
                    dirty = false;
                }
                else
                {
                    // Incrementally convert only the UTF8 bytes that are new:
                    bytes.SetLength(totalLength);
                    input.ReadBytes(bytes.result, start, length, state);
                    UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text);
                }
            }
            this.field = fieldInfos.FieldName(input.ReadVInt(state));
        }
Esempio n. 2
0
        protected internal override int ReadSkipData(int level, IndexInput skipStream, IState state)
        {
            int delta;

            if (currentFieldStoresPayloads)
            {
                // the current field stores payloads.
                // if the doc delta is odd then we have
                // to read the current payload length
                // because it differs from the length of the
                // previous payload
                delta = skipStream.ReadVInt(state);
                if ((delta & 1) != 0)
                {
                    payloadLength[level] = skipStream.ReadVInt(state);
                }
                delta = Number.URShift(delta, 1);
            }
            else
            {
                delta = skipStream.ReadVInt(state);
            }
            freqPointer[level] += skipStream.ReadVInt(state);
            proxPointer[level] += skipStream.ReadVInt(state);

            return(delta);
        }
 protected override MonotonicBlockPackedReader GetOrdIndexInstance(IndexInput data, FieldInfo field,
     NumericEntry entry)
 {
     data.Seek(entry.Offset);
     return new MonotonicBlockPackedReader((IndexInput)data.Clone(), entry.PackedIntsVersion, entry.BlockSize, entry.Count,
         true);
 }
 public MockSingleIntIndexInput(Directory dir, string fileName, IOContext context)
 {
     @in = dir.OpenInput(fileName, context);
     CodecUtil.CheckHeader(@in, MockSingleIntIndexOutput.CODEC,
                   MockSingleIntIndexOutput.VERSION_START,
                   MockSingleIntIndexOutput.VERSION_START);
 }
Esempio n. 5
0
        protected override void Dispose(bool disposing)
        {
            lock (this)
            {
                if (isDisposed)
                {
                    return;
                }
                if (disposing)
                {
                    if (entries != null)
                    {
                        entries.Clear();
                    }
                    if (stream != null)
                    {
                        stream.Close();
                    }
                }

                entries    = null;
                stream     = null;
                isDisposed = true;
            }
        }
Esempio n. 6
0
 /// <summary>
 /// Checks that the stream is positioned at the end, and throws exception
 /// if it is not. </summary>
 /// @deprecated Use <seealso cref="#checkFooter"/> instead, this should only used for files without checksums
 public static void CheckEOF(IndexInput @in)
 {
     if (@in.FilePointer != @in.Length())
     {
         throw new System.IO.IOException("did not read all bytes from file: read " + @in.FilePointer + " vs size " + @in.Length() + " (resource: " + @in + ")");
     }
 }
Esempio n. 7
0
        private int CopyFieldsNoDeletions(FieldsWriter fieldsWriter, IndexReader reader, FieldsReader matchingFieldsReader, IState state)
        {
            int maxDoc   = reader.MaxDoc;
            int docCount = 0;

            if (matchingFieldsReader != null)
            {
                // We can bulk-copy because the fieldInfos are "congruent"
                while (docCount < maxDoc)
                {
                    int        len    = System.Math.Min(MAX_RAW_MERGE_DOCS, maxDoc - docCount);
                    IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, docCount, len, state);
                    fieldsWriter.AddRawDocuments(stream, rawDocLengths, len, state);
                    docCount += len;
                    checkAbort.Work(300 * len, state);
                }
            }
            else
            {
                for (; docCount < maxDoc; docCount++)
                {
                    // NOTE: it's very important to first assign to doc then pass it to
                    // termVectorsWriter.addAllDocVectors; see LUCENE-1282
                    Document doc = reader.Document(docCount, state);
                    fieldsWriter.AddDocument(doc, state);
                    checkAbort.Work(300, state);
                }
            }
            return(docCount);
        }
 /// <summary>
 /// Construct an empty output buffer. </summary>
 public MockIndexInputWrapper(MockDirectoryWrapper dir, string name, IndexInput @delegate)
     : base("MockIndexInputWrapper(name=" + name + " delegate=" + @delegate + ")")
 {
     this.Name = name;
     this.Dir = dir;
     this.@delegate = @delegate;
 }
 protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field,
     BinaryEntry bytes)
 {
     data.Seek(bytes.AddressesOffset);
     return new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count,
         true);
 }
Esempio n. 10
0
        private int CopyFieldsWithDeletions(FieldsWriter fieldsWriter, IndexReader reader, FieldsReader matchingFieldsReader, IState state)
        {
            int docCount = 0;
            int maxDoc   = reader.MaxDoc;

            if (matchingFieldsReader != null)
            {
                // We can bulk-copy because the fieldInfos are "congruent"
                for (int j = 0; j < maxDoc;)
                {
                    if (reader.IsDeleted(j))
                    {
                        // skip deleted docs
                        ++j;
                        continue;
                    }
                    // We can optimize this case (doing a bulk byte copy) since the field
                    // numbers are identical
                    int start = j, numDocs = 0;
                    do
                    {
                        j++;
                        numDocs++;
                        if (j >= maxDoc)
                        {
                            break;
                        }
                        if (reader.IsDeleted(j))
                        {
                            j++;
                            break;
                        }
                    }while (numDocs < MAX_RAW_MERGE_DOCS);

                    IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, start, numDocs, state);
                    fieldsWriter.AddRawDocuments(stream, rawDocLengths, numDocs, state);
                    docCount += numDocs;
                    checkAbort.Work(300 * numDocs, state);
                }
            }
            else
            {
                for (int j = 0; j < maxDoc; j++)
                {
                    if (reader.IsDeleted(j))
                    {
                        // skip deleted docs
                        continue;
                    }
                    // NOTE: it's very important to first assign to doc then pass it to
                    // termVectorsWriter.addAllDocVectors; see LUCENE-1282
                    Document doc = reader.Document(j, state);
                    fieldsWriter.AddDocument(doc, state);
                    docCount++;
                    checkAbort.Work(300, state);
                }
            }
            return(docCount);
        }
Esempio n. 11
0
        private int CheckValidFormat(IndexInput in_Renamed, IState state)
        {
            int format = in_Renamed.ReadInt(state);

            if (format > FORMAT_CURRENT)
            {
                throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FORMAT_CURRENT + " or less");
            }
            return(format);
        }
Esempio n. 12
0
		/// <summary>Returns true if the provided input stream is an SimpleFSIndexInput and
		/// is a clone, that is it does not own its underlying file descriptor.
		/// </summary>
		public static bool IsSimpleFSIndexInputClone(IndexInput is_Renamed)
		{
			if (IsSimpleFSIndexInput(is_Renamed))
			{
				return ((SimpleFSIndexInput) is_Renamed).isClone_ForNUnit;
			}
			else
			{
				return false;
			}
		}
Esempio n. 13
0
        public CompoundFileReader(Directory dir, System.String name, int readBufferSize, IState state)
        {
            directory           = dir;
            fileName            = name;
            this.readBufferSize = readBufferSize;

            bool success = false;

            try
            {
                stream = dir.OpenInput(name, readBufferSize, state);

                // read the directory and init files
                int       count = stream.ReadVInt(state);
                FileEntry entry = null;
                for (int i = 0; i < count; i++)
                {
                    long          offset = stream.ReadLong(state);
                    System.String id     = stream.ReadString(state);

                    if (entry != null)
                    {
                        // set length of the previous entry
                        entry.length = offset - entry.offset;
                    }

                    entry = new FileEntry {
                        offset = offset
                    };
                    entries[id] = entry;
                }

                // set the length of the final entry
                if (entry != null)
                {
                    entry.length = stream.Length(state) - entry.offset;
                }

                success = true;
            }
            finally
            {
                if (!success && (stream != null))
                {
                    try
                    {
                        stream.Close();
                    }
                    catch (System.IO.IOException)
                    {
                    }
                }
            }
        }
Esempio n. 14
0
 /// <summary>
 /// Returns true if the provided input stream is an SimpleFSIndexInput and
 ///  is a clone, that is it does not own its underlying file descriptor.
 /// </summary>
 public static bool IsSimpleFSIndexInputClone(IndexInput @is)
 {
     if (IsSimpleFSIndexInput(@is))
     {
         return ((SimpleFSIndexInput)@is).IsClone;
     }
     else
     {
         return false;
     }
 }
Esempio n. 15
0
            public override IndexInput OpenInput(System.String name, IState state)
            {
                IndexInput ii = base.OpenInput(name, null);

                if (name.EndsWith(".prx"))
                {
                    // we decorate the proxStream with a wrapper class that allows to count the number of calls of seek()
                    ii = new SeeksCountingStream(enclosingInstance, ii);
                }
                return(ii);
            }
Esempio n. 16
0
            private IndexInput GetFieldStream(IState state)
            {
                IndexInput localFieldsStream = Enclosing_Instance.fieldsStreamTL.Get(state);

                if (localFieldsStream == null)
                {
                    localFieldsStream = (IndexInput)Enclosing_Instance.cloneableFieldsStream.Clone(state);
                    Enclosing_Instance.fieldsStreamTL.Set(localFieldsStream);
                }
                return(localFieldsStream);
            }
Esempio n. 17
0
 public /*protected internal*/ SegmentTermDocs(SegmentReader parent, IState state)
 {
     this.parent     = parent;
     this.freqStream = (IndexInput)parent.core.freqStream.Clone(state);
     lock (parent)
     {
         this.deletedDocs = parent.deletedDocs;
     }
     this.skipInterval  = parent.core.GetTermsReader().SkipInterval;
     this.maxSkipLevels = parent.core.GetTermsReader().MaxSkipLevels;
 }
Esempio n. 18
0
 /// <summary>
 /// Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns
 ///  true if the underlying file descriptor is valid, and false otherwise.
 ///  this can be used to determine if the OS file has been closed.
 ///  The descriptor becomes invalid when the non-clone instance of the
 ///  SimpleFSIndexInput that owns this descriptor is closed. However, the
 ///  descriptor may possibly become invalid in other ways as well.
 /// </summary>
 public static bool IsSimpleFSIndexInputOpen(IndexInput @is)
 {
     if (IsSimpleFSIndexInput(@is))
     {
         SimpleFSIndexInput fis = (SimpleFSIndexInput)@is;
         return fis.FDValid;
     }
     else
     {
         return false;
     }
 }
Esempio n. 19
0
		/// <summary>Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns
		/// true if the underlying file descriptor is valid, and false otherwise.
		/// This can be used to determine if the OS file has been closed.
		/// The descriptor becomes invalid when the non-clone instance of the
		/// SimpleFSIndexInput that owns this descriptor is closed. However, the
		/// descriptor may possibly become invalid in other ways as well.
		/// </summary>
		public static bool IsSimpleFSIndexInputOpen(IndexInput is_Renamed)
		{
			if (IsSimpleFSIndexInput(is_Renamed))
			{
				SimpleFSIndexInput fis = (SimpleFSIndexInput) is_Renamed;
				return fis.IsFDValid();
			}
			else
			{
				return false;
			}
		}
Esempio n. 20
0
        /// <summary>Bulk write a contiguous series of documents.  The
        /// lengths array is the length (in bytes) of each raw
        /// document.  The stream IndexInput is the
        /// fieldsStream from which we should bulk-copy all
        /// bytes.
        /// </summary>
        internal void  AddRawDocuments(IndexInput stream, int[] lengths, int numDocs, IState state)
        {
            long position = fieldsStream.FilePointer;
            long start    = position;

            for (int i = 0; i < numDocs; i++)
            {
                indexStream.WriteLong(position);
                position += lengths[i];
            }
            fieldsStream.CopyBytes(stream, position - start, state);
            System.Diagnostics.Debug.Assert(fieldsStream.FilePointer == position);
        }
Esempio n. 21
0
 // Used only by clone
 private FieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int formatSize, int docStoreOffset, IndexInput cloneableFieldsStream, IndexInput cloneableIndexStream, IState state)
 {
     this.fieldInfos            = fieldInfos;
     this.numTotalDocs          = numTotalDocs;
     this.size                  = size;
     this.format                = format;
     this.formatSize            = formatSize;
     this.docStoreOffset        = docStoreOffset;
     this.cloneableFieldsStream = cloneableFieldsStream;
     this.cloneableIndexStream  = cloneableIndexStream;
     fieldsStream               = (IndexInput)cloneableFieldsStream.Clone(state);
     indexStream                = (IndexInput)cloneableIndexStream.Clone(state);
 }
Esempio n. 22
0
        /// <summary> Construct a FieldInfos object using the directory and the name of the file
        /// IndexInput
        /// </summary>
        /// <param name="d">The directory to open the IndexInput from
        /// </param>
        /// <param name="name">The name of the file to open the IndexInput from in the Directory
        /// </param>
        /// <throws>  IOException </throws>
        public /*internal*/ FieldInfos(Directory d, String name, IState state)
        {
            IndexInput input = d.OpenInput(name, state);

            try
            {
                try
                {
                    Read(input, name, state);
                }
                catch (System.IO.IOException)
                {
                    if (format == FORMAT_PRE)
                    {
                        // LUCENE-1623: FORMAT_PRE (before there was a
                        // format) may be 2.3.2 (pre-utf8) or 2.4.x (utf8)
                        // encoding; retry with input set to pre-utf8
                        input.Seek(0, state);
                        input.SetModifiedUTF8StringsMode();
                        byNumber.Clear();
                        byName.Clear();

                        bool rethrow = false;
                        try
                        {
                            Read(input, name, state);
                        }
                        catch (Exception)
                        {
                            // Ignore any new exception & set to throw original IOE
                            rethrow = true;
                        }
                        if (rethrow)
                        {
                            // Preserve stack trace
                            throw;
                        }
                    }
                    else
                    {
                        // The IOException cannot be caused by
                        // LUCENE-1623, so re-throw it
                        throw;
                    }
                }
            }
            finally
            {
                input.Close();
            }
        }
 // used by clone
 private CompressingTermVectorsReader(CompressingTermVectorsReader reader)
 {
     this.fieldInfos = reader.fieldInfos;
     this.vectorsStream = (IndexInput)reader.vectorsStream.Clone();
     this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone();
     this.packedIntsVersion = reader.packedIntsVersion;
     this.compressionMode = reader.compressionMode;
     this.decompressor = (Decompressor)reader.decompressor.Clone();
     this.chunkSize = reader.chunkSize;
     this.numDocs = reader.numDocs;
     this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
     this.version = reader.version;
     this.closed = false;
 }
Esempio n. 24
0
        private void  Read(IndexInput input, String fileName, IState state)
        {
            int firstInt = input.ReadVInt(state);

            if (firstInt < 0)
            {
                // This is a real format
                format = firstInt;
            }
            else
            {
                format = FORMAT_PRE;
            }

            if (format != FORMAT_PRE & format != FORMAT_START)
            {
                throw new CorruptIndexException("unrecognized format " + format + " in file \"" + fileName + "\"");
            }

            int size;

            if (format == FORMAT_PRE)
            {
                size = firstInt;
            }
            else
            {
                size = input.ReadVInt(state);                 //read in the size
            }

            for (int i = 0; i < size; i++)
            {
                String name            = StringHelper.Intern(input.ReadString(state));
                byte   bits            = input.ReadByte(state);
                bool   isIndexed       = (bits & IS_INDEXED) != 0;
                bool   storeTermVector = (bits & STORE_TERMVECTOR) != 0;
                bool   storePositionsWithTermVector = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
                bool   storeOffsetWithTermVector    = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
                bool   omitNorms                = (bits & OMIT_NORMS) != 0;
                bool   storePayloads            = (bits & STORE_PAYLOADS) != 0;
                bool   omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;

                AddInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
            }

            if (input.FilePointer(state) != input.Length(state))
            {
                throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.FilePointer(state) + " vs size " + input.Length(state));
            }
        }
Esempio n. 25
0
        /// <summary>Copy the contents of the file with specified extension into the
        /// provided output stream. Use the provided buffer for moving data
        /// to reduce memory allocation.
        /// </summary>
        private void  CopyFile(FileEntry source, IndexOutput os, byte[] buffer, IState state)
        {
            IndexInput isRenamed = null;

            try
            {
                long startPtr = os.FilePointer;

                isRenamed = directory.OpenInput(source.file, state);
                long length    = isRenamed.Length(state);
                long remainder = length;
                int  chunk     = buffer.Length;

                while (remainder > 0)
                {
                    var len = (int)Math.Min(chunk, remainder);
                    isRenamed.ReadBytes(buffer, 0, len, false, state);
                    os.WriteBytes(buffer, len);
                    remainder -= len;
                    if (checkAbort != null)
                    {
                        // Roughly every 2 MB we will check if
                        // it's time to abort
                        checkAbort.Work(80, state);
                    }
                }

                // Verify that remainder is 0
                if (remainder != 0)
                {
                    throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")");
                }

                // Verify that the output length diff is equal to original file
                long endPtr = os.FilePointer;
                long diff   = endPtr - startPtr;
                if (diff != length)
                {
                    throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length);
                }
            }
            finally
            {
                if (isRenamed != null)
                {
                    isRenamed.Close();
                }
            }
        }
Esempio n. 26
0
        /// <summary>read as a d-gaps list </summary>
        private void  ReadDgaps(IndexInput input, IState state)
        {
            size  = input.ReadInt(state);            // (re)read size
            count = input.ReadInt(state);            // read count
            bits  = new byte[(size >> 3) + 1];       // allocate bits
            int last = 0;
            int n    = Count();

            while (n > 0)
            {
                last      += input.ReadVInt(state);
                bits[last] = input.ReadByte(state);
                n         -= BYTE_COUNTS[bits[last] & 0xFF];
            }
        }
	    protected override void Dispose(bool disposing)
        {
            if (isDisposed) return;

            if (disposing)
            {
                if (main != null)
                {
                    main.Dispose();
                }
            }

            main = null;
            isDisposed = true;
        }
Esempio n. 28
0
 /// <summary>Copy numBytes bytes from input to ourself. </summary>
 public virtual void CopyBytes(IndexInput input, long numBytes)
 {
     long left = numBytes;
     if (copyBuffer == null)
         copyBuffer = new byte[COPY_BUFFER_SIZE];
     while (left > 0)
     {
         int toCopy;
         if (left > COPY_BUFFER_SIZE)
             toCopy = COPY_BUFFER_SIZE;
         else
             toCopy = (int) left;
         input.ReadBytes(copyBuffer, 0, toCopy);
         WriteBytes(copyBuffer, 0, toCopy);
         left -= toCopy;
     }
 }
Esempio n. 29
0
            protected override void Dispose(bool disposing)
            {
                if (isDisposed)
                {
                    return;
                }
                if (disposing)
                {
                    if (delegate_Renamed != null)
                    {
                        delegate_Renamed.Close();
                    }
                }

                delegate_Renamed = null;
                isDisposed       = true;
            }
Esempio n. 30
0
 /// <summary>
 /// Sole constructor. </summary>
 public BlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct)
 {
     this.ValueCount = valueCount;
     BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
     BlockMask = blockSize - 1;
     int numBlocks = PackedInts.NumBlocks(valueCount, blockSize);
     long[] minValues = null;
     SubReaders = new PackedInts.Reader[numBlocks];
     for (int i = 0; i < numBlocks; ++i)
     {
         int token = @in.ReadByte() & 0xFF;
         int bitsPerValue = (int)((uint)token >> AbstractBlockPackedWriter.BPV_SHIFT);
         if (bitsPerValue > 64)
         {
             throw new Exception("Corrupted");
         }
         if ((token & AbstractBlockPackedWriter.MIN_VALUE_EQUALS_0) == 0)
         {
             if (minValues == null)
             {
                 minValues = new long[numBlocks];
             }
             minValues[i] = BlockPackedReaderIterator.ZigZagDecode(1L + BlockPackedReaderIterator.ReadVLong(@in));
         }
         if (bitsPerValue == 0)
         {
             SubReaders[i] = new PackedInts.NullReader(blockSize);
         }
         else
         {
             int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize);
             if (direct)
             {
                 long pointer = @in.FilePointer;
                 SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
                 @in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue));
             }
             else
             {
                 SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
             }
         }
     }
     this.MinValues = minValues;
 }
Esempio n. 31
0
            /// <summary>The value of the field as a String, or null.  If null, the Reader value,
            /// binary value, or TokenStream value is used.  Exactly one of StringValue(),
            /// ReaderValue(), GetBinaryValue(), and TokenStreamValue() must be set.
            /// </summary>
            public override string StringValue(IState state)
            {
                Enclosing_Instance.EnsureOpen();
                if (internalIsBinary)
                {
                    return(null);
                }

                if (fieldsData == null)
                {
                    IndexInput localFieldsStream = GetFieldStream(state);
                    try
                    {
                        localFieldsStream.Seek(pointer, state);
                        if (isCompressed)
                        {
                            var b = new byte[toRead];
                            localFieldsStream.ReadBytes(b, 0, b.Length, state);
                            fieldsData =
                                System.Text.Encoding.GetEncoding("UTF-8").GetString(Enclosing_Instance.Uncompress(b));
                        }
                        else
                        {
                            if (Enclosing_Instance.format >= FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
                            {
                                var bytes = new byte[toRead];
                                localFieldsStream.ReadBytes(bytes, 0, toRead, state);
                                fieldsData = System.Text.Encoding.GetEncoding("UTF-8").GetString(bytes);
                            }
                            else
                            {
                                //read in chars b/c we already know the length we need to read
                                var chars = new char[toRead];
                                localFieldsStream.ReadChars(chars, 0, toRead, state);
                                fieldsData = new System.String(chars);
                            }
                        }
                    }
                    catch (System.IO.IOException e)
                    {
                        throw new FieldReaderException(e);
                    }
                }
                return((System.String)fieldsData);
            }
 protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval)
 {
     this.skipStream            = new IndexInput[maxSkipLevels];
     this.skipPointer           = new long[maxSkipLevels];
     this.childPointer          = new long[maxSkipLevels];
     this.numSkipped            = new int[maxSkipLevels];
     this.maxNumberOfSkipLevels = maxSkipLevels;
     this.skipInterval          = new int[maxSkipLevels];
     this.skipStream[0]         = skipStream;
     this.inputIsBuffered       = (skipStream is BufferedIndexInput);
     this.skipInterval[0]       = skipInterval;
     for (int i = 1; i < maxSkipLevels; i++)
     {
         // cache skip intervals
         this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval;
     }
     skipDoc = new int[maxSkipLevels];
 }
Esempio n. 33
0
 /// <summary>Copy numBytes bytes from input to ourself. </summary>
 public virtual void CopyBytes(IndexInput input, long numBytes)
 {
     System.Diagnostics.Debug.Assert(numBytes >= 0, "numBytes=" + numBytes);
     long left = numBytes;
     if (copyBuffer == null)
         copyBuffer = new byte[COPY_BUFFER_SIZE];
     while (left > 0)
     {
         int toCopy;
         if (left > COPY_BUFFER_SIZE)
             toCopy = COPY_BUFFER_SIZE;
         else
             toCopy = (int) left;
         input.ReadBytes(copyBuffer, 0, toCopy);
         WriteBytes(copyBuffer, 0, toCopy);
         left -= toCopy;
     }
 }
Esempio n. 34
0
        public override void Init(IndexInput termsIn)
        {
            version = CodecUtil.CheckHeader(termsIn, PulsingPostingsWriter.CODEC,
                PulsingPostingsWriter.VERSION_START,
                PulsingPostingsWriter.VERSION_CURRENT);

            maxPositions = termsIn.ReadVInt();
            _wrappedPostingsReader.Init(termsIn);

            if (_wrappedPostingsReader is PulsingPostingsReader || version < PulsingPostingsWriter.VERSION_META_ARRAY)
            {
                fields = null;
            }
            else
            {
                fields = new SortedDictionary<int, int>();
                String summaryFileName = IndexFileNames.SegmentFileName(segmentState.SegmentInfo.Name,
                    segmentState.SegmentSuffix, PulsingPostingsWriter.SUMMARY_EXTENSION);
                IndexInput input = null;

                try
                {
                    input =
                        segmentState.Directory.OpenInput(summaryFileName, segmentState.Context);
                    CodecUtil.CheckHeader(input,
                        PulsingPostingsWriter.CODEC,
                        version,
                        PulsingPostingsWriter.VERSION_CURRENT);

                    int numField = input.ReadVInt();
                    for (int i = 0; i < numField; i++)
                    {
                        int fieldNum = input.ReadVInt();
                        int longsSize = input.ReadVInt();
                        fields.Add(fieldNum, longsSize);
                    }
                }
                finally
                {
                    IOUtils.CloseWhileHandlingException(input);
                }
            }
        }
 public SlowOpeningMockIndexInputWrapper(MockDirectoryWrapper dir, string name, IndexInput @delegate)
     : base(dir, name, @delegate)
 {
     try
     {
         Thread.Sleep(50);
     }
     catch (ThreadInterruptedException ie)
     {
         try
         {
             base.Dispose();
         } // we didnt open successfully
         catch (Exception)
         {
         }
         throw new ThreadInterruptedException("Thread Interrupted Exception", ie);
     }
 }
Esempio n. 36
0
            public override byte[] GetBinaryValue(byte[] result, IState state)
            {
                Enclosing_Instance.EnsureOpen();

                if (internalIsBinary)
                {
                    if (fieldsData == null)
                    {
                        // Allocate new buffer if result is null or too small
                        byte[] b;
                        if (result == null || result.Length < toRead)
                        {
                            b = new byte[toRead];
                        }
                        else
                        {
                            b = result;
                        }

                        IndexInput localFieldsStream = GetFieldStream(state);

                        // Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people
                        // since they are already handling this exception when getting the document
                        try
                        {
                            localFieldsStream.Seek(pointer, state);
                            localFieldsStream.ReadBytes(b, 0, toRead, state);
                            fieldsData = isCompressed ? Enclosing_Instance.Uncompress(b) : b;
                        }
                        catch (IOException e)
                        {
                            throw new FieldReaderException(e);
                        }

                        internalbinaryOffset = 0;
                        internalBinaryLength = toRead;
                    }

                    return((byte[])fieldsData);
                }
                return(null);
            }
Esempio n. 37
0
        /// <summary>Constructs a bit vector from the file <c>name</c> in Directory
        /// <c>d</c>, as written by the <see cref="Write" /> method.
        /// </summary>
        public BitVector(Directory d, System.String name, IState state)
        {
            IndexInput input = d.OpenInput(name, state);

            try
            {
                size = input.ReadInt(state);                 // read size
                if (size == -1)
                {
                    ReadDgaps(input, state);
                }
                else
                {
                    ReadBits(input, state);
                }
            }
            finally
            {
                input.Close();
            }
        }
 /// <summary>
 /// Sole constructor. </summary>
 public MonotonicBlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct)
 {
     this.ValueCount = valueCount;
     BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
     BlockMask = blockSize - 1;
     int numBlocks = PackedInts.NumBlocks(valueCount, blockSize);
     MinValues = new long[numBlocks];
     Averages = new float[numBlocks];
     SubReaders = new PackedInts.Reader[numBlocks];
     for (int i = 0; i < numBlocks; ++i)
     {
         MinValues[i] = @in.ReadVLong();
         Averages[i] = Number.IntBitsToFloat(@in.ReadInt());
         int bitsPerValue = @in.ReadVInt();
         if (bitsPerValue > 64)
         {
             throw new Exception("Corrupted");
         }
         if (bitsPerValue == 0)
         {
             SubReaders[i] = new PackedInts.NullReader(blockSize);
         }
         else
         {
             int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize);
             if (direct)
             {
                 long pointer = @in.FilePointer;
                 SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
                 @in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue));
             }
             else
             {
                 SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
             }
         }
     }
 }
        // public static boolean DEBUG = false;
        /// <summary>
        /// Sole constructor. </summary>
        public Lucene41PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, string segmentSuffix)
        {
            bool success = false;
            IndexInput docIn = null;
            IndexInput posIn = null;
            IndexInput payIn = null;
            try
            {
                docIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), ioContext);
                Version = CodecUtil.CheckHeader(docIn, Lucene41PostingsWriter.DOC_CODEC, Lucene41PostingsWriter.VERSION_START, Lucene41PostingsWriter.VERSION_CURRENT);
                forUtil = new ForUtil(docIn);

                if (fieldInfos.HasProx())
                {
                    posIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), ioContext);
                    CodecUtil.CheckHeader(posIn, Lucene41PostingsWriter.POS_CODEC, Version, Version);

                    if (fieldInfos.HasPayloads() || fieldInfos.HasOffsets())
                    {
                        payIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), ioContext);
                        CodecUtil.CheckHeader(payIn, Lucene41PostingsWriter.PAY_CODEC, Version, Version);
                    }
                }

                this.DocIn = docIn;
                this.PosIn = posIn;
                this.PayIn = payIn;
                success = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(docIn, posIn, payIn);
                }
            }
        }
        // It is not always neccessary to move the prox pointer
        // to a new document after the freq pointer has been moved.
        // Consider for example a phrase query with two terms:
        // the freq pointer for term 1 has to move to document x
        // to answer the question if the term occurs in that document. But
        // only if term 2 also matches document x, the positions have to be
        // read to figure out if term 1 and term 2 appear next
        // to each other in document x and thus satisfy the query.
        // So we move the prox pointer lazily to the document
        // as soon as positions are requested.
        private void  LazySkip(IState state)
        {
            if (proxStream == null)
            {
                // clone lazily
                proxStream = (IndexInput)parent.core.proxStream.Clone(state);
            }

            // we might have to skip the current payload
            // if it was not read yet
            SkipPayload(state);

            if (lazySkipPointer != -1)
            {
                proxStream.Seek(lazySkipPointer, state);
                lazySkipPointer = -1;
            }

            if (lazySkipProxCount != 0)
            {
                SkipPositions(lazySkipProxCount, state);
                lazySkipProxCount = 0;
            }
        }
Esempio n. 41
0
 /// <summary>
 /// Skip the next block of data.
 /// </summary>
 /// <param name="in">      the input where to read data </param>
 /// <exception cref="IOException"> If there is a low-level I/O error </exception>
 public void SkipBlock(IndexInput @in)
 {
     int numBits = @in.ReadByte();
     if (numBits == ALL_VALUES_EQUAL)
     {
         @in.ReadVInt();
         return;
     }
     Debug.Assert(numBits > 0 && numBits <= 32, numBits.ToString());
     int encodedSize = EncodedSizes[numBits];
     @in.Seek(@in.FilePointer + encodedSize);
 }
Esempio n. 42
0
        /// <summary>
        /// Read the next block of data (<code>For</code> format).
        /// </summary>
        /// <param name="in">        the input to use to read data </param>
        /// <param name="encoded">   a buffer that can be used to store encoded data </param>
        /// <param name="decoded">   where to write decoded data </param>
        /// <exception cref="IOException"> If there is a low-level I/O error </exception>
        public void ReadBlock(IndexInput @in, sbyte[] encoded, int[] decoded)
        {
            int numBits = @in.ReadByte();
            Debug.Assert(numBits <= 32, numBits.ToString());

            if (numBits == ALL_VALUES_EQUAL)
            {
                int value = @in.ReadVInt();
                CollectionsHelper.Fill(decoded, 0, Lucene41PostingsFormat.BLOCK_SIZE, value);
                return;
            }

            int encodedSize = EncodedSizes[numBits];
            @in.ReadBytes(encoded, 0, encodedSize);

            PackedInts.Decoder decoder = Decoders[numBits];
            int iters = Iterations[numBits];
            Debug.Assert(iters * decoder.ByteValueCount() >= Lucene41PostingsFormat.BLOCK_SIZE);

            decoder.Decode(encoded, 0, decoded, 0, iters);
        }
Esempio n. 43
0
 /// <summary>Read as a bit set </summary>
 private void  ReadBits(IndexInput input, IState state)
 {
     count = input.ReadInt(state);             // read count
     bits  = new byte[(size >> 3) + 1];        // allocate bits
     input.ReadBytes(bits, 0, bits.Length, state);
 }
        private async Task InitialiseFile(ISecureStore store, IEncryptor encryptor, StoreLocation location)
        {
            bool fFileNeeded = false;
            if (!_cache.FileExists(_name))
            {
                fFileNeeded = true;
            }
            else
            {
                long cachedLength = _cache.FileLength(_name);

                var metadata = await store.GetMetadata(location).ConfigureAwait(false);
                if (metadata == null)
                {
                    throw new System.IO.FileNotFoundException(_name);
                }

                var blobLength = metadata.ContentLength ?? 0;
                var blobLastModifiedUTC = metadata.LastModified ?? DateTime.UtcNow;

                if (cachedLength != blobLength)
                {
                    fFileNeeded = true;
                }
                else
                {
                    // there seems to be an error of 1 tick which happens every once in a while 
                    // for now we will say that if they are within 1 tick of each other and same length 
                    var elapsed = _cache.FileModified(_name);

                    // normalize RAMDirectory and FSDirectory times
                    if (elapsed > ticks1970)
                    {
                        elapsed -= ticks1970;
                    }

                    var cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime();
                    if (cachedLastModifiedUTC != blobLastModifiedUTC)
                    {
                        var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
                        if (timeSpan.TotalSeconds > 1)
                        {
                            fFileNeeded = true;
                        }
                    }
                }
            }

            // if the file does not exist
            // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
            if (fFileNeeded)
            {
                using (StreamOutput fileStream = _directory.CreateCachedOutputAsStream(_name))
                {
                    var data = await store.LoadData(location, null, encryptor).ConfigureAwait(false);
                    if (data == null)
                    {
                        throw new System.IO.FileNotFoundException(_name);
                    }
                    await data.Stream.CopyToStream(fileStream, CancellationToken.None).ConfigureAwait(false);
                }

                // and open it as an input 
                _indexInput = _cache.OpenInput(_name);
            }
            else
            {
                // open the file in read only mode
                _indexInput = _cache.OpenInput(_name);
            }
        }
Esempio n. 45
0
        /// <summary>Returns a <see cref="Status" /> instance detailing
        /// the state of the index.
        ///
        /// </summary>
        /// <param name="onlySegments">list of specific segment names to check
        ///
        /// <p/>As this method checks every byte in the specified
        /// segments, on a large index it can take quite a long
        /// time to run.
        ///
        /// <p/><b>WARNING</b>: make sure
        /// you only call this when the index is not opened by any
        /// writer.
        /// </param>
        public virtual Status CheckIndex_Renamed_Method(List <string> onlySegments, IState state)
        {
            System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
            SegmentInfos sis    = new SegmentInfos();
            Status       result = new Status();

            result.dir = dir;
            try
            {
                sis.Read(dir, state);
            }
            catch (System.Exception t)
            {
                Msg("ERROR: could not read any segments file in directory");
                result.missingSegments = true;
                if (infoStream != null)
                {
                    infoStream.WriteLine(t.StackTrace);
                }
                return(result);
            }

            int        numSegments      = sis.Count;
            var        segmentsFileName = sis.GetCurrentSegmentFileName();
            IndexInput input            = null;

            try
            {
                input = dir.OpenInput(segmentsFileName, state);
            }
            catch (System.Exception t)
            {
                Msg("ERROR: could not open segments file in directory");
                if (infoStream != null)
                {
                    infoStream.WriteLine(t.StackTrace);
                }
                result.cantOpenSegments = true;
                return(result);
            }
            int format = 0;

            try
            {
                format = input.ReadInt(state);
            }
            catch (System.Exception t)
            {
                Msg("ERROR: could not read segment file version in directory");
                if (infoStream != null)
                {
                    infoStream.WriteLine(t.StackTrace);
                }
                result.missingSegmentVersion = true;
                return(result);
            }
            finally
            {
                if (input != null)
                {
                    input.Close();
                }
            }

            System.String sFormat = "";
            bool          skip    = false;

            if (format == SegmentInfos.FORMAT)
            {
                sFormat = "FORMAT [Lucene Pre-2.1]";
            }
            if (format == SegmentInfos.FORMAT_LOCKLESS)
            {
                sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
            }
            else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
            {
                sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
            }
            else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
            {
                sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
            }
            else
            {
                if (format == SegmentInfos.FORMAT_CHECKSUM)
                {
                    sFormat = "FORMAT_CHECKSUM [Lucene 2.4]";
                }
                else if (format == SegmentInfos.FORMAT_DEL_COUNT)
                {
                    sFormat = "FORMAT_DEL_COUNT [Lucene 2.4]";
                }
                else if (format == SegmentInfos.FORMAT_HAS_PROX)
                {
                    sFormat = "FORMAT_HAS_PROX [Lucene 2.4]";
                }
                else if (format == SegmentInfos.FORMAT_USER_DATA)
                {
                    sFormat = "FORMAT_USER_DATA [Lucene 2.9]";
                }
                else if (format == SegmentInfos.FORMAT_DIAGNOSTICS)
                {
                    sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
                }
                else if (format < SegmentInfos.CURRENT_FORMAT)
                {
                    sFormat = "int=" + format + " [newer version of Lucene than this tool]";
                    skip    = true;
                }
                else
                {
                    sFormat = format + " [Lucene 1.3 or prior]";
                }
            }

            result.segmentsFileName = segmentsFileName;
            result.numSegments      = numSegments;
            result.segmentFormat    = sFormat;
            result.userData         = sis.UserData;
            System.String userDataString;
            if (sis.UserData.Count > 0)
            {
                userDataString = " userData=" + CollectionsHelper.CollectionToString(sis.UserData);
            }
            else
            {
                userDataString = "";
            }

            Msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString);

            if (onlySegments != null)
            {
                result.partial = true;
                if (infoStream != null)
                {
                    infoStream.Write("\nChecking only these segments:");
                }
                foreach (string s in onlySegments)
                {
                    if (infoStream != null)
                    {
                        infoStream.Write(" " + s);
                    }
                }
                result.segmentsChecked.AddRange(onlySegments);
                Msg(":");
            }

            if (skip)
            {
                Msg("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
                result.toolOutOfDate = true;
                return(result);
            }


            result.newSegments = (SegmentInfos)sis.Clone();
            result.newSegments.Clear();

            for (int i = 0; i < numSegments; i++)
            {
                SegmentInfo info = sis.Info(i);
                if (onlySegments != null && !onlySegments.Contains(info.name))
                {
                    continue;
                }
                var segInfoStat = new Status.SegmentInfoStatus();
                result.segmentInfos.Add(segInfoStat);
                Msg("  " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
                segInfoStat.name     = info.name;
                segInfoStat.docCount = info.docCount;

                int toLoseDocCount = info.docCount;

                SegmentReader reader = null;

                try
                {
                    Msg("    compound=" + info.GetUseCompoundFile(state));
                    segInfoStat.compound = info.GetUseCompoundFile(state);
                    Msg("    hasProx=" + info.HasProx);
                    segInfoStat.hasProx = info.HasProx;
                    Msg("    numFiles=" + info.Files(state).Count);
                    segInfoStat.numFiles = info.Files(state).Count;
                    Msg(System.String.Format(nf, "    size (MB)={0:f}", new System.Object[] { (info.SizeInBytes(state) / (1024.0 * 1024.0)) }));
                    segInfoStat.sizeMB = info.SizeInBytes(state) / (1024.0 * 1024.0);
                    IDictionary <string, string> diagnostics = info.Diagnostics;
                    segInfoStat.diagnostics = diagnostics;
                    if (diagnostics.Count > 0)
                    {
                        Msg("    diagnostics = " + CollectionsHelper.CollectionToString(diagnostics));
                    }

                    int docStoreOffset = info.DocStoreOffset;
                    if (docStoreOffset != -1)
                    {
                        Msg("    docStoreOffset=" + docStoreOffset);
                        segInfoStat.docStoreOffset = docStoreOffset;
                        Msg("    docStoreSegment=" + info.DocStoreSegment);
                        segInfoStat.docStoreSegment = info.DocStoreSegment;
                        Msg("    docStoreIsCompoundFile=" + info.DocStoreIsCompoundFile);
                        segInfoStat.docStoreCompoundFile = info.DocStoreIsCompoundFile;
                    }
                    System.String delFileName = info.GetDelFileName();
                    if (delFileName == null)
                    {
                        Msg("    no deletions");
                        segInfoStat.hasDeletions = false;
                    }
                    else
                    {
                        Msg("    has deletions [delFileName=" + delFileName + "]");
                        segInfoStat.hasDeletions      = true;
                        segInfoStat.deletionsFileName = delFileName;
                    }
                    if (infoStream != null)
                    {
                        infoStream.Write("    test: open reader.........");
                    }
                    reader = SegmentReader.Get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, state);

                    segInfoStat.openReaderPassed = true;

                    int numDocs = reader.NumDocs();
                    toLoseDocCount = numDocs;
                    if (reader.HasDeletions)
                    {
                        if (reader.deletedDocs.Count() != info.GetDelCount(state))
                        {
                            throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
                        }
                        if (reader.deletedDocs.Count() > reader.MaxDoc)
                        {
                            throw new System.SystemException("too many deleted docs: MaxDoc=" + reader.MaxDoc + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
                        }
                        if (info.docCount - numDocs != info.GetDelCount(state))
                        {
                            throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs reader=" + (info.docCount - numDocs));
                        }
                        segInfoStat.numDeleted = info.docCount - numDocs;
                        Msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]");
                    }
                    else
                    {
                        if (info.GetDelCount(state) != 0)
                        {
                            throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs reader=" + (info.docCount - numDocs));
                        }
                        Msg("OK");
                    }
                    if (reader.MaxDoc != info.docCount)
                    {
                        throw new System.SystemException("SegmentReader.MaxDoc " + reader.MaxDoc + " != SegmentInfos.docCount " + info.docCount);
                    }

                    // Test getFieldNames()
                    if (infoStream != null)
                    {
                        infoStream.Write("    test: fields..............");
                    }
                    ICollection <string> fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL);
                    Msg("OK [" + fieldNames.Count + " fields]");
                    segInfoStat.numFields = fieldNames.Count;

                    // Test Field Norms
                    segInfoStat.fieldNormStatus = TestFieldNorms(fieldNames, reader, state);

                    // Test the Term Index
                    segInfoStat.termIndexStatus = TestTermIndex(info, reader, state);

                    // Test Stored Fields
                    segInfoStat.storedFieldStatus = TestStoredFields(info, reader, nf, state);

                    // Test Term Vectors
                    segInfoStat.termVectorStatus = TestTermVectors(info, reader, nf, state);

                    // Rethrow the first exception we encountered
                    //  This will cause stats for failed segments to be incremented properly
                    if (segInfoStat.fieldNormStatus.error != null)
                    {
                        throw new SystemException("Field Norm test failed");
                    }
                    else if (segInfoStat.termIndexStatus.error != null)
                    {
                        throw new SystemException("Term Index test failed");
                    }
                    else if (segInfoStat.storedFieldStatus.error != null)
                    {
                        throw new SystemException("Stored Field test failed");
                    }
                    else if (segInfoStat.termVectorStatus.error != null)
                    {
                        throw new System.SystemException("Term Vector test failed");
                    }

                    Msg("");
                }
                catch (System.Exception t)
                {
                    Msg("FAILED");
                    const string comment = "fixIndex() would remove reference to this segment";
                    Msg("    WARNING: " + comment + "; full exception:");
                    if (infoStream != null)
                    {
                        infoStream.WriteLine(t.StackTrace);
                    }
                    Msg("");
                    result.totLoseDocCount += toLoseDocCount;
                    result.numBadSegments++;
                    continue;
                }
                finally
                {
                    if (reader != null)
                    {
                        reader.Close();
                    }
                }

                // Keeper
                result.newSegments.Add((SegmentInfo)info.Clone());
            }

            if (0 == result.numBadSegments)
            {
                result.clean = true;
                Msg("No problems were detected with this index.\n");
            }
            else
            {
                Msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected");
            }

            return(result);
        }
        public virtual void TestManySubFiles()
        {
            Directory d = NewFSDirectory(CreateTempDir("CFSManySubFiles"));
            int FILE_COUNT = AtLeast(500);

            for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
            {
                IndexOutput @out = d.CreateOutput("file." + fileIdx, NewIOContext(Random()));
                @out.WriteByte((byte)(sbyte)fileIdx);
                @out.Dispose();
            }

            CompoundFileDirectory cfd = new CompoundFileDirectory(d, "c.cfs", NewIOContext(Random()), true);
            for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
            {
                string fileName = "file." + fileIdx;
                d.Copy(cfd, fileName, fileName, NewIOContext(Random()));
            }
            cfd.Dispose();

            IndexInput[] ins = new IndexInput[FILE_COUNT];
            CompoundFileDirectory cfr = new CompoundFileDirectory(d, "c.cfs", NewIOContext(Random()), false);
            for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
            {
                ins[fileIdx] = cfr.OpenInput("file." + fileIdx, NewIOContext(Random()));
            }

            for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
            {
                Assert.AreEqual((byte)fileIdx, ins[fileIdx].ReadByte());
            }

            for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
            {
                ins[fileIdx].Dispose();
            }
            cfr.Dispose();
            d.Dispose();
        }
 private void AssertSameStreams(string msg, IndexInput expected, IndexInput actual, long seekTo)
 {
     if (seekTo >= 0 && seekTo < expected.Length())
     {
         expected.Seek(seekTo);
         actual.Seek(seekTo);
         AssertSameStreams(msg + ", seek(mid)", expected, actual);
     }
 }
Esempio n. 48
0
        public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
        {
            _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];

            #if FULLDEBUG
            Debug.WriteLine(String.Format("opening {0} ", _name));
            #endif
            _fileMutex = BlobMutexManager.GrabMutex(_name);
            _fileMutex.WaitOne();
            try
            {
                _azureDirectory = azuredirectory;
                _blobContainer = azuredirectory.BlobContainer;
                _blob = blob;

                string fileName = _name;

                bool fFileNeeded = false;
                if (!CacheDirectory.FileExists(fileName))
                {
                    fFileNeeded = true;
                }
                else
                {
                    long cachedLength = CacheDirectory.FileLength(fileName);
                    long blobLength = blob.Properties.Length;
                    long.TryParse(blob.Metadata["CachedLength"], out blobLength);

                    long longLastModified = 0;
                    DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
                    //if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified))
                    //    blobLastModifiedUTC = new DateTime(longLastModified, DateTimeKind.Local).ToUniversalTime();

                    if (cachedLength != blobLength)
                        fFileNeeded = true;
                    else
                    {
                        // there seems to be an error of 1 tick which happens every once in a while
                        // for now we will say that if they are within 1 tick of each other and same length
                        DateTime cachedLastModifiedUTC = DateTime.MinValue;
                        if (CacheDirectory is RAMDirectory) {
                            cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Local).ToUniversalTime();
                        }
                        else if (CacheDirectory is FSDirectory) {
                            cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Utc).ToUniversalTime().AddYears(1969);
                        }
                        if (cachedLastModifiedUTC != blobLastModifiedUTC)
                        {
                            TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
                            if (timeSpan.TotalSeconds > 1)
                                fFileNeeded = true;
                            else
                            {
            #if FULLDEBUG
                                Debug.WriteLine(timeSpan.TotalSeconds);
            #endif
                                // file not needed
                            }
                        }
                    }
                }

                // if the file does not exist
                // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
                if (fFileNeeded)
                {
            #if COMPRESSBLOBS
                    if (_azureDirectory.ShouldCompressFile(_name))
                    {
                        // then we will get it fresh into local deflatedName
                        // StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName));
                        MemoryStream deflatedStream = new MemoryStream();

                        // get the deflated blob
                        _blob.DownloadToStream(deflatedStream);

                        Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length));

                        // seek back to begininng
                        deflatedStream.Seek(0, SeekOrigin.Begin);

                        // open output file for uncompressed contents
                        StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);

                        // create decompressor
                        DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress);

                        byte[] bytes = new byte[65535];
                        int nRead = 0;
                        do
                        {
                            nRead = decompressor.Read(bytes, 0, 65535);
                            if (nRead > 0)
                                fileStream.Write(bytes, 0, nRead);
                        } while (nRead == 65535);
                        decompressor.Close(); // this should close the deflatedFileStream too

                        fileStream.Close();

                    }
                    else
            #endif
                    {
                        StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);

                        // get the blob
                        _blob.DownloadToStream(fileStream);

                        fileStream.Flush();
                        Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));

                        fileStream.Close();
                    }

                    // and open it as an input
                    _indexInput = CacheDirectory.OpenInput(fileName);
                }
                else
                {
            #if FULLDEBUG
                    Debug.WriteLine(String.Format("Using cached file for {0}", _name));
            #endif

                    // open the file in read only mode
                    _indexInput = CacheDirectory.OpenInput(fileName);
                }
            }
            finally
            {
                _fileMutex.ReleaseMutex();
            }
        }
Esempio n. 49
0
 protected override void Dispose(bool disposing)
 {
     _fileMutex.WaitOne();
     try
     {
     #if FULLDEBUG
         Debug.WriteLine(String.Format("CLOSED READSTREAM local {0}", _name));
     #endif
         _indexInput.Dispose();
         _indexInput = null;
         _azureDirectory = null;
         _blobContainer = null;
         _blob = null;
         GC.SuppressFinalize(this);
     }
     finally
     {
         _fileMutex.ReleaseMutex();
     }
 }
Esempio n. 50
0
        /// <summary> Construct a new SegmentInfo instance by reading a
        /// previously saved SegmentInfo from input.
        ///
        /// </summary>
        /// <param name="dir">directory to load from
        /// </param>
        /// <param name="format">format of the segments info file
        /// </param>
        /// <param name="input">input handle to read segment info from
        /// </param>
        internal SegmentInfo(Directory dir, int format, IndexInput input, IState state)
        {
            this.dir = dir;
            name     = input.ReadString(state);
            docCount = input.ReadInt(state);
            if (format <= SegmentInfos.FORMAT_LOCKLESS)
            {
                delGen = input.ReadLong(state);
                if (format <= SegmentInfos.FORMAT_SHARED_DOC_STORE)
                {
                    docStoreOffset = input.ReadInt(state);
                    if (docStoreOffset != -1)
                    {
                        docStoreSegment        = input.ReadString(state);
                        docStoreIsCompoundFile = (1 == input.ReadByte(state));
                    }
                    else
                    {
                        docStoreSegment        = name;
                        docStoreIsCompoundFile = false;
                    }
                }
                else
                {
                    docStoreOffset         = -1;
                    docStoreSegment        = name;
                    docStoreIsCompoundFile = false;
                }
                if (format <= SegmentInfos.FORMAT_SINGLE_NORM_FILE)
                {
                    hasSingleNormFile = (1 == input.ReadByte(state));
                }
                else
                {
                    hasSingleNormFile = false;
                }
                int numNormGen = input.ReadInt(state);
                if (numNormGen == NO)
                {
                    normGen = null;
                }
                else
                {
                    normGen = new long[numNormGen];
                    for (int j = 0; j < numNormGen; j++)
                    {
                        normGen[j] = input.ReadLong(state);
                    }
                }
                isCompoundFile = (sbyte)input.ReadByte(state);
                preLockless    = (isCompoundFile == CHECK_DIR);
                if (format <= SegmentInfos.FORMAT_DEL_COUNT)
                {
                    delCount = input.ReadInt(state);
                    System.Diagnostics.Debug.Assert(delCount <= docCount);
                }
                else
                {
                    delCount = -1;
                }
                if (format <= SegmentInfos.FORMAT_HAS_PROX)
                {
                    hasProx = input.ReadByte(state) == 1;
                }
                else
                {
                    hasProx = true;
                }

                if (format <= SegmentInfos.FORMAT_DIAGNOSTICS)
                {
                    diagnostics = input.ReadStringStringMap(state);
                }
                else
                {
                    diagnostics = new Dictionary <string, string>();
                }
            }
            else
            {
                delGen                 = CHECK_DIR;
                normGen                = null;
                isCompoundFile         = (sbyte)(CHECK_DIR);
                preLockless            = true;
                hasSingleNormFile      = false;
                docStoreOffset         = -1;
                docStoreIsCompoundFile = false;
                docStoreSegment        = null;
                delCount               = -1;
                hasProx                = true;
                diagnostics            = new Dictionary <string, string>();
            }
        }
Esempio n. 51
0
 internal FaultyIndexInput(IndexInput delegate_Renamed)
 {
     this.delegate_Renamed = delegate_Renamed;
 }
Esempio n. 52
0
 internal CSIndexInput(IndexInput @base, long fileOffset, long length, IState state) : this(@base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE, state)
 {
 }
Esempio n. 53
0
 internal CSIndexInput(IndexInput @base, long fileOffset, long length, int readBufferSize, IState state) : base(readBufferSize)
 {
     this.base_Renamed = (IndexInput)@base.Clone(state);
     this.fileOffset   = fileOffset;
     this.length       = length;
 }
Esempio n. 54
0
        internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size, IState state)
        {
            bool success = false;

            isOriginal = true;
            try
            {
                fieldInfos = fn;

                cloneableFieldsStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_EXTENSION, readBufferSize, state);
                cloneableIndexStream  = d.OpenInput(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION, readBufferSize, state);

                // First version of fdx did not include a format
                // header, but, the first int will always be 0 in that
                // case
                int firstInt = cloneableIndexStream.ReadInt(state);
                format = firstInt == 0 ? 0 : firstInt;

                if (format > FieldsWriter.FORMAT_CURRENT)
                {
                    throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FieldsWriter.FORMAT_CURRENT + " or lower");
                }

                formatSize = format > FieldsWriter.FORMAT ? 4 : 0;

                if (format < FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
                {
                    cloneableFieldsStream.SetModifiedUTF8StringsMode();
                }

                fieldsStream = (IndexInput)cloneableFieldsStream.Clone(state);

                long indexSize = cloneableIndexStream.Length(state) - formatSize;

                if (docStoreOffset != -1)
                {
                    // We read only a slice out of this shared fields file
                    this.docStoreOffset = docStoreOffset;
                    this.size           = size;

                    // Verify the file is long enough to hold all of our
                    // docs
                    System.Diagnostics.Debug.Assert(((int)(indexSize / 8)) >= size + this.docStoreOffset, "indexSize=" + indexSize + " size=" + size + " docStoreOffset=" + docStoreOffset);
                }
                else
                {
                    this.docStoreOffset = 0;
                    this.size           = (int)(indexSize >> 3);
                }

                indexStream  = (IndexInput)cloneableIndexStream.Clone(state);
                numTotalDocs = (int)(indexSize >> 3);
                success      = true;
            }
            finally
            {
                // With lock-less commits, it's entirely possible (and
                // fine) to hit a FileNotFound exception above. In
                // this case, we want to explicitly close any subset
                // of things that were opened so that we don't have to
                // wait for a GC to do so.
                if (!success)
                {
                    Dispose();
                }
            }
        }
Esempio n. 55
0
 public StreamInput(IndexInput input)
 {
     Input = input;
 }
        private SecureStoreIndexInput(SecureStoreIndexInput cloneInput)
        {
            _fileMutex = BlobMutexManager.GrabMutex(cloneInput._name);
            _fileMutex.WaitOne();

            try
            {
                _directory = cloneInput._directory;
                _cache = cloneInput._cache;
                _name = cloneInput._name;
                _indexInput = cloneInput._indexInput.Clone() as IndexInput;
            }
            catch (Exception)
            {
                // sometimes we get access denied on the 2nd stream...but not always. I haven't tracked it down yet
                // but this covers our tail until I do
                LeoTrace.WriteLine(String.Format("Falling back to memory clone for {0}", cloneInput._name));
            }
            finally
            {
                _fileMutex.ReleaseMutex();
            }
        }
        private void AssertSameSeekBehavior(string msg, IndexInput expected, IndexInput actual)
        {
            // seek to 0
            long point = 0;
            AssertSameStreams(msg + ", seek(0)", expected, actual, point);

            // seek to middle
            point = expected.Length() / 2l;
            AssertSameStreams(msg + ", seek(mid)", expected, actual, point);

            // seek to end - 2
            point = expected.Length() - 2;
            AssertSameStreams(msg + ", seek(end-2)", expected, actual, point);

            // seek to end - 1
            point = expected.Length() - 1;
            AssertSameStreams(msg + ", seek(end-1)", expected, actual, point);

            // seek to the end
            point = expected.Length();
            AssertSameStreams(msg + ", seek(end)", expected, actual, point);

            // seek past end
            point = expected.Length() + 1;
            AssertSameStreams(msg + ", seek(end+1)", expected, actual, point);
        }
 protected override void Dispose(bool disposing)
 {
     _fileMutex.WaitOne();
     try
     {
         _indexInput.Dispose();
         _indexInput = null;
         _directory = null;
         _cache = null;
         GC.SuppressFinalize(this);
     }
     finally
     {
         _fileMutex.ReleaseMutex();
     }
 }
        private void AssertSameStreams(string msg, IndexInput expected, IndexInput test)
        {
            Assert.IsNotNull(expected, msg + " null expected");
            Assert.IsNotNull(test, msg + " null test");
            Assert.AreEqual(expected.Length(), test.Length(), msg + " length");
            Assert.AreEqual(expected.FilePointer, test.FilePointer, msg + " position");

            var expectedBuffer = new byte[512];
            var testBuffer = new byte[expectedBuffer.Length];

            long remainder = expected.Length() - expected.FilePointer;
            while (remainder > 0)
            {
                int readLen = (int)Math.Min(remainder, expectedBuffer.Length);
                expected.ReadBytes(expectedBuffer, 0, readLen);
                test.ReadBytes(testBuffer, 0, readLen);
                AssertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
                remainder -= readLen;
            }
        }
Esempio n. 60
0
        internal SegmentTermEnum(IndexInput i, FieldInfos fis, bool isi, IState state)
        {
            input         = i;
            fieldInfos    = fis;
            isIndex       = isi;
            maxSkipLevels = 1;             // use single-level skip lists for formats > -3

            int firstInt = input.ReadInt(state);

            if (firstInt >= 0)
            {
                // original-format file, without explicit format version number
                format = 0;
                size   = firstInt;

                // back-compatible settings
                indexInterval = 128;
                skipInterval  = System.Int32.MaxValue;                // switch off skipTo optimization
            }
            else
            {
                // we have a format version number
                format = firstInt;

                // check that it is a format we can understand
                if (format < TermInfosWriter.FORMAT_CURRENT)
                {
                    throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher");
                }

                size = input.ReadLong(state);                 // read the size

                if (format == -1)
                {
                    if (!isIndex)
                    {
                        indexInterval        = input.ReadInt(state);
                        formatM1SkipInterval = input.ReadInt(state);
                    }
                    // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in
                    // skipTo implementation of these versions
                    skipInterval = System.Int32.MaxValue;
                }
                else
                {
                    indexInterval = input.ReadInt(state);
                    skipInterval  = input.ReadInt(state);
                    if (format <= TermInfosWriter.FORMAT)
                    {
                        // this new format introduces multi-level skipping
                        maxSkipLevels = input.ReadInt(state);
                    }
                }
                System.Diagnostics.Debug.Assert(indexInterval > 0, "indexInterval=" + indexInterval + " is negative; must be > 0");
                System.Diagnostics.Debug.Assert(skipInterval > 0, "skipInterval=" + skipInterval + " is negative; must be > 0");
            }
            if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
            {
                termBuffer.SetPreUTF8Strings();
                scanBuffer.SetPreUTF8Strings();
                prevBuffer.SetPreUTF8Strings();
            }
        }