Exemple #1
0
        public void Read(IndexInput input, FieldInfos fieldInfos)
        {
            this.term = null; // invalidate cache
            int start = input.ReadVInt();
            int length = input.ReadVInt();
            int totalLength = start + length;
            if (preUTF8Strings)
            {
                text.SetLength(totalLength);
                input.ReadChars(text.result, start, length);
            }
            else
            {

                if (dirty)
                {
                    // Fully convert all bytes since bytes is dirty
                    UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes);
                    bytes.SetLength(totalLength);
                    input.ReadBytes(bytes.result, start, length);
                    UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text);
                    dirty = false;
                }
                else
                {
                    // Incrementally convert only the UTF8 bytes that are new:
                    bytes.SetLength(totalLength);
                    input.ReadBytes(bytes.result, start, length);
                    UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text);
                }
            }
            this.field = fieldInfos.FieldName(input.ReadVInt());
        }
Exemple #2
0
 public void Read(IndexInput input, FieldInfos fieldInfos)
 {
     this.Term = null; // invalidate cache
     NewSuffixStart = input.ReadVInt();
     int length = input.ReadVInt();
     int totalLength = NewSuffixStart + length;
     Debug.Assert(totalLength <= ByteBlockPool.BYTE_BLOCK_SIZE - 2, "termLength=" + totalLength + ",resource=" + input);
     if (Bytes.Bytes.Length < totalLength)
     {
         Bytes.Grow(totalLength);
     }
     Bytes.Length = totalLength;
     input.ReadBytes(Bytes.Bytes, NewSuffixStart, length);
     int fieldNumber = input.ReadVInt();
     if (fieldNumber != CurrentFieldNumber)
     {
         CurrentFieldNumber = fieldNumber;
         // NOTE: too much sneakiness here, seriously this is a negative vint?!
         if (CurrentFieldNumber == -1)
         {
             Field = "";
         }
         else
         {
             Debug.Assert(fieldInfos.FieldInfo(CurrentFieldNumber) != null, CurrentFieldNumber.ToString());
             Field = String.Intern(fieldInfos.FieldInfo(CurrentFieldNumber).Name);
         }
     }
     else
     {
         Debug.Assert(Field.Equals(fieldInfos.FieldInfo(fieldNumber).Name), "currentFieldNumber=" + CurrentFieldNumber + " field=" + Field + " vs " + fieldInfos.FieldInfo(fieldNumber) == null ? "null" : fieldInfos.FieldInfo(fieldNumber).Name);
     }
 }
Exemple #3
0
			public Norm(SegmentReader enclosingInstance, IndexInput in_Renamed, int number, long normSeek)
			{
				InitBlock(enclosingInstance);
				this.in_Renamed = in_Renamed;
				this.number = number;
				this.normSeek = normSeek;
			}
 public Lucene41SkipReader(IndexInput skipStream, int maxSkipLevels, int blockSize, bool hasPos, bool hasOffsets, bool hasPayloads)
     : base(skipStream, maxSkipLevels, blockSize, 8)
 {
     this.BlockSize = blockSize;
     DocPointer_Renamed = new long[maxSkipLevels];
     if (hasPos)
     {
         PosPointer_Renamed = new long[maxSkipLevels];
         PosBufferUpto_Renamed = new int[maxSkipLevels];
         if (hasPayloads)
         {
             PayloadByteUpto_Renamed = new int[maxSkipLevels];
         }
         else
         {
             PayloadByteUpto_Renamed = null;
         }
         if (hasOffsets || hasPayloads)
         {
             PayPointer_Renamed = new long[maxSkipLevels];
         }
         else
         {
             PayPointer_Renamed = null;
         }
     }
     else
     {
         PosPointer_Renamed = null;
     }
 }
 public Lucene3xSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval)
     : base(skipStream, maxSkipLevels, skipInterval)
 {
     FreqPointer_Renamed = new long[maxSkipLevels];
     ProxPointer_Renamed = new long[maxSkipLevels];
     PayloadLength_Renamed = new int[maxSkipLevels];
 }
Exemple #6
0
 internal DefaultSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval)
     : base(skipStream, maxSkipLevels, skipInterval)
 {
     freqPointer = new long[maxSkipLevels];
     proxPointer = new long[maxSkipLevels];
     payloadLength = new int[maxSkipLevels];
 }
Exemple #7
0
		public SegmentTermDocs(SegmentReader parent)
		{
			this.parent = parent;
			this.freqStream = (IndexInput) parent.freqStream.Clone();
			this.deletedDocs = parent.deletedDocs;
			this.skipInterval = parent.tis.GetSkipInterval();
		}
        internal SegmentTermEnum(IndexInput i, FieldInfos fis, bool isi)
        {
            input = i;
            fieldInfos = fis;
            isIndex = isi;
            maxSkipLevels = 1; // use single-level skip lists for formats > -3

            int firstInt = input.ReadInt();
            if (firstInt >= 0)
            {
                // original-format file, without explicit format version number
                format = 0;
                size = firstInt;

                // back-compatible settings
                indexInterval = 128;
                skipInterval = System.Int32.MaxValue; // switch off skipTo optimization
            }
            else
            {
                // we have a format version number
                format = firstInt;

                // check that it is a format we can understand
                if (format < TermInfosWriter.FORMAT_CURRENT)
                    throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher");

                size = input.ReadLong(); // read the size

                if (format == - 1)
                {
                    if (!isIndex)
                    {
                        indexInterval = input.ReadInt();
                        formatM1SkipInterval = input.ReadInt();
                    }
                    // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in
                    // skipTo implementation of these versions
                    skipInterval = System.Int32.MaxValue;
                }
                else
                {
                    indexInterval = input.ReadInt();
                    skipInterval = input.ReadInt();
                    if (format <= TermInfosWriter.FORMAT)
                    {
                        // this new format introduces multi-level skipping
                        maxSkipLevels = input.ReadInt();
                    }
                }
                System.Diagnostics.Debug.Assert(indexInterval > 0, "indexInterval=" + indexInterval + " is negative; must be > 0");
                System.Diagnostics.Debug.Assert(skipInterval > 0, "skipInterval=" + skipInterval + " is negative; must be > 0");
            }
            if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
            {
                termBuffer.SetPreUTF8Strings();
                scanBuffer.SetPreUTF8Strings();
                prevBuffer.SetPreUTF8Strings();
            }
        }
 public SegmentTermDocs(IndexInput freqStream, TermInfosReader tis, FieldInfos fieldInfos)
 {
     this.FreqStream = (IndexInput)freqStream.Clone();
     this.Tis = tis;
     this.FieldInfos = fieldInfos;
     SkipInterval = tis.SkipInterval;
     MaxSkipLevels = tis.MaxSkipLevels;
 }
 internal DirectPacked64SingleBlockReader(int bitsPerValue, int valueCount, IndexInput @in)
     : base(valueCount, bitsPerValue)
 {
     this.@in = @in;
     StartPointer = @in.FilePointer;
     ValuesPerBlock = 64 / bitsPerValue;
     Mask = ~(~0L << bitsPerValue);
 }
        /*internal*/
        public FieldsReader(Directory d, System.String segment, FieldInfos fn)
        {
            fieldInfos = fn;

            fieldsStream = d.OpenInput(segment + ".fdt");
            indexStream = d.OpenInput(segment + ".fdx");

            size = (int) (indexStream.Length() / 8);
        }
		private int CheckValidFormat(IndexInput in_Renamed)
		{
			int format = in_Renamed.ReadInt();
			if (format > TermVectorsWriter.FORMAT_VERSION)
			{
				throw new System.IO.IOException("Incompatible format version: " + format + " expected " + TermVectorsWriter.FORMAT_VERSION + " or less");
			}
			return format;
		}
		public void  Read(IndexInput input, FieldInfos fieldInfos)
		{
			this.term = null; // invalidate cache
			int start = input.ReadVInt();
			int length = input.ReadVInt();
			int totalLength = start + length;
			SetTextLength(totalLength);
			input.ReadChars(this.text, start, length);
			this.field = fieldInfos.FieldName(input.ReadVInt());
		}
 /*protected internal*/
 public SegmentTermDocs(SegmentReader parent)
 {
     this.parent = parent;
     this.freqStream = (IndexInput) parent.core.freqStream.Clone();
     lock (parent)
     {
         this.deletedDocs = parent.deletedDocs;
     }
     this.skipInterval = parent.core.GetTermsReader().SkipInterval;
     this.maxSkipLevels = parent.core.GetTermsReader().MaxSkipLevels;
 }
Exemple #15
0
		public /*protected internal*/ SegmentTermDocs(SegmentReader parent)
		{
			this.parent = parent;
			this.freqStream = (IndexInput) parent.core.freqStream.Clone();
			lock (parent)
			{
				this.deletedDocs = parent.deletedDocs;
			}
			this.skipInterval = parent.core.GetTermsReader().GetSkipInterval();
			this.maxSkipLevels = parent.core.GetTermsReader().GetMaxSkipLevels();
		}
		public CompoundFileReader(Directory dir, System.String name, int readBufferSize)
		{
			directory = dir;
			fileName = name;
			this.readBufferSize = readBufferSize;
			
			bool success = false;
			
			try
			{
				stream = dir.OpenInput(name, readBufferSize);
				
				// read the directory and init files
				int count = stream.ReadVInt();
				FileEntry entry = null;
				for (int i = 0; i < count; i++)
				{
					long offset = stream.ReadLong();
					System.String id = stream.ReadString();
					
					if (entry != null)
					{
						// set length of the previous entry
						entry.length = offset - entry.offset;
					}
					
					entry = new FileEntry();
					entry.offset = offset;
					entries[id] = entry;
				}
				
				// set the length of the final entry
				if (entry != null)
				{
					entry.length = stream.Length() - entry.offset;
				}
				
				success = true;
			}
			finally
			{
				if (!success && (stream != null))
				{
					try
					{
						stream.Close();
					}
					catch (System.IO.IOException e)
					{
					}
				}
			}
		}
		// Used only by clone
		private FieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int formatSize, int docStoreOffset, IndexInput cloneableFieldsStream, IndexInput cloneableIndexStream)
		{
			this.fieldInfos = fieldInfos;
			this.numTotalDocs = numTotalDocs;
			this.size = size;
			this.format = format;
			this.formatSize = formatSize;
			this.docStoreOffset = docStoreOffset;
			this.cloneableFieldsStream = cloneableFieldsStream;
			this.cloneableIndexStream = cloneableIndexStream;
			fieldsStream = (IndexInput) cloneableFieldsStream.Clone();
			indexStream = (IndexInput) cloneableIndexStream.Clone();
		}
        public DirectPackedReader(int bitsPerValue, int valueCount, IndexInput @in)
            : base(valueCount, bitsPerValue)
        {
            this.@in = @in;

            StartPointer = @in.FilePointer;
            if (bitsPerValue == 64)
            {
                ValueMask = -1L;
            }
            else
            {
                ValueMask = (1L << bitsPerValue) - 1;
            }
        }
        /*internal*/
        public TermVectorsReader(Directory d, System.String segment, FieldInfos fieldInfos)
        {
            if (d.FileExists(segment + TermVectorsWriter.TVX_EXTENSION))
            {
                tvx = d.OpenInput(segment + TermVectorsWriter.TVX_EXTENSION);
                CheckValidFormat(tvx);
                tvd = d.OpenInput(segment + TermVectorsWriter.TVD_EXTENSION);
                tvdFormat = CheckValidFormat(tvd);
                tvf = d.OpenInput(segment + TermVectorsWriter.TVF_EXTENSION);
                tvfFormat = CheckValidFormat(tvf);
                size = (int) tvx.Length() / 8;
            }

            this.fieldInfos = fieldInfos;
        }
Exemple #20
0
		internal SegmentTermEnum(IndexInput i, FieldInfos fis, bool isi)
		{
			input = i;
			fieldInfos = fis;
			isIndex = isi;
			
			int firstInt = input.ReadInt();
			if (firstInt >= 0)
			{
				// original-format file, without explicit format version number
				format = 0;
				size = firstInt;
				
				// back-compatible settings
				indexInterval = 128;
				skipInterval = System.Int32.MaxValue; // switch off skipTo optimization
			}
			else
			{
				// we have a format version number
				format = firstInt;
				
				// check that it is a format we can understand
				if (format < TermInfosWriter.FORMAT)
					throw new System.IO.IOException("Unknown format version:" + format);
				
				size = input.ReadLong(); // read the size
				
				if (format == - 1)
				{
					if (!isIndex)
					{
						indexInterval = input.ReadInt();
						formatM1SkipInterval = input.ReadInt();
					}
					// switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in 
					// skipTo implementation of these versions
					skipInterval = System.Int32.MaxValue;
				}
				else
				{
					indexInterval = input.ReadInt();
					skipInterval = input.ReadInt();
				}
			}
		}
		internal TermVectorsReader(Directory d, System.String segment, FieldInfos fieldInfos, int readBufferSize, int docStoreOffset, int size)
		{
			bool success = false;
			
			try
			{
				if (d.FileExists(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION))
				{
					tvx = d.OpenInput(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION, readBufferSize);
					CheckValidFormat(tvx);
					tvd = d.OpenInput(segment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION, readBufferSize);
					tvdFormat = CheckValidFormat(tvd);
					tvf = d.OpenInput(segment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION, readBufferSize);
					tvfFormat = CheckValidFormat(tvf);
					if (- 1 == docStoreOffset)
					{
						this.docStoreOffset = 0;
						this.size = (int) (tvx.Length() >> 3);
					}
					else
					{
						this.docStoreOffset = docStoreOffset;
						this.size = size;
						// Verify the file is long enough to hold all of our
						// docs
						System.Diagnostics.Debug.Assert(((int) (tvx.Length() / 8)) >= size + docStoreOffset);
					}
				}
				
				this.fieldInfos = fieldInfos;
				success = true;
			}
			finally
			{
				// With lock-less commits, it's entirely possible (and
				// fine) to hit a FileNotFound exception above. In
				// this case, we want to explicitly close any subset
				// of things that were opened so that we don't have to
				// wait for a GC to do so.
				if (!success)
				{
					Close();
				}
			}
		}
        private IndexInput[] skipStream; // skipStream for each level

        #endregion Fields

        #region Constructors

        public MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval)
        {
            this.skipStream = new IndexInput[maxSkipLevels];
            this.skipPointer = new long[maxSkipLevels];
            this.childPointer = new long[maxSkipLevels];
            this.numSkipped = new int[maxSkipLevels];
            this.maxNumberOfSkipLevels = maxSkipLevels;
            this.skipInterval = new int[maxSkipLevels];
            this.skipStream[0] = skipStream;
            this.inputIsBuffered = (skipStream is BufferedIndexInput);
            this.skipInterval[0] = skipInterval;
            for (int i = 1; i < maxSkipLevels; i++)
            {
                // cache skip intervals
                this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval;
            }
            skipDoc = new int[maxSkipLevels];
        }
        /// <summary>
        /// Sole constructor. </summary>
        public Lucene40StoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context)
        {
            string segment = si.Name;
            bool success = false;
            FieldInfos = fn;
            try
            {
                FieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context);
                string indexStreamFN = IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
                IndexStream = d.OpenInput(indexStreamFN, context);

                CodecUtil.CheckHeader(IndexStream, Lucene40StoredFieldsWriter.CODEC_NAME_IDX, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
                CodecUtil.CheckHeader(FieldsStream, Lucene40StoredFieldsWriter.CODEC_NAME_DAT, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
                Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_DAT == FieldsStream.FilePointer);
                Debug.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX == IndexStream.FilePointer);
                long indexSize = IndexStream.Length() - Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX;
                this.Size_Renamed = (int)(indexSize >> 3);
                // Verify two sources of "maxDoc" agree:
                if (this.Size_Renamed != si.DocCount)
                {
                    throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.Size_Renamed + " but segmentInfo shows " + si.DocCount);
                }
                NumTotalDocs = (int)(indexSize >> 3);
                success = true;
            }
            finally
            {
                // With lock-less commits, it's entirely possible (and
                // fine) to hit a FileNotFound exception above. In
                // this case, we want to explicitly close any subset
                // of things that were opened so that we don't have to
                // wait for a GC to do so.
                if (!success)
                {
                    try
                    {
                        Dispose();
                    } // ensure we throw our original exception
                    catch (Exception)
                    {
                    }
                }
            }
        }
		internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size)
		{
			bool success = false;
			
			try
			{
				fieldInfos = fn;
				
				cloneableFieldsStream = d.OpenInput(segment + ".fdt", readBufferSize);
				fieldsStream = (IndexInput) cloneableFieldsStream.Clone();
				indexStream = d.OpenInput(segment + ".fdx", readBufferSize);
				
				if (docStoreOffset != - 1)
				{
					// We read only a slice out of this shared fields file
					this.docStoreOffset = docStoreOffset;
					this.size = size;
					
					// Verify the file is long enough to hold all of our
					// docs
					System.Diagnostics.Debug.Assert(((int)(indexStream.Length() / 8)) >= size + this.docStoreOffset);
				}
				else
				{
					this.docStoreOffset = 0;
					this.size = (int) (indexStream.Length() >> 3);
				}
				
				numTotalDocs = (int) (indexStream.Length() >> 3);
				success = true;
			}
			finally
			{
				// With lock-less commits, it's entirely possible (and
				// fine) to hit a FileNotFound exception above. In
				// this case, we want to explicitly close any subset
				// of things that were opened so that we don't have to
				// wait for a GC to do so.
				if (!success)
				{
					Close();
				}
			}
		}
        private void  AssertSameStreams(System.String msg, IndexInput expected, IndexInput test)
        {
            Assert.IsNotNull(expected, msg + " null expected");
            Assert.IsNotNull(test, msg + " null test");
            Assert.AreEqual(expected.Length(), test.Length(), msg + " length");
            Assert.AreEqual(expected.FilePointer, test.FilePointer, msg + " position");

            byte[] expectedBuffer = new byte[512];
            byte[] testBuffer     = new byte[expectedBuffer.Length];

            long remainder = expected.Length() - expected.FilePointer;

            while (remainder > 0)
            {
                int readLen = (int)System.Math.Min(remainder, expectedBuffer.Length);
                expected.ReadBytes(expectedBuffer, 0, readLen);
                test.ReadBytes(testBuffer, 0, readLen);
                AssertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
                remainder -= readLen;
            }
        }
        public virtual void  TestSingleFile()
        {
            int[] data = new int[] { 0, 1, 10, 100 };
            for (int i = 0; i < data.Length; i++)
            {
                System.String name = "t" + data[i];
                CreateSequenceFile(dir, name, (byte)0, data[i]);
                CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
                csw.AddFile(name);
                csw.Close();

                CompoundFileReader csr      = new CompoundFileReader(dir, name + ".cfs");
                IndexInput         expected = dir.OpenInput(name);
                IndexInput         actual   = csr.OpenInput(name);
                AssertSameStreams(name, expected, actual);
                AssertSameSeekBehavior(name, expected, actual);
                expected.Close();
                actual.Close();
                csr.Close();
            }
        }
Exemple #27
0
        /// <summary>Constructs a bit vector from the file <code>name</code> in Directory
        /// <code>d</code>, as written by the {@link #write} method.
        /// </summary>
        public BitVector(Directory d, System.String name)
        {
            IndexInput input = d.OpenInput(name);

            try
            {
                size = input.ReadInt();                 // read size
                if (size == -1)
                {
                    ReadDgaps(input);
                }
                else
                {
                    ReadBits(input);
                }
            }
            finally
            {
                input.Close();
            }
        }
 /** Copy numBytes from srcIn to destIn */
 void copyBytes(IndexInput srcIn, IndexOutput destIn, long numBytes)
 {
     // TODO: we could do this more efficiently (save a copy)
     // because it's always from a ByteSliceReader ->
     // IndexOutput
     while (numBytes > 0)
     {
         int chunk;
         if (numBytes > 4096)
         {
             chunk = 4096;
         }
         else
         {
             chunk = (int)numBytes;
         }
         srcIn.ReadBytes(copyByteBuffer, 0, chunk);
         destIn.WriteBytes(copyByteBuffer, 0, chunk);
         numBytes -= chunk;
     }
 }
Exemple #29
0
		/// <summary>Copy the contents of the file with specified extension into the
		/// provided output stream. Use the provided buffer for moving data
		/// to reduce memory allocation.
		/// </summary>
		private void  CopyFile(FileEntry source, IndexOutput os, byte[] buffer)
		{
			IndexInput isRenamed = null;
			try
			{
				long startPtr = os.FilePointer;
				
				isRenamed = directory.OpenInput(source.file);
				long length = isRenamed.Length();
				long remainder = length;
				int chunk = buffer.Length;
				
				while (remainder > 0)
				{
					var len = (int) Math.Min(chunk, remainder);
					isRenamed.ReadBytes(buffer, 0, len, false);
					os.WriteBytes(buffer, len);
					remainder -= len;
					if (checkAbort != null)
					// Roughly every 2 MB we will check if
					// it's time to abort
						checkAbort.Work(80);
				}
				
				// Verify that remainder is 0
				if (remainder != 0)
					throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")");
				
				// Verify that the output length diff is equal to original file
				long endPtr = os.FilePointer;
				long diff = endPtr - startPtr;
				if (diff != length)
					throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length);
			}
			finally
			{
				if (isRenamed != null)
					isRenamed.Close();
			}
		}
Exemple #30
0
        private void  Demo_FSIndexInputBug(Directory fsdir, System.String file)
        {
            // Setup the test file - we need more than 1024 bytes
            IndexOutput os = fsdir.CreateOutput(file);

            for (int i = 0; i < 2000; i++)
            {
                os.WriteByte((byte)i);
            }
            os.Close();

            IndexInput in_Renamed = fsdir.OpenInput(file);

            // This read primes the buffer in IndexInput
            byte b = in_Renamed.ReadByte();

            // Close the file
            in_Renamed.Close();

            // ERROR: this call should fail, but succeeds because the buffer
            // is still filled
            b = in_Renamed.ReadByte();

            // ERROR: this call should fail, but succeeds for some reason as well
            in_Renamed.Seek(1099);

            try
            {
                // OK: this call correctly fails. We are now past the 1024 internal
                // buffer, so an actual IO is attempted, which fails
                b = in_Renamed.ReadByte();
                Assert.Fail("expected readByte() to throw exception");
            }
            catch (System.Exception e)
            {
                // expected exception
            }
        }
Exemple #31
0
        private void  Initialize(SegmentInfo si)
        {
            segment = si.name;

            // Use compound file directory for some files, if it exists
            Directory cfsDir = Directory();

            if (Directory().FileExists(segment + ".cfs"))
            {
                cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
                cfsDir    = cfsReader;
            }

            // No compound file exists - use the multi-file format
            fieldInfos   = new FieldInfos(cfsDir, segment + ".fnm");
            fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);

            tis = new TermInfosReader(cfsDir, segment, fieldInfos);

            // NOTE: the bitvector is stored using the regular directory, not cfs
            if (HasDeletions(si))
            {
                deletedDocs = new BitVector(Directory(), segment + ".del");
            }

            // make sure that all index files have been read or are kept open
            // so that if an index update removes them we'll still have them
            freqStream = cfsDir.OpenInput(segment + ".frq");
            proxStream = cfsDir.OpenInput(segment + ".prx");
            OpenNorms(cfsDir);

            if (fieldInfos.HasVectors())
            {
                // open term vector files only as needed
                termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
            }
        }
        // It is not always neccessary to move the prox pointer
        // to a new document after the freq pointer has been moved.
        // Consider for example a phrase query with two terms:
        // the freq pointer for term 1 has to move to document x
        // to answer the question if the term occurs in that document. But
        // only if term 2 also matches document x, the positions have to be
        // read to figure out if term 1 and term 2 appear next
        // to each other in document x and thus satisfy the query.
        // So we move the prox pointer lazily to the document
        // as soon as positions are requested.
        private void  LazySkip()
        {
            if (proxStream == null)
            {
                // clone lazily
                proxStream = (IndexInput)parent.core.proxStream.Clone();
            }

            // we might have to skip the current payload
            // if it was not read yet
            SkipPayload();

            if (lazySkipPointer != -1)
            {
                proxStream.Seek(lazySkipPointer);
                lazySkipPointer = -1;
            }

            if (lazySkipProxCount != 0)
            {
                SkipPositions(lazySkipProxCount);
                lazySkipProxCount = 0;
            }
        }
Exemple #33
0
        public virtual void  TestReadPastEOF()
        {
            SetUp_2();
            CompoundFileReader cr         = new CompoundFileReader(dir, "f.comp");
            IndexInput         is_Renamed = cr.OpenInput("f2");

            is_Renamed.Seek(is_Renamed.Length() - 10);
            byte[] b = new byte[100];
            is_Renamed.ReadBytes(b, 0, 10);

            try
            {
                byte test = is_Renamed.ReadByte();
                Assert.Fail("Single byte read past end of file");
            }
            catch (System.IO.IOException e)
            {
                /* success */
                //System.out.println("SUCCESS: single byte read past end of file: " + e);
            }

            is_Renamed.Seek(is_Renamed.Length() - 10);
            try
            {
                is_Renamed.ReadBytes(b, 0, 50);
                Assert.Fail("Block read past end of file");
            }
            catch (System.IO.IOException e)
            {
                /* success */
                //System.out.println("SUCCESS: block read past end of file: " + e);
            }

            is_Renamed.Close();
            cr.Close();
        }
            public override System.Object DoBody(System.String segmentFileName)
            {
                IndexInput input = directory.OpenInput(segmentFileName);

                int  format  = 0;
                long version = 0;

                try
                {
                    format = input.ReadInt();
                    if (format < 0)
                    {
                        if (format < Lucene.Net.Index.SegmentInfos.FORMAT_SINGLE_NORM_FILE)
                        {
                            throw new System.IO.IOException("Unknown format version: " + format);
                        }
                        version = input.ReadLong();                         // read version
                    }
                }
                finally
                {
                    input.Close();
                }

                if (format < 0)
                {
                    return((long)version);
                }

                // We cannot be sure about the format of the file.
                // Therefore we have to read the whole file and cannot simply seek to the version entry.
                SegmentInfos sis = new SegmentInfos();

                sis.Read(directory, segmentFileName);
                return((long)sis.GetVersion());
            }
Exemple #35
0
        /// <summary> Current version number from segments file.</summary>
        public static long ReadCurrentVersion(Directory directory)
        {
            IndexInput input   = directory.OpenInput(IndexFileNames.SEGMENTS);
            int        format  = 0;
            long       version = 0;

            try
            {
                format = input.ReadInt();
                if (format < 0)
                {
                    if (format < FORMAT)
                    {
                        throw new System.IO.IOException("Unknown format version: " + format);
                    }
                    version = input.ReadLong();                     // read version
                }
            }
            finally
            {
                input.Close();
            }

            if (format < 0)
            {
                return(version);
            }

            // We cannot be sure about the format of the file.
            // Therefore we have to read the whole file and cannot simply seek to the version entry.

            SegmentInfos sis = new SegmentInfos();

            sis.Read(directory);
            return(sis.GetVersion());
        }
Exemple #36
0
        private System.Collections.ArrayList ReadDeleteableFiles()
        {
            System.Collections.ArrayList result = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10));
            if (!directory.FileExists(IndexFileNames.DELETABLE))
            {
                return(result);
            }

            IndexInput input = directory.OpenInput(IndexFileNames.DELETABLE);

            try
            {
                for (int i = input.ReadInt(); i > 0; i--)
                {
                    // read file names
                    result.Add(input.ReadString());
                }
            }
            finally
            {
                input.Close();
            }
            return(result);
        }
Exemple #37
0
		private void  Initialize(SegmentInfo si)
		{
			segment = si.name;
			this.si = si;
			
			bool success = false;
			
			try
			{
				// Use compound file directory for some files, if it exists
				Directory cfsDir = Directory();
				if (si.GetUseCompoundFile())
				{
					cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
					cfsDir = cfsReader;
				}
				
				// No compound file exists - use the multi-file format
				fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
				fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
				
				// Verify two sources of "maxDoc" agree:
				if (fieldsReader.Size() != si.docCount)
				{
					throw new System.SystemException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
				}
				
				tis = new TermInfosReader(cfsDir, segment, fieldInfos);
				
				// NOTE: the bitvector is stored using the regular directory, not cfs
				if (HasDeletions(si))
				{
					deletedDocs = new BitVector(Directory(), si.GetDelFileName());
					
					// Verify # deletes does not exceed maxDoc for this segment:
					if (deletedDocs.Count() > MaxDoc())
					{
						throw new System.SystemException("number of deletes (" + deletedDocs.Count() + ") exceeds max doc (" + MaxDoc() + ") for segment " + si.name);
					}
				}
				
				// make sure that all index files have been read or are kept open
				// so that if an index update removes them we'll still have them
				freqStream = cfsDir.OpenInput(segment + ".frq");
				proxStream = cfsDir.OpenInput(segment + ".prx");
				OpenNorms(cfsDir);
				
				if (fieldInfos.HasVectors())
				{
					// open term vector files only as needed
					termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
				}
				success = true;
			}
			finally
			{
				
				// With lock-less commits, it's entirely possible (and
				// fine) to hit a FileNotFound exception above.  In
				// this case, we want to explicitly close any subset
				// of things that were opened so that we don't have to
				// wait for a GC to do so.
				if (!success)
				{
					DoClose();
				}
			}
		}
 internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length)
 {
     this.base_Renamed = base_Renamed;
     this.fileOffset   = fileOffset;
     this.length       = length;
 }
			internal CountingStream(TestMultiLevelSkipList enclosingInstance, IndexInput input)
			{
				InitBlock(enclosingInstance);
				this.input = input;
			}
Exemple #40
0
 internal SegmentTermPositions(SegmentReader p) : base(p)
 {
     this.proxStream = (IndexInput)parent.proxStream.Clone();
 }
 /// <summary>Read as a bit set </summary>
 private void  ReadBits(IndexInput input)
 {
     count = input.ReadInt();             // read count
     bits  = new byte[(size >> 3) + 1];   // allocate bits
     input.ReadBytes(bits, 0, bits.Length);
 }
        /// <summary> Construct a new SegmentInfo instance by reading a
        /// previously saved SegmentInfo from input.
        ///
        /// </summary>
        /// <param name="dir">directory to load from
        /// </param>
        /// <param name="format">format of the segments info file
        /// </param>
        /// <param name="input">input handle to read segment info from
        /// </param>
        internal SegmentInfo(Directory dir, int format, IndexInput input)
        {
            this.dir = dir;
            name     = input.ReadString();
            docCount = input.ReadInt();
            if (format <= SegmentInfos.FORMAT_LOCKLESS)
            {
                delGen = input.ReadLong();
                if (format <= SegmentInfos.FORMAT_SHARED_DOC_STORE)
                {
                    docStoreOffset = input.ReadInt();
                    if (docStoreOffset != -1)
                    {
                        docStoreSegment        = input.ReadString();
                        docStoreIsCompoundFile = (1 == input.ReadByte());
                    }
                    else
                    {
                        docStoreSegment        = name;
                        docStoreIsCompoundFile = false;
                    }
                }
                else
                {
                    docStoreOffset         = -1;
                    docStoreSegment        = name;
                    docStoreIsCompoundFile = false;
                }
                if (format <= SegmentInfos.FORMAT_SINGLE_NORM_FILE)
                {
                    hasSingleNormFile = (1 == input.ReadByte());
                }
                else
                {
                    hasSingleNormFile = false;
                }
                int numNormGen = input.ReadInt();
                if (numNormGen == NO)
                {
                    normGen = null;
                }
                else
                {
                    normGen = new long[numNormGen];
                    for (int j = 0; j < numNormGen; j++)
                    {
                        normGen[j] = input.ReadLong();
                    }
                }
                isCompoundFile = (sbyte)input.ReadByte();
                preLockless    = (isCompoundFile == CHECK_DIR);
                if (format <= SegmentInfos.FORMAT_DEL_COUNT)
                {
                    delCount = input.ReadInt();
                    System.Diagnostics.Debug.Assert(delCount <= docCount);
                }
                else
                {
                    delCount = -1;
                }
                if (format <= SegmentInfos.FORMAT_HAS_PROX)
                {
                    hasProx = input.ReadByte() == 1;
                }
                else
                {
                    hasProx = true;
                }

                if (format <= SegmentInfos.FORMAT_DIAGNOSTICS)
                {
                    diagnostics = input.ReadStringStringMap();
                }
                else
                {
                    diagnostics = new Dictionary <string, string>();
                }
            }
            else
            {
                delGen                 = CHECK_DIR;
                normGen                = null;
                isCompoundFile         = (sbyte)(CHECK_DIR);
                preLockless            = true;
                hasSingleNormFile      = false;
                docStoreOffset         = -1;
                docStoreIsCompoundFile = false;
                docStoreSegment        = null;
                delCount               = -1;
                hasProx                = true;
                diagnostics            = new Dictionary <string, string>();
            }
        }
            public System.Object run()
            {
                System.String segmentFileName   = null;
                long          lastGen           = -1;
                long          gen               = 0;
                int           genLookaheadCount = 0;

                System.IO.IOException exc = null;
                bool retry = false;

                int method = 0;

                // Loop until we succeed in calling doBody() without
                // hitting an IOException.  An IOException most likely
                // means a commit was in process and has finished, in
                // the time it took us to load the now-old infos files
                // (and segments files).  It's also possible it's a
                // true error (corrupt index).  To distinguish these,
                // on each retry we must see "forward progress" on
                // which generation we are trying to load.  If we
                // don't, then the original error is real and we throw
                // it.

                // We have three methods for determining the current
                // generation.  We try each in sequence.

                while (true)
                {
                    // Method 1: list the directory and use the highest
                    // segments_N file.  This method works well as long
                    // as there is no stale caching on the directory
                    // contents:
                    System.String[] files = null;

                    if (0 == method)
                    {
                        if (directory != null)
                        {
                            files = directory.List();
                        }
                        else
                        {
                            files = System.IO.Directory.GetFileSystemEntries(fileDirectory.FullName);
                            for (int i = 0; i < files.Length; i++)
                            {
                                files[i] = System.IO.Path.GetFileName(files[i]);
                            }
                        }

                        gen = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files);

                        if (gen == -1)
                        {
                            System.String s = "";
                            for (int i = 0; i < files.Length; i++)
                            {
                                s += (" " + files[i]);
                            }
                            throw new System.IO.FileNotFoundException("no segments* file found: files:" + s);
                        }
                    }

                    // Method 2 (fallback if Method 1 isn't reliable):
                    // if the directory listing seems to be stale, then
                    // try loading the "segments.gen" file.
                    if (1 == method || (0 == method && lastGen == gen && retry))
                    {
                        method = 1;

                        for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++)
                        {
                            IndexInput genInput = null;
                            try
                            {
                                genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN);
                            }
                            catch (System.IO.IOException e)
                            {
                                Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e);
                            }
                            if (genInput != null)
                            {
                                try
                                {
                                    int version = genInput.ReadInt();
                                    if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS)
                                    {
                                        long gen0 = genInput.ReadLong();
                                        long gen1 = genInput.ReadLong();
                                        Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1);
                                        if (gen0 == gen1)
                                        {
                                            // The file is consistent.
                                            if (gen0 > gen)
                                            {
                                                Lucene.Net.Index.SegmentInfos.Message("fallback to '" + IndexFileNames.SEGMENTS_GEN + "' check: now try generation " + gen0 + " > " + gen);
                                                gen = gen0;
                                            }
                                            break;
                                        }
                                    }
                                }
                                catch (System.IO.IOException err2)
                                {
                                    // will retry
                                }
                                finally
                                {
                                    genInput.Close();
                                }
                            }
                            try
                            {
                                System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec));
                            }
                            catch (System.Threading.ThreadInterruptedException e)
                            {
                                // will retry
                            }
                        }
                    }

                    // Method 3 (fallback if Methods 2 & 3 are not
                    // reliable): since both directory cache and file
                    // contents cache seem to be stale, just advance the
                    // generation.
                    if (2 == method || (1 == method && lastGen == gen && retry))
                    {
                        method = 2;

                        if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount)
                        {
                            gen++;
                            genLookaheadCount++;
                            Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen);
                        }
                    }

                    if (lastGen == gen)
                    {
                        // This means we're about to try the same
                        // segments_N last tried.  This is allowed,
                        // exactly once, because writer could have been in
                        // the process of writing segments_N last time.

                        if (retry)
                        {
                            // OK, we've tried the same segments_N file
                            // twice in a row, so this must be a real
                            // error.  We throw the original exception we
                            // got.
                            throw exc;
                        }
                        else
                        {
                            retry = true;
                        }
                    }
                    else
                    {
                        // Segment file has advanced since our last loop, so
                        // reset retry:
                        retry = false;
                    }

                    lastGen = gen;

                    segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);

                    try
                    {
                        System.Object v = DoBody(segmentFileName);
                        if (exc != null)
                        {
                            Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
                        }
                        return(v);
                    }
                    catch (System.IO.IOException err)
                    {
                        // Save the original root cause:
                        if (exc == null)
                        {
                            exc = err;
                        }

                        Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);

                        if (!retry && gen > 1)
                        {
                            // This is our first time trying this segments
                            // file (because retry is false), and, there is
                            // possibly a segments_(N-1) (because gen > 1).
                            // So, check if the segments_(N-1) exists and
                            // try it if so:
                            System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);

                            if (directory.FileExists(prevSegmentFileName))
                            {
                                Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'");
                                try
                                {
                                    System.Object v = DoBody(prevSegmentFileName);
                                    if (exc != null)
                                    {
                                        Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName);
                                    }
                                    return(v);
                                }
                                catch (System.IO.IOException err2)
                                {
                                    Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
                                }
                            }
                        }
                    }
                }
            }
Exemple #44
0
 internal static bool IsCSIndexInput(IndexInput is_Renamed)
 {
     return(is_Renamed is CompoundFileReader.CSIndexInput);
 }
 internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length, int readBufferSize) : base(readBufferSize)
 {
     this.base_Renamed = (IndexInput)base_Renamed.Clone();
     this.fileOffset   = fileOffset;
     this.length       = length;
 }
 internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length) : this(base_Renamed, fileOffset, length, BufferedIndexInput.BUFFER_SIZE)
 {
 }
 internal SkipBuffer(IndexInput input, int length)
 {
     data    = new byte[length];
     pointer = input.GetFilePointer();
     input.ReadBytes(data, 0, length);
 }
 /// <summary> Subclasses must implement the actual skip data encoding in this method.
 ///
 /// </summary>
 /// <param name="level">the level skip data shall be read from
 /// </param>
 /// <param name="skipStream">the skip stream to read from
 /// </param>
 protected internal abstract int ReadSkipData(int level, IndexInput skipStream);
 // Used only by clone
 private FieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int formatSize, int docStoreOffset, IndexInput cloneableFieldsStream, IndexInput cloneableIndexStream)
 {
     this.fieldInfos            = fieldInfos;
     this.numTotalDocs          = numTotalDocs;
     this.size                  = size;
     this.format                = format;
     this.formatSize            = formatSize;
     this.docStoreOffset        = docStoreOffset;
     this.cloneableFieldsStream = cloneableFieldsStream;
     this.cloneableIndexStream  = cloneableIndexStream;
     fieldsStream               = (IndexInput)cloneableFieldsStream.Clone();
     indexStream                = (IndexInput)cloneableIndexStream.Clone();
 }
			internal FaultyIndexInput(IndexInput delegate_Renamed)
			{
				this.delegate_Renamed = delegate_Renamed;
			}
        /// <summary> Read a particular segmentFileName.  Note that this may
        /// throw an IOException if a commit is in process.
        ///
        /// </summary>
        /// <param name="directory">-- directory containing the segments file
        /// </param>
        /// <param name="segmentFileName">-- segment file to load
        /// </param>
        public void  Read(Directory directory, System.String segmentFileName)
        {
            bool success = false;

            IndexInput input = directory.OpenInput(segmentFileName);

            if (segmentFileName.Equals(IndexFileNames.SEGMENTS))
            {
                generation = 0;
            }
            else
            {
#if !PRE_LUCENE_NET_2_0_0_COMPATIBLE
                generation = Lucene.Net.Documents.NumberTools.ToLong(segmentFileName.Substring(1 + IndexFileNames.SEGMENTS.Length));
#else
                generation = System.Convert.ToInt64(segmentFileName.Substring(1 + IndexFileNames.SEGMENTS.Length), 16);
#endif
            }
            lastGeneration = generation;

            try
            {
                int format = input.ReadInt();
                if (format < 0)
                {
                    // file contains explicit format info
                    // check that it is a format we can understand
                    if (format < FORMAT_SINGLE_NORM_FILE)
                    {
                        throw new System.IO.IOException("Unknown format version: " + format);
                    }
                    version = input.ReadLong();                    // read version
                    counter = input.ReadInt();                     // read counter
                }
                else
                {
                    // file is in old format without explicit format info
                    counter = format;
                }

                for (int i = input.ReadInt(); i > 0; i--)
                {
                    // read segmentInfos
                    Add(new SegmentInfo(directory, format, input));
                }

                if (format >= 0)
                {
                    // in old format the version number may be at the end of the file
                    if (input.GetFilePointer() >= input.Length())
                    {
                        version = System.DateTime.Now.Millisecond;
                    }
                    // old file format without version number
                    else
                    {
                        version = input.ReadLong();                         // read version
                    }
                }
                success = true;
            }
            finally
            {
                input.Close();
                if (!success)
                {
                    // Clear any segment infos we had loaded so we
                    // have a clean slate on retry:
                    Clear();
                }
            }
        }
        /// <summary>Optimized implementation. </summary>
        public virtual bool SkipTo(int target)
        {
            if (df >= skipInterval)
            {
                // optimized case

                if (skipStream == null)
                {
                    skipStream = (IndexInput)freqStream.Clone();                      // lazily clone
                }
                if (!haveSkipped)
                {
                    // lazily seek skip stream
                    skipStream.Seek(skipPointer);
                    haveSkipped = true;
                }

                // scan skip data
                int  lastSkipDoc     = skipDoc;
                long lastFreqPointer = freqStream.GetFilePointer();
                long lastProxPointer = -1;
                int  numSkipped      = -1 - (count % skipInterval);

                while (target > skipDoc)
                {
                    lastSkipDoc     = skipDoc;
                    lastFreqPointer = freqPointer;
                    lastProxPointer = proxPointer;

                    if (skipDoc != 0 && skipDoc >= doc)
                    {
                        numSkipped += skipInterval;
                    }

                    if (skipCount >= numSkips)
                    {
                        break;
                    }

                    skipDoc     += skipStream.ReadVInt();
                    freqPointer += skipStream.ReadVInt();
                    proxPointer += skipStream.ReadVInt();

                    skipCount++;
                }

                // if we found something to skip, then skip it
                if (lastFreqPointer > freqStream.GetFilePointer())
                {
                    freqStream.Seek(lastFreqPointer);
                    SkipProx(lastProxPointer);

                    doc    = lastSkipDoc;
                    count += numSkipped;
                }
            }

            // done skipping, now just scan
            do
            {
                if (!Next())
                {
                    return(false);
                }
            }while (target > doc);
            return(true);
        }
		private void  Read(IndexInput input, System.String fileName)
		{
			int firstInt = input.ReadVInt();
			
			if (firstInt < 0)
			{
				// This is a real format
				format = firstInt;
			}
			else
			{
				format = FORMAT_PRE;
			}
			
			if (format != FORMAT_PRE & format != FORMAT_START)
			{
				throw new CorruptIndexException("unrecognized format " + format + " in file \"" + fileName + "\"");
			}
			
			int size;
			if (format == FORMAT_PRE)
			{
				size = firstInt;
			}
			else
			{
				size = input.ReadVInt(); //read in the size
			}
			
			for (int i = 0; i < size; i++)
			{
				System.String name = StringHelper.Intern(input.ReadString());
				byte bits = input.ReadByte();
				bool isIndexed = (bits & IS_INDEXED) != 0;
				bool storeTermVector = (bits & STORE_TERMVECTOR) != 0;
				bool storePositionsWithTermVector = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
				bool storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
				bool omitNorms = (bits & OMIT_NORMS) != 0;
				bool storePayloads = (bits & STORE_PAYLOADS) != 0;
				bool omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;
				
				AddInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
			}
			
			if (input.GetFilePointer() != input.Length())
			{
				throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.GetFilePointer() + " vs size " + input.Length());
			}
		}
Exemple #54
0
        public virtual void  TestRandomAccessClones()
        {
            SetUp_2();
            CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

            // Open two files
            IndexInput e1 = cr.OpenInput("f11");
            IndexInput e2 = cr.OpenInput("f3");

            IndexInput a1 = (IndexInput)e1.Clone();
            IndexInput a2 = (IndexInput)e2.Clone();

            // Seek the first pair
            e1.Seek(100);
            a1.Seek(100);
            Assert.AreEqual(100, e1.GetFilePointer());
            Assert.AreEqual(100, a1.GetFilePointer());
            byte be1 = e1.ReadByte();
            byte ba1 = a1.ReadByte();

            Assert.AreEqual(be1, ba1);

            // Now seek the second pair
            e2.Seek(1027);
            a2.Seek(1027);
            Assert.AreEqual(1027, e2.GetFilePointer());
            Assert.AreEqual(1027, a2.GetFilePointer());
            byte be2 = e2.ReadByte();
            byte ba2 = a2.ReadByte();

            Assert.AreEqual(be2, ba2);

            // Now make sure the first one didn't move
            Assert.AreEqual(101, e1.GetFilePointer());
            Assert.AreEqual(101, a1.GetFilePointer());
            be1 = e1.ReadByte();
            ba1 = a1.ReadByte();
            Assert.AreEqual(be1, ba1);

            // Now more the first one again, past the buffer length
            e1.Seek(1910);
            a1.Seek(1910);
            Assert.AreEqual(1910, e1.GetFilePointer());
            Assert.AreEqual(1910, a1.GetFilePointer());
            be1 = e1.ReadByte();
            ba1 = a1.ReadByte();
            Assert.AreEqual(be1, ba1);

            // Now make sure the second set didn't move
            Assert.AreEqual(1028, e2.GetFilePointer());
            Assert.AreEqual(1028, a2.GetFilePointer());
            be2 = e2.ReadByte();
            ba2 = a2.ReadByte();
            Assert.AreEqual(be2, ba2);

            // Move the second set back, again cross the buffer size
            e2.Seek(17);
            a2.Seek(17);
            Assert.AreEqual(17, e2.GetFilePointer());
            Assert.AreEqual(17, a2.GetFilePointer());
            be2 = e2.ReadByte();
            ba2 = a2.ReadByte();
            Assert.AreEqual(be2, ba2);

            // Finally, make sure the first set didn't move
            // Now make sure the first one didn't move
            Assert.AreEqual(1911, e1.GetFilePointer());
            Assert.AreEqual(1911, a1.GetFilePointer());
            be1 = e1.ReadByte();
            ba1 = a1.ReadByte();
            Assert.AreEqual(be1, ba1);

            e1.Close();
            e2.Close();
            a1.Close();
            a2.Close();
            cr.Close();
        }
            protected override void Dispose(bool disposing)
            {
                if (isDisposed) return;
                if (disposing)
                {
                    if (delegate_Renamed != null)
                    {
                        delegate_Renamed.Close();
                    }
                }

                delegate_Renamed = null;
                isDisposed = true;
            }
        internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size)
        {
            bool success = false;

            isOriginal = true;
            try
            {
                fieldInfos = fn;

                cloneableFieldsStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_EXTENSION, readBufferSize);
                cloneableIndexStream  = d.OpenInput(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION, readBufferSize);

                // First version of fdx did not include a format
                // header, but, the first int will always be 0 in that
                // case
                int firstInt = cloneableIndexStream.ReadInt();
                format = firstInt == 0 ? 0 : firstInt;

                if (format > FieldsWriter.FORMAT_CURRENT)
                {
                    throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FieldsWriter.FORMAT_CURRENT + " or lower");
                }

                formatSize = format > FieldsWriter.FORMAT ? 4 : 0;

                if (format < FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
                {
                    cloneableFieldsStream.SetModifiedUTF8StringsMode();
                }

                fieldsStream = (IndexInput)cloneableFieldsStream.Clone();

                long indexSize = cloneableIndexStream.Length() - formatSize;

                if (docStoreOffset != -1)
                {
                    // We read only a slice out of this shared fields file
                    this.docStoreOffset = docStoreOffset;
                    this.size           = size;

                    // Verify the file is long enough to hold all of our
                    // docs
                    System.Diagnostics.Debug.Assert(((int)(indexSize / 8)) >= size + this.docStoreOffset, "indexSize=" + indexSize + " size=" + size + " docStoreOffset=" + docStoreOffset);
                }
                else
                {
                    this.docStoreOffset = 0;
                    this.size           = (int)(indexSize >> 3);
                }

                indexStream  = (IndexInput)cloneableIndexStream.Clone();
                numTotalDocs = (int)(indexSize >> 3);
                success      = true;
            }
            finally
            {
                // With lock-less commits, it's entirely possible (and
                // fine) to hit a FileNotFound exception above. In
                // this case, we want to explicitly close any subset
                // of things that were opened so that we don't have to
                // wait for a GC to do so.
                if (!success)
                {
                    Dispose();
                }
            }
        }
 internal DefaultSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) : base(skipStream, maxSkipLevels, skipInterval)
 {
     freqPointer   = new long[maxSkipLevels];
     proxPointer   = new long[maxSkipLevels];
     payloadLength = new int[maxSkipLevels];
 }
Exemple #58
0
        internal SegmentTermEnum(IndexInput i, FieldInfos fis, bool isi)
        {
            input         = i;
            fieldInfos    = fis;
            isIndex       = isi;
            maxSkipLevels = 1;             // use single-level skip lists for formats > -3

            int firstInt = input.ReadInt();

            if (firstInt >= 0)
            {
                // original-format file, without explicit format version number
                format = 0;
                size   = firstInt;

                // back-compatible settings
                indexInterval = 128;
                skipInterval  = System.Int32.MaxValue;                // switch off skipTo optimization
            }
            else
            {
                // we have a format version number
                format = firstInt;

                // check that it is a format we can understand
                if (format < TermInfosWriter.FORMAT_CURRENT)
                {
                    throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher");
                }

                size = input.ReadLong();                 // read the size

                if (format == -1)
                {
                    if (!isIndex)
                    {
                        indexInterval        = input.ReadInt();
                        formatM1SkipInterval = input.ReadInt();
                    }
                    // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in
                    // skipTo implementation of these versions
                    skipInterval = System.Int32.MaxValue;
                }
                else
                {
                    indexInterval = input.ReadInt();
                    skipInterval  = input.ReadInt();
                    if (format <= TermInfosWriter.FORMAT)
                    {
                        // this new format introduces multi-level skipping
                        maxSkipLevels = input.ReadInt();
                    }
                }
            }
            if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
            {
                termBuffer.SetPreUTF8Strings();
                scanBuffer.SetPreUTF8Strings();
                prevBuffer.SetPreUTF8Strings();
            }
        }
Exemple #59
0
        internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size)
        {
            bool success = false;
            isOriginal = true;
            try
            {
                fieldInfos = fn;

                cloneableFieldsStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_EXTENSION, readBufferSize);
                cloneableIndexStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION, readBufferSize);

                // First version of fdx did not include a format
                // header, but, the first int will always be 0 in that
                // case
                int firstInt = cloneableIndexStream.ReadInt();
                format = firstInt == 0 ? 0 : firstInt;

                if (format > FieldsWriter.FORMAT_CURRENT)
                    throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FieldsWriter.FORMAT_CURRENT + " or lower");

                formatSize = format > FieldsWriter.FORMAT ? 4 : 0;

                if (format < FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
                    cloneableFieldsStream.SetModifiedUTF8StringsMode();

                fieldsStream = (IndexInput) cloneableFieldsStream.Clone();

                long indexSize = cloneableIndexStream.Length() - formatSize;

                if (docStoreOffset != - 1)
                {
                    // We read only a slice out of this shared fields file
                    this.docStoreOffset = docStoreOffset;
                    this.size = size;

                    // Verify the file is long enough to hold all of our
                    // docs
                    System.Diagnostics.Debug.Assert(((int)(indexSize / 8)) >= size + this.docStoreOffset, "indexSize=" + indexSize + " size=" + size + " docStoreOffset=" + docStoreOffset);
                }
                else
                {
                    this.docStoreOffset = 0;
                    this.size = (int) (indexSize >> 3);
                }

                indexStream = (IndexInput) cloneableIndexStream.Clone();
                numTotalDocs = (int) (indexSize >> 3);
                success = true;
            }
            finally
            {
                // With lock-less commits, it's entirely possible (and
                // fine) to hit a FileNotFound exception above. In
                // this case, we want to explicitly close any subset
                // of things that were opened so that we don't have to
                // wait for a GC to do so.
                if (!success)
                {
                    Dispose();
                }
            }
        }
Exemple #60
0
 internal CountingStream(TestMultiLevelSkipList enclosingInstance, IndexInput input)
 {
     InitBlock(enclosingInstance);
     this.input = input;
 }