public void Read(IndexInput input, FieldInfos fieldInfos, IState state) { this.term = null; // invalidate cache int start = input.ReadVInt(state); int length = input.ReadVInt(state); int totalLength = start + length; if (preUTF8Strings) { text.SetLength(totalLength); input.ReadChars(text.result, start, length, state); } else { if (dirty) { // Fully convert all bytes since bytes is dirty UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes); bytes.SetLength(totalLength); input.ReadBytes(bytes.result, start, length, state); UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text); dirty = false; } else { // Incrementally convert only the UTF8 bytes that are new: bytes.SetLength(totalLength); input.ReadBytes(bytes.result, start, length, state); UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text); } } this.field = fieldInfos.FieldName(input.ReadVInt(state)); }
protected internal override int ReadSkipData(int level, IndexInput skipStream, IState state) { int delta; if (currentFieldStoresPayloads) { // the current field stores payloads. // if the doc delta is odd then we have // to read the current payload length // because it differs from the length of the // previous payload delta = skipStream.ReadVInt(state); if ((delta & 1) != 0) { payloadLength[level] = skipStream.ReadVInt(state); } delta = Number.URShift(delta, 1); } else { delta = skipStream.ReadVInt(state); } freqPointer[level] += skipStream.ReadVInt(state); proxPointer[level] += skipStream.ReadVInt(state); return(delta); }
protected override MonotonicBlockPackedReader GetOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) { data.Seek(entry.Offset); return new MonotonicBlockPackedReader((IndexInput)data.Clone(), entry.PackedIntsVersion, entry.BlockSize, entry.Count, true); }
public MockSingleIntIndexInput(Directory dir, string fileName, IOContext context) { @in = dir.OpenInput(fileName, context); CodecUtil.CheckHeader(@in, MockSingleIntIndexOutput.CODEC, MockSingleIntIndexOutput.VERSION_START, MockSingleIntIndexOutput.VERSION_START); }
protected override void Dispose(bool disposing) { lock (this) { if (isDisposed) { return; } if (disposing) { if (entries != null) { entries.Clear(); } if (stream != null) { stream.Close(); } } entries = null; stream = null; isDisposed = true; } }
/// <summary> /// Checks that the stream is positioned at the end, and throws exception /// if it is not. </summary> /// @deprecated Use <seealso cref="#checkFooter"/> instead, this should only used for files without checksums public static void CheckEOF(IndexInput @in) { if (@in.FilePointer != @in.Length()) { throw new System.IO.IOException("did not read all bytes from file: read " + @in.FilePointer + " vs size " + @in.Length() + " (resource: " + @in + ")"); } }
private int CopyFieldsNoDeletions(FieldsWriter fieldsWriter, IndexReader reader, FieldsReader matchingFieldsReader, IState state) { int maxDoc = reader.MaxDoc; int docCount = 0; if (matchingFieldsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" while (docCount < maxDoc) { int len = System.Math.Min(MAX_RAW_MERGE_DOCS, maxDoc - docCount); IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, docCount, len, state); fieldsWriter.AddRawDocuments(stream, rawDocLengths, len, state); docCount += len; checkAbort.Work(300 * len, state); } } else { for (; docCount < maxDoc; docCount++) { // NOTE: it's very important to first assign to doc then pass it to // termVectorsWriter.addAllDocVectors; see LUCENE-1282 Document doc = reader.Document(docCount, state); fieldsWriter.AddDocument(doc, state); checkAbort.Work(300, state); } } return(docCount); }
/// <summary> /// Construct an empty output buffer. </summary> public MockIndexInputWrapper(MockDirectoryWrapper dir, string name, IndexInput @delegate) : base("MockIndexInputWrapper(name=" + name + " delegate=" + @delegate + ")") { this.Name = name; this.Dir = dir; this.@delegate = @delegate; }
protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) { data.Seek(bytes.AddressesOffset); return new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count, true); }
private int CopyFieldsWithDeletions(FieldsWriter fieldsWriter, IndexReader reader, FieldsReader matchingFieldsReader, IState state) { int docCount = 0; int maxDoc = reader.MaxDoc; if (matchingFieldsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" for (int j = 0; j < maxDoc;) { if (reader.IsDeleted(j)) { // skip deleted docs ++j; continue; } // We can optimize this case (doing a bulk byte copy) since the field // numbers are identical int start = j, numDocs = 0; do { j++; numDocs++; if (j >= maxDoc) { break; } if (reader.IsDeleted(j)) { j++; break; } }while (numDocs < MAX_RAW_MERGE_DOCS); IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, start, numDocs, state); fieldsWriter.AddRawDocuments(stream, rawDocLengths, numDocs, state); docCount += numDocs; checkAbort.Work(300 * numDocs, state); } } else { for (int j = 0; j < maxDoc; j++) { if (reader.IsDeleted(j)) { // skip deleted docs continue; } // NOTE: it's very important to first assign to doc then pass it to // termVectorsWriter.addAllDocVectors; see LUCENE-1282 Document doc = reader.Document(j, state); fieldsWriter.AddDocument(doc, state); docCount++; checkAbort.Work(300, state); } } return(docCount); }
private int CheckValidFormat(IndexInput in_Renamed, IState state) { int format = in_Renamed.ReadInt(state); if (format > FORMAT_CURRENT) { throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FORMAT_CURRENT + " or less"); } return(format); }
/// <summary>Returns true if the provided input stream is an SimpleFSIndexInput and /// is a clone, that is it does not own its underlying file descriptor. /// </summary> public static bool IsSimpleFSIndexInputClone(IndexInput is_Renamed) { if (IsSimpleFSIndexInput(is_Renamed)) { return ((SimpleFSIndexInput) is_Renamed).isClone_ForNUnit; } else { return false; } }
public CompoundFileReader(Directory dir, System.String name, int readBufferSize, IState state) { directory = dir; fileName = name; this.readBufferSize = readBufferSize; bool success = false; try { stream = dir.OpenInput(name, readBufferSize, state); // read the directory and init files int count = stream.ReadVInt(state); FileEntry entry = null; for (int i = 0; i < count; i++) { long offset = stream.ReadLong(state); System.String id = stream.ReadString(state); if (entry != null) { // set length of the previous entry entry.length = offset - entry.offset; } entry = new FileEntry { offset = offset }; entries[id] = entry; } // set the length of the final entry if (entry != null) { entry.length = stream.Length(state) - entry.offset; } success = true; } finally { if (!success && (stream != null)) { try { stream.Close(); } catch (System.IO.IOException) { } } } }
/// <summary> /// Returns true if the provided input stream is an SimpleFSIndexInput and /// is a clone, that is it does not own its underlying file descriptor. /// </summary> public static bool IsSimpleFSIndexInputClone(IndexInput @is) { if (IsSimpleFSIndexInput(@is)) { return ((SimpleFSIndexInput)@is).IsClone; } else { return false; } }
public override IndexInput OpenInput(System.String name, IState state) { IndexInput ii = base.OpenInput(name, null); if (name.EndsWith(".prx")) { // we decorate the proxStream with a wrapper class that allows to count the number of calls of seek() ii = new SeeksCountingStream(enclosingInstance, ii); } return(ii); }
private IndexInput GetFieldStream(IState state) { IndexInput localFieldsStream = Enclosing_Instance.fieldsStreamTL.Get(state); if (localFieldsStream == null) { localFieldsStream = (IndexInput)Enclosing_Instance.cloneableFieldsStream.Clone(state); Enclosing_Instance.fieldsStreamTL.Set(localFieldsStream); } return(localFieldsStream); }
public /*protected internal*/ SegmentTermDocs(SegmentReader parent, IState state) { this.parent = parent; this.freqStream = (IndexInput)parent.core.freqStream.Clone(state); lock (parent) { this.deletedDocs = parent.deletedDocs; } this.skipInterval = parent.core.GetTermsReader().SkipInterval; this.maxSkipLevels = parent.core.GetTermsReader().MaxSkipLevels; }
/// <summary> /// Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns /// true if the underlying file descriptor is valid, and false otherwise. /// this can be used to determine if the OS file has been closed. /// The descriptor becomes invalid when the non-clone instance of the /// SimpleFSIndexInput that owns this descriptor is closed. However, the /// descriptor may possibly become invalid in other ways as well. /// </summary> public static bool IsSimpleFSIndexInputOpen(IndexInput @is) { if (IsSimpleFSIndexInput(@is)) { SimpleFSIndexInput fis = (SimpleFSIndexInput)@is; return fis.FDValid; } else { return false; } }
/// <summary>Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns /// true if the underlying file descriptor is valid, and false otherwise. /// This can be used to determine if the OS file has been closed. /// The descriptor becomes invalid when the non-clone instance of the /// SimpleFSIndexInput that owns this descriptor is closed. However, the /// descriptor may possibly become invalid in other ways as well. /// </summary> public static bool IsSimpleFSIndexInputOpen(IndexInput is_Renamed) { if (IsSimpleFSIndexInput(is_Renamed)) { SimpleFSIndexInput fis = (SimpleFSIndexInput) is_Renamed; return fis.IsFDValid(); } else { return false; } }
/// <summary>Bulk write a contiguous series of documents. The /// lengths array is the length (in bytes) of each raw /// document. The stream IndexInput is the /// fieldsStream from which we should bulk-copy all /// bytes. /// </summary> internal void AddRawDocuments(IndexInput stream, int[] lengths, int numDocs, IState state) { long position = fieldsStream.FilePointer; long start = position; for (int i = 0; i < numDocs; i++) { indexStream.WriteLong(position); position += lengths[i]; } fieldsStream.CopyBytes(stream, position - start, state); System.Diagnostics.Debug.Assert(fieldsStream.FilePointer == position); }
// Used only by clone private FieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int formatSize, int docStoreOffset, IndexInput cloneableFieldsStream, IndexInput cloneableIndexStream, IState state) { this.fieldInfos = fieldInfos; this.numTotalDocs = numTotalDocs; this.size = size; this.format = format; this.formatSize = formatSize; this.docStoreOffset = docStoreOffset; this.cloneableFieldsStream = cloneableFieldsStream; this.cloneableIndexStream = cloneableIndexStream; fieldsStream = (IndexInput)cloneableFieldsStream.Clone(state); indexStream = (IndexInput)cloneableIndexStream.Clone(state); }
/// <summary> Construct a FieldInfos object using the directory and the name of the file /// IndexInput /// </summary> /// <param name="d">The directory to open the IndexInput from /// </param> /// <param name="name">The name of the file to open the IndexInput from in the Directory /// </param> /// <throws> IOException </throws> public /*internal*/ FieldInfos(Directory d, String name, IState state) { IndexInput input = d.OpenInput(name, state); try { try { Read(input, name, state); } catch (System.IO.IOException) { if (format == FORMAT_PRE) { // LUCENE-1623: FORMAT_PRE (before there was a // format) may be 2.3.2 (pre-utf8) or 2.4.x (utf8) // encoding; retry with input set to pre-utf8 input.Seek(0, state); input.SetModifiedUTF8StringsMode(); byNumber.Clear(); byName.Clear(); bool rethrow = false; try { Read(input, name, state); } catch (Exception) { // Ignore any new exception & set to throw original IOE rethrow = true; } if (rethrow) { // Preserve stack trace throw; } } else { // The IOException cannot be caused by // LUCENE-1623, so re-throw it throw; } } } finally { input.Close(); } }
// used by clone private CompressingTermVectorsReader(CompressingTermVectorsReader reader) { this.fieldInfos = reader.fieldInfos; this.vectorsStream = (IndexInput)reader.vectorsStream.Clone(); this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone(); this.packedIntsVersion = reader.packedIntsVersion; this.compressionMode = reader.compressionMode; this.decompressor = (Decompressor)reader.decompressor.Clone(); this.chunkSize = reader.chunkSize; this.numDocs = reader.numDocs; this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0); this.version = reader.version; this.closed = false; }
private void Read(IndexInput input, String fileName, IState state) { int firstInt = input.ReadVInt(state); if (firstInt < 0) { // This is a real format format = firstInt; } else { format = FORMAT_PRE; } if (format != FORMAT_PRE & format != FORMAT_START) { throw new CorruptIndexException("unrecognized format " + format + " in file \"" + fileName + "\""); } int size; if (format == FORMAT_PRE) { size = firstInt; } else { size = input.ReadVInt(state); //read in the size } for (int i = 0; i < size; i++) { String name = StringHelper.Intern(input.ReadString(state)); byte bits = input.ReadByte(state); bool isIndexed = (bits & IS_INDEXED) != 0; bool storeTermVector = (bits & STORE_TERMVECTOR) != 0; bool storePositionsWithTermVector = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0; bool storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0; bool omitNorms = (bits & OMIT_NORMS) != 0; bool storePayloads = (bits & STORE_PAYLOADS) != 0; bool omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0; AddInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); } if (input.FilePointer(state) != input.Length(state)) { throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.FilePointer(state) + " vs size " + input.Length(state)); } }
/// <summary>Copy the contents of the file with specified extension into the /// provided output stream. Use the provided buffer for moving data /// to reduce memory allocation. /// </summary> private void CopyFile(FileEntry source, IndexOutput os, byte[] buffer, IState state) { IndexInput isRenamed = null; try { long startPtr = os.FilePointer; isRenamed = directory.OpenInput(source.file, state); long length = isRenamed.Length(state); long remainder = length; int chunk = buffer.Length; while (remainder > 0) { var len = (int)Math.Min(chunk, remainder); isRenamed.ReadBytes(buffer, 0, len, false, state); os.WriteBytes(buffer, len); remainder -= len; if (checkAbort != null) { // Roughly every 2 MB we will check if // it's time to abort checkAbort.Work(80, state); } } // Verify that remainder is 0 if (remainder != 0) { throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")"); } // Verify that the output length diff is equal to original file long endPtr = os.FilePointer; long diff = endPtr - startPtr; if (diff != length) { throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length); } } finally { if (isRenamed != null) { isRenamed.Close(); } } }
/// <summary>read as a d-gaps list </summary> private void ReadDgaps(IndexInput input, IState state) { size = input.ReadInt(state); // (re)read size count = input.ReadInt(state); // read count bits = new byte[(size >> 3) + 1]; // allocate bits int last = 0; int n = Count(); while (n > 0) { last += input.ReadVInt(state); bits[last] = input.ReadByte(state); n -= BYTE_COUNTS[bits[last] & 0xFF]; } }
protected override void Dispose(bool disposing) { if (isDisposed) return; if (disposing) { if (main != null) { main.Dispose(); } } main = null; isDisposed = true; }
/// <summary>Copy numBytes bytes from input to ourself. </summary> public virtual void CopyBytes(IndexInput input, long numBytes) { long left = numBytes; if (copyBuffer == null) copyBuffer = new byte[COPY_BUFFER_SIZE]; while (left > 0) { int toCopy; if (left > COPY_BUFFER_SIZE) toCopy = COPY_BUFFER_SIZE; else toCopy = (int) left; input.ReadBytes(copyBuffer, 0, toCopy); WriteBytes(copyBuffer, 0, toCopy); left -= toCopy; } }
protected override void Dispose(bool disposing) { if (isDisposed) { return; } if (disposing) { if (delegate_Renamed != null) { delegate_Renamed.Close(); } } delegate_Renamed = null; isDisposed = true; }
/// <summary> /// Sole constructor. </summary> public BlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct) { this.ValueCount = valueCount; BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE); BlockMask = blockSize - 1; int numBlocks = PackedInts.NumBlocks(valueCount, blockSize); long[] minValues = null; SubReaders = new PackedInts.Reader[numBlocks]; for (int i = 0; i < numBlocks; ++i) { int token = @in.ReadByte() & 0xFF; int bitsPerValue = (int)((uint)token >> AbstractBlockPackedWriter.BPV_SHIFT); if (bitsPerValue > 64) { throw new Exception("Corrupted"); } if ((token & AbstractBlockPackedWriter.MIN_VALUE_EQUALS_0) == 0) { if (minValues == null) { minValues = new long[numBlocks]; } minValues[i] = BlockPackedReaderIterator.ZigZagDecode(1L + BlockPackedReaderIterator.ReadVLong(@in)); } if (bitsPerValue == 0) { SubReaders[i] = new PackedInts.NullReader(blockSize); } else { int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize); if (direct) { long pointer = @in.FilePointer; SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); @in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue)); } else { SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); } } } this.MinValues = minValues; }
/// <summary>The value of the field as a String, or null. If null, the Reader value, /// binary value, or TokenStream value is used. Exactly one of StringValue(), /// ReaderValue(), GetBinaryValue(), and TokenStreamValue() must be set. /// </summary> public override string StringValue(IState state) { Enclosing_Instance.EnsureOpen(); if (internalIsBinary) { return(null); } if (fieldsData == null) { IndexInput localFieldsStream = GetFieldStream(state); try { localFieldsStream.Seek(pointer, state); if (isCompressed) { var b = new byte[toRead]; localFieldsStream.ReadBytes(b, 0, b.Length, state); fieldsData = System.Text.Encoding.GetEncoding("UTF-8").GetString(Enclosing_Instance.Uncompress(b)); } else { if (Enclosing_Instance.format >= FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { var bytes = new byte[toRead]; localFieldsStream.ReadBytes(bytes, 0, toRead, state); fieldsData = System.Text.Encoding.GetEncoding("UTF-8").GetString(bytes); } else { //read in chars b/c we already know the length we need to read var chars = new char[toRead]; localFieldsStream.ReadChars(chars, 0, toRead, state); fieldsData = new System.String(chars); } } } catch (System.IO.IOException e) { throw new FieldReaderException(e); } } return((System.String)fieldsData); }
protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) { this.skipStream = new IndexInput[maxSkipLevels]; this.skipPointer = new long[maxSkipLevels]; this.childPointer = new long[maxSkipLevels]; this.numSkipped = new int[maxSkipLevels]; this.maxNumberOfSkipLevels = maxSkipLevels; this.skipInterval = new int[maxSkipLevels]; this.skipStream[0] = skipStream; this.inputIsBuffered = (skipStream is BufferedIndexInput); this.skipInterval[0] = skipInterval; for (int i = 1; i < maxSkipLevels; i++) { // cache skip intervals this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval; } skipDoc = new int[maxSkipLevels]; }
/// <summary>Copy numBytes bytes from input to ourself. </summary> public virtual void CopyBytes(IndexInput input, long numBytes) { System.Diagnostics.Debug.Assert(numBytes >= 0, "numBytes=" + numBytes); long left = numBytes; if (copyBuffer == null) copyBuffer = new byte[COPY_BUFFER_SIZE]; while (left > 0) { int toCopy; if (left > COPY_BUFFER_SIZE) toCopy = COPY_BUFFER_SIZE; else toCopy = (int) left; input.ReadBytes(copyBuffer, 0, toCopy); WriteBytes(copyBuffer, 0, toCopy); left -= toCopy; } }
public override void Init(IndexInput termsIn) { version = CodecUtil.CheckHeader(termsIn, PulsingPostingsWriter.CODEC, PulsingPostingsWriter.VERSION_START, PulsingPostingsWriter.VERSION_CURRENT); maxPositions = termsIn.ReadVInt(); _wrappedPostingsReader.Init(termsIn); if (_wrappedPostingsReader is PulsingPostingsReader || version < PulsingPostingsWriter.VERSION_META_ARRAY) { fields = null; } else { fields = new SortedDictionary<int, int>(); String summaryFileName = IndexFileNames.SegmentFileName(segmentState.SegmentInfo.Name, segmentState.SegmentSuffix, PulsingPostingsWriter.SUMMARY_EXTENSION); IndexInput input = null; try { input = segmentState.Directory.OpenInput(summaryFileName, segmentState.Context); CodecUtil.CheckHeader(input, PulsingPostingsWriter.CODEC, version, PulsingPostingsWriter.VERSION_CURRENT); int numField = input.ReadVInt(); for (int i = 0; i < numField; i++) { int fieldNum = input.ReadVInt(); int longsSize = input.ReadVInt(); fields.Add(fieldNum, longsSize); } } finally { IOUtils.CloseWhileHandlingException(input); } } }
public SlowOpeningMockIndexInputWrapper(MockDirectoryWrapper dir, string name, IndexInput @delegate) : base(dir, name, @delegate) { try { Thread.Sleep(50); } catch (ThreadInterruptedException ie) { try { base.Dispose(); } // we didnt open successfully catch (Exception) { } throw new ThreadInterruptedException("Thread Interrupted Exception", ie); } }
public override byte[] GetBinaryValue(byte[] result, IState state) { Enclosing_Instance.EnsureOpen(); if (internalIsBinary) { if (fieldsData == null) { // Allocate new buffer if result is null or too small byte[] b; if (result == null || result.Length < toRead) { b = new byte[toRead]; } else { b = result; } IndexInput localFieldsStream = GetFieldStream(state); // Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people // since they are already handling this exception when getting the document try { localFieldsStream.Seek(pointer, state); localFieldsStream.ReadBytes(b, 0, toRead, state); fieldsData = isCompressed ? Enclosing_Instance.Uncompress(b) : b; } catch (IOException e) { throw new FieldReaderException(e); } internalbinaryOffset = 0; internalBinaryLength = toRead; } return((byte[])fieldsData); } return(null); }
/// <summary>Constructs a bit vector from the file <c>name</c> in Directory /// <c>d</c>, as written by the <see cref="Write" /> method. /// </summary> public BitVector(Directory d, System.String name, IState state) { IndexInput input = d.OpenInput(name, state); try { size = input.ReadInt(state); // read size if (size == -1) { ReadDgaps(input, state); } else { ReadBits(input, state); } } finally { input.Close(); } }
/// <summary> /// Sole constructor. </summary> public MonotonicBlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct) { this.ValueCount = valueCount; BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE); BlockMask = blockSize - 1; int numBlocks = PackedInts.NumBlocks(valueCount, blockSize); MinValues = new long[numBlocks]; Averages = new float[numBlocks]; SubReaders = new PackedInts.Reader[numBlocks]; for (int i = 0; i < numBlocks; ++i) { MinValues[i] = @in.ReadVLong(); Averages[i] = Number.IntBitsToFloat(@in.ReadInt()); int bitsPerValue = @in.ReadVInt(); if (bitsPerValue > 64) { throw new Exception("Corrupted"); } if (bitsPerValue == 0) { SubReaders[i] = new PackedInts.NullReader(blockSize); } else { int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize); if (direct) { long pointer = @in.FilePointer; SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); @in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue)); } else { SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); } } } }
// public static boolean DEBUG = false; /// <summary> /// Sole constructor. </summary> public Lucene41PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, string segmentSuffix) { bool success = false; IndexInput docIn = null; IndexInput posIn = null; IndexInput payIn = null; try { docIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), ioContext); Version = CodecUtil.CheckHeader(docIn, Lucene41PostingsWriter.DOC_CODEC, Lucene41PostingsWriter.VERSION_START, Lucene41PostingsWriter.VERSION_CURRENT); forUtil = new ForUtil(docIn); if (fieldInfos.HasProx()) { posIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), ioContext); CodecUtil.CheckHeader(posIn, Lucene41PostingsWriter.POS_CODEC, Version, Version); if (fieldInfos.HasPayloads() || fieldInfos.HasOffsets()) { payIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), ioContext); CodecUtil.CheckHeader(payIn, Lucene41PostingsWriter.PAY_CODEC, Version, Version); } } this.DocIn = docIn; this.PosIn = posIn; this.PayIn = payIn; success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(docIn, posIn, payIn); } } }
// It is not always neccessary to move the prox pointer // to a new document after the freq pointer has been moved. // Consider for example a phrase query with two terms: // the freq pointer for term 1 has to move to document x // to answer the question if the term occurs in that document. But // only if term 2 also matches document x, the positions have to be // read to figure out if term 1 and term 2 appear next // to each other in document x and thus satisfy the query. // So we move the prox pointer lazily to the document // as soon as positions are requested. private void LazySkip(IState state) { if (proxStream == null) { // clone lazily proxStream = (IndexInput)parent.core.proxStream.Clone(state); } // we might have to skip the current payload // if it was not read yet SkipPayload(state); if (lazySkipPointer != -1) { proxStream.Seek(lazySkipPointer, state); lazySkipPointer = -1; } if (lazySkipProxCount != 0) { SkipPositions(lazySkipProxCount, state); lazySkipProxCount = 0; } }
/// <summary> /// Skip the next block of data. /// </summary> /// <param name="in"> the input where to read data </param> /// <exception cref="IOException"> If there is a low-level I/O error </exception> public void SkipBlock(IndexInput @in) { int numBits = @in.ReadByte(); if (numBits == ALL_VALUES_EQUAL) { @in.ReadVInt(); return; } Debug.Assert(numBits > 0 && numBits <= 32, numBits.ToString()); int encodedSize = EncodedSizes[numBits]; @in.Seek(@in.FilePointer + encodedSize); }
/// <summary> /// Read the next block of data (<code>For</code> format). /// </summary> /// <param name="in"> the input to use to read data </param> /// <param name="encoded"> a buffer that can be used to store encoded data </param> /// <param name="decoded"> where to write decoded data </param> /// <exception cref="IOException"> If there is a low-level I/O error </exception> public void ReadBlock(IndexInput @in, sbyte[] encoded, int[] decoded) { int numBits = @in.ReadByte(); Debug.Assert(numBits <= 32, numBits.ToString()); if (numBits == ALL_VALUES_EQUAL) { int value = @in.ReadVInt(); CollectionsHelper.Fill(decoded, 0, Lucene41PostingsFormat.BLOCK_SIZE, value); return; } int encodedSize = EncodedSizes[numBits]; @in.ReadBytes(encoded, 0, encodedSize); PackedInts.Decoder decoder = Decoders[numBits]; int iters = Iterations[numBits]; Debug.Assert(iters * decoder.ByteValueCount() >= Lucene41PostingsFormat.BLOCK_SIZE); decoder.Decode(encoded, 0, decoded, 0, iters); }
/// <summary>Read as a bit set </summary> private void ReadBits(IndexInput input, IState state) { count = input.ReadInt(state); // read count bits = new byte[(size >> 3) + 1]; // allocate bits input.ReadBytes(bits, 0, bits.Length, state); }
private async Task InitialiseFile(ISecureStore store, IEncryptor encryptor, StoreLocation location) { bool fFileNeeded = false; if (!_cache.FileExists(_name)) { fFileNeeded = true; } else { long cachedLength = _cache.FileLength(_name); var metadata = await store.GetMetadata(location).ConfigureAwait(false); if (metadata == null) { throw new System.IO.FileNotFoundException(_name); } var blobLength = metadata.ContentLength ?? 0; var blobLastModifiedUTC = metadata.LastModified ?? DateTime.UtcNow; if (cachedLength != blobLength) { fFileNeeded = true; } else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length var elapsed = _cache.FileModified(_name); // normalize RAMDirectory and FSDirectory times if (elapsed > ticks1970) { elapsed -= ticks1970; } var cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime(); if (cachedLastModifiedUTC != blobLastModifiedUTC) { var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { using (StreamOutput fileStream = _directory.CreateCachedOutputAsStream(_name)) { var data = await store.LoadData(location, null, encryptor).ConfigureAwait(false); if (data == null) { throw new System.IO.FileNotFoundException(_name); } await data.Stream.CopyToStream(fileStream, CancellationToken.None).ConfigureAwait(false); } // and open it as an input _indexInput = _cache.OpenInput(_name); } else { // open the file in read only mode _indexInput = _cache.OpenInput(_name); } }
/// <summary>Returns a <see cref="Status" /> instance detailing /// the state of the index. /// /// </summary> /// <param name="onlySegments">list of specific segment names to check /// /// <p/>As this method checks every byte in the specified /// segments, on a large index it can take quite a long /// time to run. /// /// <p/><b>WARNING</b>: make sure /// you only call this when the index is not opened by any /// writer. /// </param> public virtual Status CheckIndex_Renamed_Method(List <string> onlySegments, IState state) { System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat; SegmentInfos sis = new SegmentInfos(); Status result = new Status(); result.dir = dir; try { sis.Read(dir, state); } catch (System.Exception t) { Msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; if (infoStream != null) { infoStream.WriteLine(t.StackTrace); } return(result); } int numSegments = sis.Count; var segmentsFileName = sis.GetCurrentSegmentFileName(); IndexInput input = null; try { input = dir.OpenInput(segmentsFileName, state); } catch (System.Exception t) { Msg("ERROR: could not open segments file in directory"); if (infoStream != null) { infoStream.WriteLine(t.StackTrace); } result.cantOpenSegments = true; return(result); } int format = 0; try { format = input.ReadInt(state); } catch (System.Exception t) { Msg("ERROR: could not read segment file version in directory"); if (infoStream != null) { infoStream.WriteLine(t.StackTrace); } result.missingSegmentVersion = true; return(result); } finally { if (input != null) { input.Close(); } } System.String sFormat = ""; bool skip = false; if (format == SegmentInfos.FORMAT) { sFormat = "FORMAT [Lucene Pre-2.1]"; } if (format == SegmentInfos.FORMAT_LOCKLESS) { sFormat = "FORMAT_LOCKLESS [Lucene 2.1]"; } else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE) { sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]"; } else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE) { sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]"; } else { if (format == SegmentInfos.FORMAT_CHECKSUM) { sFormat = "FORMAT_CHECKSUM [Lucene 2.4]"; } else if (format == SegmentInfos.FORMAT_DEL_COUNT) { sFormat = "FORMAT_DEL_COUNT [Lucene 2.4]"; } else if (format == SegmentInfos.FORMAT_HAS_PROX) { sFormat = "FORMAT_HAS_PROX [Lucene 2.4]"; } else if (format == SegmentInfos.FORMAT_USER_DATA) { sFormat = "FORMAT_USER_DATA [Lucene 2.9]"; } else if (format == SegmentInfos.FORMAT_DIAGNOSTICS) { sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]"; } else if (format < SegmentInfos.CURRENT_FORMAT) { sFormat = "int=" + format + " [newer version of Lucene than this tool]"; skip = true; } else { sFormat = format + " [Lucene 1.3 or prior]"; } } result.segmentsFileName = segmentsFileName; result.numSegments = numSegments; result.segmentFormat = sFormat; result.userData = sis.UserData; System.String userDataString; if (sis.UserData.Count > 0) { userDataString = " userData=" + CollectionsHelper.CollectionToString(sis.UserData); } else { userDataString = ""; } Msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString); if (onlySegments != null) { result.partial = true; if (infoStream != null) { infoStream.Write("\nChecking only these segments:"); } foreach (string s in onlySegments) { if (infoStream != null) { infoStream.Write(" " + s); } } result.segmentsChecked.AddRange(onlySegments); Msg(":"); } if (skip) { Msg("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting"); result.toolOutOfDate = true; return(result); } result.newSegments = (SegmentInfos)sis.Clone(); result.newSegments.Clear(); for (int i = 0; i < numSegments; i++) { SegmentInfo info = sis.Info(i); if (onlySegments != null && !onlySegments.Contains(info.name)) { continue; } var segInfoStat = new Status.SegmentInfoStatus(); result.segmentInfos.Add(segInfoStat); Msg(" " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount); segInfoStat.name = info.name; segInfoStat.docCount = info.docCount; int toLoseDocCount = info.docCount; SegmentReader reader = null; try { Msg(" compound=" + info.GetUseCompoundFile(state)); segInfoStat.compound = info.GetUseCompoundFile(state); Msg(" hasProx=" + info.HasProx); segInfoStat.hasProx = info.HasProx; Msg(" numFiles=" + info.Files(state).Count); segInfoStat.numFiles = info.Files(state).Count; Msg(System.String.Format(nf, " size (MB)={0:f}", new System.Object[] { (info.SizeInBytes(state) / (1024.0 * 1024.0)) })); segInfoStat.sizeMB = info.SizeInBytes(state) / (1024.0 * 1024.0); IDictionary <string, string> diagnostics = info.Diagnostics; segInfoStat.diagnostics = diagnostics; if (diagnostics.Count > 0) { Msg(" diagnostics = " + CollectionsHelper.CollectionToString(diagnostics)); } int docStoreOffset = info.DocStoreOffset; if (docStoreOffset != -1) { Msg(" docStoreOffset=" + docStoreOffset); segInfoStat.docStoreOffset = docStoreOffset; Msg(" docStoreSegment=" + info.DocStoreSegment); segInfoStat.docStoreSegment = info.DocStoreSegment; Msg(" docStoreIsCompoundFile=" + info.DocStoreIsCompoundFile); segInfoStat.docStoreCompoundFile = info.DocStoreIsCompoundFile; } System.String delFileName = info.GetDelFileName(); if (delFileName == null) { Msg(" no deletions"); segInfoStat.hasDeletions = false; } else { Msg(" has deletions [delFileName=" + delFileName + "]"); segInfoStat.hasDeletions = true; segInfoStat.deletionsFileName = delFileName; } if (infoStream != null) { infoStream.Write(" test: open reader........."); } reader = SegmentReader.Get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, state); segInfoStat.openReaderPassed = true; int numDocs = reader.NumDocs(); toLoseDocCount = numDocs; if (reader.HasDeletions) { if (reader.deletedDocs.Count() != info.GetDelCount(state)) { throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs deletedDocs.count()=" + reader.deletedDocs.Count()); } if (reader.deletedDocs.Count() > reader.MaxDoc) { throw new System.SystemException("too many deleted docs: MaxDoc=" + reader.MaxDoc + " vs deletedDocs.count()=" + reader.deletedDocs.Count()); } if (info.docCount - numDocs != info.GetDelCount(state)) { throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs reader=" + (info.docCount - numDocs)); } segInfoStat.numDeleted = info.docCount - numDocs; Msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]"); } else { if (info.GetDelCount(state) != 0) { throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs reader=" + (info.docCount - numDocs)); } Msg("OK"); } if (reader.MaxDoc != info.docCount) { throw new System.SystemException("SegmentReader.MaxDoc " + reader.MaxDoc + " != SegmentInfos.docCount " + info.docCount); } // Test getFieldNames() if (infoStream != null) { infoStream.Write(" test: fields.............."); } ICollection <string> fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL); Msg("OK [" + fieldNames.Count + " fields]"); segInfoStat.numFields = fieldNames.Count; // Test Field Norms segInfoStat.fieldNormStatus = TestFieldNorms(fieldNames, reader, state); // Test the Term Index segInfoStat.termIndexStatus = TestTermIndex(info, reader, state); // Test Stored Fields segInfoStat.storedFieldStatus = TestStoredFields(info, reader, nf, state); // Test Term Vectors segInfoStat.termVectorStatus = TestTermVectors(info, reader, nf, state); // Rethrow the first exception we encountered // This will cause stats for failed segments to be incremented properly if (segInfoStat.fieldNormStatus.error != null) { throw new SystemException("Field Norm test failed"); } else if (segInfoStat.termIndexStatus.error != null) { throw new SystemException("Term Index test failed"); } else if (segInfoStat.storedFieldStatus.error != null) { throw new SystemException("Stored Field test failed"); } else if (segInfoStat.termVectorStatus.error != null) { throw new System.SystemException("Term Vector test failed"); } Msg(""); } catch (System.Exception t) { Msg("FAILED"); const string comment = "fixIndex() would remove reference to this segment"; Msg(" WARNING: " + comment + "; full exception:"); if (infoStream != null) { infoStream.WriteLine(t.StackTrace); } Msg(""); result.totLoseDocCount += toLoseDocCount; result.numBadSegments++; continue; } finally { if (reader != null) { reader.Close(); } } // Keeper result.newSegments.Add((SegmentInfo)info.Clone()); } if (0 == result.numBadSegments) { result.clean = true; Msg("No problems were detected with this index.\n"); } else { Msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected"); } return(result); }
public virtual void TestManySubFiles() { Directory d = NewFSDirectory(CreateTempDir("CFSManySubFiles")); int FILE_COUNT = AtLeast(500); for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { IndexOutput @out = d.CreateOutput("file." + fileIdx, NewIOContext(Random())); @out.WriteByte((byte)(sbyte)fileIdx); @out.Dispose(); } CompoundFileDirectory cfd = new CompoundFileDirectory(d, "c.cfs", NewIOContext(Random()), true); for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { string fileName = "file." + fileIdx; d.Copy(cfd, fileName, fileName, NewIOContext(Random())); } cfd.Dispose(); IndexInput[] ins = new IndexInput[FILE_COUNT]; CompoundFileDirectory cfr = new CompoundFileDirectory(d, "c.cfs", NewIOContext(Random()), false); for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { ins[fileIdx] = cfr.OpenInput("file." + fileIdx, NewIOContext(Random())); } for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { Assert.AreEqual((byte)fileIdx, ins[fileIdx].ReadByte()); } for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { ins[fileIdx].Dispose(); } cfr.Dispose(); d.Dispose(); }
private void AssertSameStreams(string msg, IndexInput expected, IndexInput actual, long seekTo) { if (seekTo >= 0 && seekTo < expected.Length()) { expected.Seek(seekTo); actual.Seek(seekTo); AssertSameStreams(msg + ", seek(mid)", expected, actual); } }
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; string fileName = _name; bool fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); long blobLength = blob.Properties.Length; long.TryParse(blob.Metadata["CachedLength"], out blobLength); long longLastModified = 0; DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime; //if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified)) // blobLastModifiedUTC = new DateTime(longLastModified, DateTimeKind.Local).ToUniversalTime(); if (cachedLength != blobLength) fFileNeeded = true; else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length DateTime cachedLastModifiedUTC = DateTime.MinValue; if (CacheDirectory is RAMDirectory) { cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Local).ToUniversalTime(); } else if (CacheDirectory is FSDirectory) { cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Utc).ToUniversalTime().AddYears(1969); } if (cachedLastModifiedUTC != blobLastModifiedUTC) { TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) fFileNeeded = true; else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { #if COMPRESSBLOBS if (_azureDirectory.ShouldCompressFile(_name)) { // then we will get it fresh into local deflatedName // StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName)); MemoryStream deflatedStream = new MemoryStream(); // get the deflated blob _blob.DownloadToStream(deflatedStream); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length)); // seek back to begininng deflatedStream.Seek(0, SeekOrigin.Begin); // open output file for uncompressed contents StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName); // create decompressor DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress); byte[] bytes = new byte[65535]; int nRead = 0; do { nRead = decompressor.Read(bytes, 0, 65535); if (nRead > 0) fileStream.Write(bytes, 0, nRead); } while (nRead == 65535); decompressor.Close(); // this should close the deflatedFileStream too fileStream.Close(); } else #endif { StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName); // get the blob _blob.DownloadToStream(fileStream); fileStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); fileStream.Close(); } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName); } } finally { _fileMutex.ReleaseMutex(); } }
protected override void Dispose(bool disposing) { _fileMutex.WaitOne(); try { #if FULLDEBUG Debug.WriteLine(String.Format("CLOSED READSTREAM local {0}", _name)); #endif _indexInput.Dispose(); _indexInput = null; _azureDirectory = null; _blobContainer = null; _blob = null; GC.SuppressFinalize(this); } finally { _fileMutex.ReleaseMutex(); } }
/// <summary> Construct a new SegmentInfo instance by reading a /// previously saved SegmentInfo from input. /// /// </summary> /// <param name="dir">directory to load from /// </param> /// <param name="format">format of the segments info file /// </param> /// <param name="input">input handle to read segment info from /// </param> internal SegmentInfo(Directory dir, int format, IndexInput input, IState state) { this.dir = dir; name = input.ReadString(state); docCount = input.ReadInt(state); if (format <= SegmentInfos.FORMAT_LOCKLESS) { delGen = input.ReadLong(state); if (format <= SegmentInfos.FORMAT_SHARED_DOC_STORE) { docStoreOffset = input.ReadInt(state); if (docStoreOffset != -1) { docStoreSegment = input.ReadString(state); docStoreIsCompoundFile = (1 == input.ReadByte(state)); } else { docStoreSegment = name; docStoreIsCompoundFile = false; } } else { docStoreOffset = -1; docStoreSegment = name; docStoreIsCompoundFile = false; } if (format <= SegmentInfos.FORMAT_SINGLE_NORM_FILE) { hasSingleNormFile = (1 == input.ReadByte(state)); } else { hasSingleNormFile = false; } int numNormGen = input.ReadInt(state); if (numNormGen == NO) { normGen = null; } else { normGen = new long[numNormGen]; for (int j = 0; j < numNormGen; j++) { normGen[j] = input.ReadLong(state); } } isCompoundFile = (sbyte)input.ReadByte(state); preLockless = (isCompoundFile == CHECK_DIR); if (format <= SegmentInfos.FORMAT_DEL_COUNT) { delCount = input.ReadInt(state); System.Diagnostics.Debug.Assert(delCount <= docCount); } else { delCount = -1; } if (format <= SegmentInfos.FORMAT_HAS_PROX) { hasProx = input.ReadByte(state) == 1; } else { hasProx = true; } if (format <= SegmentInfos.FORMAT_DIAGNOSTICS) { diagnostics = input.ReadStringStringMap(state); } else { diagnostics = new Dictionary <string, string>(); } } else { delGen = CHECK_DIR; normGen = null; isCompoundFile = (sbyte)(CHECK_DIR); preLockless = true; hasSingleNormFile = false; docStoreOffset = -1; docStoreIsCompoundFile = false; docStoreSegment = null; delCount = -1; hasProx = true; diagnostics = new Dictionary <string, string>(); } }
internal FaultyIndexInput(IndexInput delegate_Renamed) { this.delegate_Renamed = delegate_Renamed; }
internal CSIndexInput(IndexInput @base, long fileOffset, long length, IState state) : this(@base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE, state) { }
internal CSIndexInput(IndexInput @base, long fileOffset, long length, int readBufferSize, IState state) : base(readBufferSize) { this.base_Renamed = (IndexInput)@base.Clone(state); this.fileOffset = fileOffset; this.length = length; }
internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size, IState state) { bool success = false; isOriginal = true; try { fieldInfos = fn; cloneableFieldsStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_EXTENSION, readBufferSize, state); cloneableIndexStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION, readBufferSize, state); // First version of fdx did not include a format // header, but, the first int will always be 0 in that // case int firstInt = cloneableIndexStream.ReadInt(state); format = firstInt == 0 ? 0 : firstInt; if (format > FieldsWriter.FORMAT_CURRENT) { throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FieldsWriter.FORMAT_CURRENT + " or lower"); } formatSize = format > FieldsWriter.FORMAT ? 4 : 0; if (format < FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { cloneableFieldsStream.SetModifiedUTF8StringsMode(); } fieldsStream = (IndexInput)cloneableFieldsStream.Clone(state); long indexSize = cloneableIndexStream.Length(state) - formatSize; if (docStoreOffset != -1) { // We read only a slice out of this shared fields file this.docStoreOffset = docStoreOffset; this.size = size; // Verify the file is long enough to hold all of our // docs System.Diagnostics.Debug.Assert(((int)(indexSize / 8)) >= size + this.docStoreOffset, "indexSize=" + indexSize + " size=" + size + " docStoreOffset=" + docStoreOffset); } else { this.docStoreOffset = 0; this.size = (int)(indexSize >> 3); } indexStream = (IndexInput)cloneableIndexStream.Clone(state); numTotalDocs = (int)(indexSize >> 3); success = true; } finally { // With lock-less commits, it's entirely possible (and // fine) to hit a FileNotFound exception above. In // this case, we want to explicitly close any subset // of things that were opened so that we don't have to // wait for a GC to do so. if (!success) { Dispose(); } } }
public StreamInput(IndexInput input) { Input = input; }
private SecureStoreIndexInput(SecureStoreIndexInput cloneInput) { _fileMutex = BlobMutexManager.GrabMutex(cloneInput._name); _fileMutex.WaitOne(); try { _directory = cloneInput._directory; _cache = cloneInput._cache; _name = cloneInput._name; _indexInput = cloneInput._indexInput.Clone() as IndexInput; } catch (Exception) { // sometimes we get access denied on the 2nd stream...but not always. I haven't tracked it down yet // but this covers our tail until I do LeoTrace.WriteLine(String.Format("Falling back to memory clone for {0}", cloneInput._name)); } finally { _fileMutex.ReleaseMutex(); } }
private void AssertSameSeekBehavior(string msg, IndexInput expected, IndexInput actual) { // seek to 0 long point = 0; AssertSameStreams(msg + ", seek(0)", expected, actual, point); // seek to middle point = expected.Length() / 2l; AssertSameStreams(msg + ", seek(mid)", expected, actual, point); // seek to end - 2 point = expected.Length() - 2; AssertSameStreams(msg + ", seek(end-2)", expected, actual, point); // seek to end - 1 point = expected.Length() - 1; AssertSameStreams(msg + ", seek(end-1)", expected, actual, point); // seek to the end point = expected.Length(); AssertSameStreams(msg + ", seek(end)", expected, actual, point); // seek past end point = expected.Length() + 1; AssertSameStreams(msg + ", seek(end+1)", expected, actual, point); }
protected override void Dispose(bool disposing) { _fileMutex.WaitOne(); try { _indexInput.Dispose(); _indexInput = null; _directory = null; _cache = null; GC.SuppressFinalize(this); } finally { _fileMutex.ReleaseMutex(); } }
private void AssertSameStreams(string msg, IndexInput expected, IndexInput test) { Assert.IsNotNull(expected, msg + " null expected"); Assert.IsNotNull(test, msg + " null test"); Assert.AreEqual(expected.Length(), test.Length(), msg + " length"); Assert.AreEqual(expected.FilePointer, test.FilePointer, msg + " position"); var expectedBuffer = new byte[512]; var testBuffer = new byte[expectedBuffer.Length]; long remainder = expected.Length() - expected.FilePointer; while (remainder > 0) { int readLen = (int)Math.Min(remainder, expectedBuffer.Length); expected.ReadBytes(expectedBuffer, 0, readLen); test.ReadBytes(testBuffer, 0, readLen); AssertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen); remainder -= readLen; } }
internal SegmentTermEnum(IndexInput i, FieldInfos fis, bool isi, IState state) { input = i; fieldInfos = fis; isIndex = isi; maxSkipLevels = 1; // use single-level skip lists for formats > -3 int firstInt = input.ReadInt(state); if (firstInt >= 0) { // original-format file, without explicit format version number format = 0; size = firstInt; // back-compatible settings indexInterval = 128; skipInterval = System.Int32.MaxValue; // switch off skipTo optimization } else { // we have a format version number format = firstInt; // check that it is a format we can understand if (format < TermInfosWriter.FORMAT_CURRENT) { throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher"); } size = input.ReadLong(state); // read the size if (format == -1) { if (!isIndex) { indexInterval = input.ReadInt(state); formatM1SkipInterval = input.ReadInt(state); } // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in // skipTo implementation of these versions skipInterval = System.Int32.MaxValue; } else { indexInterval = input.ReadInt(state); skipInterval = input.ReadInt(state); if (format <= TermInfosWriter.FORMAT) { // this new format introduces multi-level skipping maxSkipLevels = input.ReadInt(state); } } System.Diagnostics.Debug.Assert(indexInterval > 0, "indexInterval=" + indexInterval + " is negative; must be > 0"); System.Diagnostics.Debug.Assert(skipInterval > 0, "skipInterval=" + skipInterval + " is negative; must be > 0"); } if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { termBuffer.SetPreUTF8Strings(); scanBuffer.SetPreUTF8Strings(); prevBuffer.SetPreUTF8Strings(); } }