public void Read(IndexInput input, FieldInfos fieldInfos, IState state)
{
this.term = null; // invalidate cache
int start = input.ReadVInt(state);
int length = input.ReadVInt(state);
int totalLength = start + length;
if (preUTF8Strings)
{
text.SetLength(totalLength);
input.ReadChars(text.result, start, length, state);
}
else
{
if (dirty)
{
// Fully convert all bytes since bytes is dirty
UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes);
bytes.SetLength(totalLength);
input.ReadBytes(bytes.result, start, length, state);
UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text);
dirty = false;
}
else
{
// Incrementally convert only the UTF8 bytes that are new:
bytes.SetLength(totalLength);
input.ReadBytes(bytes.result, start, length, state);
UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text);
}
}
this.field = fieldInfos.FieldName(input.ReadVInt(state));
}
protected internal override int ReadSkipData(int level, IndexInput skipStream, IState state)
{
int delta;
if (currentFieldStoresPayloads)
{
// the current field stores payloads.
// if the doc delta is odd then we have
// to read the current payload length
// because it differs from the length of the
// previous payload
delta = skipStream.ReadVInt(state);
if ((delta & 1) != 0)
{
payloadLength[level] = skipStream.ReadVInt(state);
}
delta = Number.URShift(delta, 1);
}
else
{
delta = skipStream.ReadVInt(state);
}
freqPointer[level] += skipStream.ReadVInt(state);
proxPointer[level] += skipStream.ReadVInt(state);
return(delta);
}
protected override MonotonicBlockPackedReader GetOrdIndexInstance(IndexInput data, FieldInfo field,
NumericEntry entry)
{
data.Seek(entry.Offset);
return new MonotonicBlockPackedReader((IndexInput)data.Clone(), entry.PackedIntsVersion, entry.BlockSize, entry.Count,
true);
}
public MockSingleIntIndexInput(Directory dir, string fileName, IOContext context)
{
@in = dir.OpenInput(fileName, context);
CodecUtil.CheckHeader(@in, MockSingleIntIndexOutput.CODEC,
MockSingleIntIndexOutput.VERSION_START,
MockSingleIntIndexOutput.VERSION_START);
}
protected override void Dispose(bool disposing)
{
lock (this)
{
if (isDisposed)
{
return;
}
if (disposing)
{
if (entries != null)
{
entries.Clear();
}
if (stream != null)
{
stream.Close();
}
}
entries = null;
stream = null;
isDisposed = true;
}
}
/// <summary>
/// Checks that the stream is positioned at the end, and throws exception
/// if it is not. </summary>
/// @deprecated Use <seealso cref="#checkFooter"/> instead, this should only used for files without checksums
public static void CheckEOF(IndexInput @in)
{
if (@in.FilePointer != @in.Length())
{
throw new System.IO.IOException("did not read all bytes from file: read " + @in.FilePointer + " vs size " + @in.Length() + " (resource: " + @in + ")");
}
}
private int CopyFieldsNoDeletions(FieldsWriter fieldsWriter, IndexReader reader, FieldsReader matchingFieldsReader, IState state)
{
int maxDoc = reader.MaxDoc;
int docCount = 0;
if (matchingFieldsReader != null)
{
// We can bulk-copy because the fieldInfos are "congruent"
while (docCount < maxDoc)
{
int len = System.Math.Min(MAX_RAW_MERGE_DOCS, maxDoc - docCount);
IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, docCount, len, state);
fieldsWriter.AddRawDocuments(stream, rawDocLengths, len, state);
docCount += len;
checkAbort.Work(300 * len, state);
}
}
else
{
for (; docCount < maxDoc; docCount++)
{
// NOTE: it's very important to first assign to doc then pass it to
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
Document doc = reader.Document(docCount, state);
fieldsWriter.AddDocument(doc, state);
checkAbort.Work(300, state);
}
}
return(docCount);
}
/// <summary>
/// Construct an empty output buffer. </summary>
public MockIndexInputWrapper(MockDirectoryWrapper dir, string name, IndexInput @delegate)
: base("MockIndexInputWrapper(name=" + name + " delegate=" + @delegate + ")")
{
this.Name = name;
this.Dir = dir;
this.@delegate = @delegate;
}
protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field,
BinaryEntry bytes)
{
data.Seek(bytes.AddressesOffset);
return new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count,
true);
}
private int CopyFieldsWithDeletions(FieldsWriter fieldsWriter, IndexReader reader, FieldsReader matchingFieldsReader, IState state)
{
int docCount = 0;
int maxDoc = reader.MaxDoc;
if (matchingFieldsReader != null)
{
// We can bulk-copy because the fieldInfos are "congruent"
for (int j = 0; j < maxDoc;)
{
if (reader.IsDeleted(j))
{
// skip deleted docs
++j;
continue;
}
// We can optimize this case (doing a bulk byte copy) since the field
// numbers are identical
int start = j, numDocs = 0;
do
{
j++;
numDocs++;
if (j >= maxDoc)
{
break;
}
if (reader.IsDeleted(j))
{
j++;
break;
}
}while (numDocs < MAX_RAW_MERGE_DOCS);
IndexInput stream = matchingFieldsReader.RawDocs(rawDocLengths, start, numDocs, state);
fieldsWriter.AddRawDocuments(stream, rawDocLengths, numDocs, state);
docCount += numDocs;
checkAbort.Work(300 * numDocs, state);
}
}
else
{
for (int j = 0; j < maxDoc; j++)
{
if (reader.IsDeleted(j))
{
// skip deleted docs
continue;
}
// NOTE: it's very important to first assign to doc then pass it to
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
Document doc = reader.Document(j, state);
fieldsWriter.AddDocument(doc, state);
docCount++;
checkAbort.Work(300, state);
}
}
return(docCount);
}
private int CheckValidFormat(IndexInput in_Renamed, IState state)
{
int format = in_Renamed.ReadInt(state);
if (format > FORMAT_CURRENT)
{
throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FORMAT_CURRENT + " or less");
}
return(format);
}
/// <summary>Returns true if the provided input stream is an SimpleFSIndexInput and
/// is a clone, that is it does not own its underlying file descriptor.
/// </summary>
public static bool IsSimpleFSIndexInputClone(IndexInput is_Renamed)
{
if (IsSimpleFSIndexInput(is_Renamed))
{
return ((SimpleFSIndexInput) is_Renamed).isClone_ForNUnit;
}
else
{
return false;
}
}
public CompoundFileReader(Directory dir, System.String name, int readBufferSize, IState state)
{
directory = dir;
fileName = name;
this.readBufferSize = readBufferSize;
bool success = false;
try
{
stream = dir.OpenInput(name, readBufferSize, state);
// read the directory and init files
int count = stream.ReadVInt(state);
FileEntry entry = null;
for (int i = 0; i < count; i++)
{
long offset = stream.ReadLong(state);
System.String id = stream.ReadString(state);
if (entry != null)
{
// set length of the previous entry
entry.length = offset - entry.offset;
}
entry = new FileEntry {
offset = offset
};
entries[id] = entry;
}
// set the length of the final entry
if (entry != null)
{
entry.length = stream.Length(state) - entry.offset;
}
success = true;
}
finally
{
if (!success && (stream != null))
{
try
{
stream.Close();
}
catch (System.IO.IOException)
{
}
}
}
}
/// <summary>
/// Returns true if the provided input stream is an SimpleFSIndexInput and
/// is a clone, that is it does not own its underlying file descriptor.
/// </summary>
public static bool IsSimpleFSIndexInputClone(IndexInput @is)
{
if (IsSimpleFSIndexInput(@is))
{
return ((SimpleFSIndexInput)@is).IsClone;
}
else
{
return false;
}
}
public override IndexInput OpenInput(System.String name, IState state)
{
IndexInput ii = base.OpenInput(name, null);
if (name.EndsWith(".prx"))
{
// we decorate the proxStream with a wrapper class that allows to count the number of calls of seek()
ii = new SeeksCountingStream(enclosingInstance, ii);
}
return(ii);
}
private IndexInput GetFieldStream(IState state)
{
IndexInput localFieldsStream = Enclosing_Instance.fieldsStreamTL.Get(state);
if (localFieldsStream == null)
{
localFieldsStream = (IndexInput)Enclosing_Instance.cloneableFieldsStream.Clone(state);
Enclosing_Instance.fieldsStreamTL.Set(localFieldsStream);
}
return(localFieldsStream);
}
public /*protected internal*/ SegmentTermDocs(SegmentReader parent, IState state)
{
this.parent = parent;
this.freqStream = (IndexInput)parent.core.freqStream.Clone(state);
lock (parent)
{
this.deletedDocs = parent.deletedDocs;
}
this.skipInterval = parent.core.GetTermsReader().SkipInterval;
this.maxSkipLevels = parent.core.GetTermsReader().MaxSkipLevels;
}
/// <summary>
/// Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns
/// true if the underlying file descriptor is valid, and false otherwise.
/// this can be used to determine if the OS file has been closed.
/// The descriptor becomes invalid when the non-clone instance of the
/// SimpleFSIndexInput that owns this descriptor is closed. However, the
/// descriptor may possibly become invalid in other ways as well.
/// </summary>
public static bool IsSimpleFSIndexInputOpen(IndexInput @is)
{
if (IsSimpleFSIndexInput(@is))
{
SimpleFSIndexInput fis = (SimpleFSIndexInput)@is;
return fis.FDValid;
}
else
{
return false;
}
}
/// <summary>Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns
/// true if the underlying file descriptor is valid, and false otherwise.
/// This can be used to determine if the OS file has been closed.
/// The descriptor becomes invalid when the non-clone instance of the
/// SimpleFSIndexInput that owns this descriptor is closed. However, the
/// descriptor may possibly become invalid in other ways as well.
/// </summary>
public static bool IsSimpleFSIndexInputOpen(IndexInput is_Renamed)
{
if (IsSimpleFSIndexInput(is_Renamed))
{
SimpleFSIndexInput fis = (SimpleFSIndexInput) is_Renamed;
return fis.IsFDValid();
}
else
{
return false;
}
}
/// <summary>Bulk write a contiguous series of documents. The
/// lengths array is the length (in bytes) of each raw
/// document. The stream IndexInput is the
/// fieldsStream from which we should bulk-copy all
/// bytes.
/// </summary>
internal void AddRawDocuments(IndexInput stream, int[] lengths, int numDocs, IState state)
{
long position = fieldsStream.FilePointer;
long start = position;
for (int i = 0; i < numDocs; i++)
{
indexStream.WriteLong(position);
position += lengths[i];
}
fieldsStream.CopyBytes(stream, position - start, state);
System.Diagnostics.Debug.Assert(fieldsStream.FilePointer == position);
}
// Used only by clone
private FieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int formatSize, int docStoreOffset, IndexInput cloneableFieldsStream, IndexInput cloneableIndexStream, IState state)
{
this.fieldInfos = fieldInfos;
this.numTotalDocs = numTotalDocs;
this.size = size;
this.format = format;
this.formatSize = formatSize;
this.docStoreOffset = docStoreOffset;
this.cloneableFieldsStream = cloneableFieldsStream;
this.cloneableIndexStream = cloneableIndexStream;
fieldsStream = (IndexInput)cloneableFieldsStream.Clone(state);
indexStream = (IndexInput)cloneableIndexStream.Clone(state);
}
/// <summary> Construct a FieldInfos object using the directory and the name of the file
/// IndexInput
/// </summary>
/// <param name="d">The directory to open the IndexInput from
/// </param>
/// <param name="name">The name of the file to open the IndexInput from in the Directory
/// </param>
/// <throws> IOException </throws>
public /*internal*/ FieldInfos(Directory d, String name, IState state)
{
IndexInput input = d.OpenInput(name, state);
try
{
try
{
Read(input, name, state);
}
catch (System.IO.IOException)
{
if (format == FORMAT_PRE)
{
// LUCENE-1623: FORMAT_PRE (before there was a
// format) may be 2.3.2 (pre-utf8) or 2.4.x (utf8)
// encoding; retry with input set to pre-utf8
input.Seek(0, state);
input.SetModifiedUTF8StringsMode();
byNumber.Clear();
byName.Clear();
bool rethrow = false;
try
{
Read(input, name, state);
}
catch (Exception)
{
// Ignore any new exception & set to throw original IOE
rethrow = true;
}
if (rethrow)
{
// Preserve stack trace
throw;
}
}
else
{
// The IOException cannot be caused by
// LUCENE-1623, so re-throw it
throw;
}
}
}
finally
{
input.Close();
}
}
// used by clone
private CompressingTermVectorsReader(CompressingTermVectorsReader reader)
{
this.fieldInfos = reader.fieldInfos;
this.vectorsStream = (IndexInput)reader.vectorsStream.Clone();
this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone();
this.packedIntsVersion = reader.packedIntsVersion;
this.compressionMode = reader.compressionMode;
this.decompressor = (Decompressor)reader.decompressor.Clone();
this.chunkSize = reader.chunkSize;
this.numDocs = reader.numDocs;
this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
this.version = reader.version;
this.closed = false;
}
private void Read(IndexInput input, String fileName, IState state)
{
int firstInt = input.ReadVInt(state);
if (firstInt < 0)
{
// This is a real format
format = firstInt;
}
else
{
format = FORMAT_PRE;
}
if (format != FORMAT_PRE & format != FORMAT_START)
{
throw new CorruptIndexException("unrecognized format " + format + " in file \"" + fileName + "\"");
}
int size;
if (format == FORMAT_PRE)
{
size = firstInt;
}
else
{
size = input.ReadVInt(state); //read in the size
}
for (int i = 0; i < size; i++)
{
String name = StringHelper.Intern(input.ReadString(state));
byte bits = input.ReadByte(state);
bool isIndexed = (bits & IS_INDEXED) != 0;
bool storeTermVector = (bits & STORE_TERMVECTOR) != 0;
bool storePositionsWithTermVector = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
bool storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
bool omitNorms = (bits & OMIT_NORMS) != 0;
bool storePayloads = (bits & STORE_PAYLOADS) != 0;
bool omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;
AddInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
}
if (input.FilePointer(state) != input.Length(state))
{
throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.FilePointer(state) + " vs size " + input.Length(state));
}
}
/// <summary>Copy the contents of the file with specified extension into the
/// provided output stream. Use the provided buffer for moving data
/// to reduce memory allocation.
/// </summary>
private void CopyFile(FileEntry source, IndexOutput os, byte[] buffer, IState state)
{
IndexInput isRenamed = null;
try
{
long startPtr = os.FilePointer;
isRenamed = directory.OpenInput(source.file, state);
long length = isRenamed.Length(state);
long remainder = length;
int chunk = buffer.Length;
while (remainder > 0)
{
var len = (int)Math.Min(chunk, remainder);
isRenamed.ReadBytes(buffer, 0, len, false, state);
os.WriteBytes(buffer, len);
remainder -= len;
if (checkAbort != null)
{
// Roughly every 2 MB we will check if
// it's time to abort
checkAbort.Work(80, state);
}
}
// Verify that remainder is 0
if (remainder != 0)
{
throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")");
}
// Verify that the output length diff is equal to original file
long endPtr = os.FilePointer;
long diff = endPtr - startPtr;
if (diff != length)
{
throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length);
}
}
finally
{
if (isRenamed != null)
{
isRenamed.Close();
}
}
}
/// <summary>read as a d-gaps list </summary>
private void ReadDgaps(IndexInput input, IState state)
{
size = input.ReadInt(state); // (re)read size
count = input.ReadInt(state); // read count
bits = new byte[(size >> 3) + 1]; // allocate bits
int last = 0;
int n = Count();
while (n > 0)
{
last += input.ReadVInt(state);
bits[last] = input.ReadByte(state);
n -= BYTE_COUNTS[bits[last] & 0xFF];
}
}
protected override void Dispose(bool disposing)
{
if (isDisposed) return;
if (disposing)
{
if (main != null)
{
main.Dispose();
}
}
main = null;
isDisposed = true;
}
/// <summary>Copy numBytes bytes from input to ourself. </summary>
public virtual void CopyBytes(IndexInput input, long numBytes)
{
long left = numBytes;
if (copyBuffer == null)
copyBuffer = new byte[COPY_BUFFER_SIZE];
while (left > 0)
{
int toCopy;
if (left > COPY_BUFFER_SIZE)
toCopy = COPY_BUFFER_SIZE;
else
toCopy = (int) left;
input.ReadBytes(copyBuffer, 0, toCopy);
WriteBytes(copyBuffer, 0, toCopy);
left -= toCopy;
}
}
protected override void Dispose(bool disposing)
{
if (isDisposed)
{
return;
}
if (disposing)
{
if (delegate_Renamed != null)
{
delegate_Renamed.Close();
}
}
delegate_Renamed = null;
isDisposed = true;
}
/// <summary>
/// Sole constructor. </summary>
public BlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct)
{
this.ValueCount = valueCount;
BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
BlockMask = blockSize - 1;
int numBlocks = PackedInts.NumBlocks(valueCount, blockSize);
long[] minValues = null;
SubReaders = new PackedInts.Reader[numBlocks];
for (int i = 0; i < numBlocks; ++i)
{
int token = @in.ReadByte() & 0xFF;
int bitsPerValue = (int)((uint)token >> AbstractBlockPackedWriter.BPV_SHIFT);
if (bitsPerValue > 64)
{
throw new Exception("Corrupted");
}
if ((token & AbstractBlockPackedWriter.MIN_VALUE_EQUALS_0) == 0)
{
if (minValues == null)
{
minValues = new long[numBlocks];
}
minValues[i] = BlockPackedReaderIterator.ZigZagDecode(1L + BlockPackedReaderIterator.ReadVLong(@in));
}
if (bitsPerValue == 0)
{
SubReaders[i] = new PackedInts.NullReader(blockSize);
}
else
{
int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize);
if (direct)
{
long pointer = @in.FilePointer;
SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
@in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue));
}
else
{
SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
}
}
}
this.MinValues = minValues;
}
/// <summary>The value of the field as a String, or null. If null, the Reader value,
/// binary value, or TokenStream value is used. Exactly one of StringValue(),
/// ReaderValue(), GetBinaryValue(), and TokenStreamValue() must be set.
/// </summary>
public override string StringValue(IState state)
{
Enclosing_Instance.EnsureOpen();
if (internalIsBinary)
{
return(null);
}
if (fieldsData == null)
{
IndexInput localFieldsStream = GetFieldStream(state);
try
{
localFieldsStream.Seek(pointer, state);
if (isCompressed)
{
var b = new byte[toRead];
localFieldsStream.ReadBytes(b, 0, b.Length, state);
fieldsData =
System.Text.Encoding.GetEncoding("UTF-8").GetString(Enclosing_Instance.Uncompress(b));
}
else
{
if (Enclosing_Instance.format >= FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
{
var bytes = new byte[toRead];
localFieldsStream.ReadBytes(bytes, 0, toRead, state);
fieldsData = System.Text.Encoding.GetEncoding("UTF-8").GetString(bytes);
}
else
{
//read in chars b/c we already know the length we need to read
var chars = new char[toRead];
localFieldsStream.ReadChars(chars, 0, toRead, state);
fieldsData = new System.String(chars);
}
}
}
catch (System.IO.IOException e)
{
throw new FieldReaderException(e);
}
}
return((System.String)fieldsData);
}
protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval)
{
this.skipStream = new IndexInput[maxSkipLevels];
this.skipPointer = new long[maxSkipLevels];
this.childPointer = new long[maxSkipLevels];
this.numSkipped = new int[maxSkipLevels];
this.maxNumberOfSkipLevels = maxSkipLevels;
this.skipInterval = new int[maxSkipLevels];
this.skipStream[0] = skipStream;
this.inputIsBuffered = (skipStream is BufferedIndexInput);
this.skipInterval[0] = skipInterval;
for (int i = 1; i < maxSkipLevels; i++)
{
// cache skip intervals
this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval;
}
skipDoc = new int[maxSkipLevels];
}
/// <summary>Copy numBytes bytes from input to ourself. </summary>
public virtual void CopyBytes(IndexInput input, long numBytes)
{
System.Diagnostics.Debug.Assert(numBytes >= 0, "numBytes=" + numBytes);
long left = numBytes;
if (copyBuffer == null)
copyBuffer = new byte[COPY_BUFFER_SIZE];
while (left > 0)
{
int toCopy;
if (left > COPY_BUFFER_SIZE)
toCopy = COPY_BUFFER_SIZE;
else
toCopy = (int) left;
input.ReadBytes(copyBuffer, 0, toCopy);
WriteBytes(copyBuffer, 0, toCopy);
left -= toCopy;
}
}
public override void Init(IndexInput termsIn)
{
version = CodecUtil.CheckHeader(termsIn, PulsingPostingsWriter.CODEC,
PulsingPostingsWriter.VERSION_START,
PulsingPostingsWriter.VERSION_CURRENT);
maxPositions = termsIn.ReadVInt();
_wrappedPostingsReader.Init(termsIn);
if (_wrappedPostingsReader is PulsingPostingsReader || version < PulsingPostingsWriter.VERSION_META_ARRAY)
{
fields = null;
}
else
{
fields = new SortedDictionary<int, int>();
String summaryFileName = IndexFileNames.SegmentFileName(segmentState.SegmentInfo.Name,
segmentState.SegmentSuffix, PulsingPostingsWriter.SUMMARY_EXTENSION);
IndexInput input = null;
try
{
input =
segmentState.Directory.OpenInput(summaryFileName, segmentState.Context);
CodecUtil.CheckHeader(input,
PulsingPostingsWriter.CODEC,
version,
PulsingPostingsWriter.VERSION_CURRENT);
int numField = input.ReadVInt();
for (int i = 0; i < numField; i++)
{
int fieldNum = input.ReadVInt();
int longsSize = input.ReadVInt();
fields.Add(fieldNum, longsSize);
}
}
finally
{
IOUtils.CloseWhileHandlingException(input);
}
}
}
public SlowOpeningMockIndexInputWrapper(MockDirectoryWrapper dir, string name, IndexInput @delegate)
: base(dir, name, @delegate)
{
try
{
Thread.Sleep(50);
}
catch (ThreadInterruptedException ie)
{
try
{
base.Dispose();
} // we didnt open successfully
catch (Exception)
{
}
throw new ThreadInterruptedException("Thread Interrupted Exception", ie);
}
}
public override byte[] GetBinaryValue(byte[] result, IState state)
{
Enclosing_Instance.EnsureOpen();
if (internalIsBinary)
{
if (fieldsData == null)
{
// Allocate new buffer if result is null or too small
byte[] b;
if (result == null || result.Length < toRead)
{
b = new byte[toRead];
}
else
{
b = result;
}
IndexInput localFieldsStream = GetFieldStream(state);
// Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people
// since they are already handling this exception when getting the document
try
{
localFieldsStream.Seek(pointer, state);
localFieldsStream.ReadBytes(b, 0, toRead, state);
fieldsData = isCompressed ? Enclosing_Instance.Uncompress(b) : b;
}
catch (IOException e)
{
throw new FieldReaderException(e);
}
internalbinaryOffset = 0;
internalBinaryLength = toRead;
}
return((byte[])fieldsData);
}
return(null);
}
/// <summary>Constructs a bit vector from the file <c>name</c> in Directory
/// <c>d</c>, as written by the <see cref="Write" /> method.
/// </summary>
public BitVector(Directory d, System.String name, IState state)
{
IndexInput input = d.OpenInput(name, state);
try
{
size = input.ReadInt(state); // read size
if (size == -1)
{
ReadDgaps(input, state);
}
else
{
ReadBits(input, state);
}
}
finally
{
input.Close();
}
}
/// <summary>
/// Sole constructor. </summary>
public MonotonicBlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct)
{
this.ValueCount = valueCount;
BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
BlockMask = blockSize - 1;
int numBlocks = PackedInts.NumBlocks(valueCount, blockSize);
MinValues = new long[numBlocks];
Averages = new float[numBlocks];
SubReaders = new PackedInts.Reader[numBlocks];
for (int i = 0; i < numBlocks; ++i)
{
MinValues[i] = @in.ReadVLong();
Averages[i] = Number.IntBitsToFloat(@in.ReadInt());
int bitsPerValue = @in.ReadVInt();
if (bitsPerValue > 64)
{
throw new Exception("Corrupted");
}
if (bitsPerValue == 0)
{
SubReaders[i] = new PackedInts.NullReader(blockSize);
}
else
{
int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize);
if (direct)
{
long pointer = @in.FilePointer;
SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
@in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue));
}
else
{
SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
}
}
}
}
// public static boolean DEBUG = false;
/// <summary>
/// Sole constructor. </summary>
public Lucene41PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, string segmentSuffix)
{
bool success = false;
IndexInput docIn = null;
IndexInput posIn = null;
IndexInput payIn = null;
try
{
docIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), ioContext);
Version = CodecUtil.CheckHeader(docIn, Lucene41PostingsWriter.DOC_CODEC, Lucene41PostingsWriter.VERSION_START, Lucene41PostingsWriter.VERSION_CURRENT);
forUtil = new ForUtil(docIn);
if (fieldInfos.HasProx())
{
posIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), ioContext);
CodecUtil.CheckHeader(posIn, Lucene41PostingsWriter.POS_CODEC, Version, Version);
if (fieldInfos.HasPayloads() || fieldInfos.HasOffsets())
{
payIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), ioContext);
CodecUtil.CheckHeader(payIn, Lucene41PostingsWriter.PAY_CODEC, Version, Version);
}
}
this.DocIn = docIn;
this.PosIn = posIn;
this.PayIn = payIn;
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(docIn, posIn, payIn);
}
}
}
// It is not always neccessary to move the prox pointer
// to a new document after the freq pointer has been moved.
// Consider for example a phrase query with two terms:
// the freq pointer for term 1 has to move to document x
// to answer the question if the term occurs in that document. But
// only if term 2 also matches document x, the positions have to be
// read to figure out if term 1 and term 2 appear next
// to each other in document x and thus satisfy the query.
// So we move the prox pointer lazily to the document
// as soon as positions are requested.
private void LazySkip(IState state)
{
if (proxStream == null)
{
// clone lazily
proxStream = (IndexInput)parent.core.proxStream.Clone(state);
}
// we might have to skip the current payload
// if it was not read yet
SkipPayload(state);
if (lazySkipPointer != -1)
{
proxStream.Seek(lazySkipPointer, state);
lazySkipPointer = -1;
}
if (lazySkipProxCount != 0)
{
SkipPositions(lazySkipProxCount, state);
lazySkipProxCount = 0;
}
}
/// <summary>
/// Skip the next block of data.
/// </summary>
/// <param name="in"> the input where to read data </param>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
public void SkipBlock(IndexInput @in)
{
int numBits = @in.ReadByte();
if (numBits == ALL_VALUES_EQUAL)
{
@in.ReadVInt();
return;
}
Debug.Assert(numBits > 0 && numBits <= 32, numBits.ToString());
int encodedSize = EncodedSizes[numBits];
@in.Seek(@in.FilePointer + encodedSize);
}
/// <summary>
/// Read the next block of data (<code>For</code> format).
/// </summary>
/// <param name="in"> the input to use to read data </param>
/// <param name="encoded"> a buffer that can be used to store encoded data </param>
/// <param name="decoded"> where to write decoded data </param>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
public void ReadBlock(IndexInput @in, sbyte[] encoded, int[] decoded)
{
int numBits = @in.ReadByte();
Debug.Assert(numBits <= 32, numBits.ToString());
if (numBits == ALL_VALUES_EQUAL)
{
int value = @in.ReadVInt();
CollectionsHelper.Fill(decoded, 0, Lucene41PostingsFormat.BLOCK_SIZE, value);
return;
}
int encodedSize = EncodedSizes[numBits];
@in.ReadBytes(encoded, 0, encodedSize);
PackedInts.Decoder decoder = Decoders[numBits];
int iters = Iterations[numBits];
Debug.Assert(iters * decoder.ByteValueCount() >= Lucene41PostingsFormat.BLOCK_SIZE);
decoder.Decode(encoded, 0, decoded, 0, iters);
}
/// <summary>Read as a bit set </summary>
private void ReadBits(IndexInput input, IState state)
{
count = input.ReadInt(state); // read count
bits = new byte[(size >> 3) + 1]; // allocate bits
input.ReadBytes(bits, 0, bits.Length, state);
}
private async Task InitialiseFile(ISecureStore store, IEncryptor encryptor, StoreLocation location)
{
bool fFileNeeded = false;
if (!_cache.FileExists(_name))
{
fFileNeeded = true;
}
else
{
long cachedLength = _cache.FileLength(_name);
var metadata = await store.GetMetadata(location).ConfigureAwait(false);
if (metadata == null)
{
throw new System.IO.FileNotFoundException(_name);
}
var blobLength = metadata.ContentLength ?? 0;
var blobLastModifiedUTC = metadata.LastModified ?? DateTime.UtcNow;
if (cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
var elapsed = _cache.FileModified(_name);
// normalize RAMDirectory and FSDirectory times
if (elapsed > ticks1970)
{
elapsed -= ticks1970;
}
var cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime();
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
using (StreamOutput fileStream = _directory.CreateCachedOutputAsStream(_name))
{
var data = await store.LoadData(location, null, encryptor).ConfigureAwait(false);
if (data == null)
{
throw new System.IO.FileNotFoundException(_name);
}
await data.Stream.CopyToStream(fileStream, CancellationToken.None).ConfigureAwait(false);
}
// and open it as an input
_indexInput = _cache.OpenInput(_name);
}
else
{
// open the file in read only mode
_indexInput = _cache.OpenInput(_name);
}
}
/// <summary>Returns a <see cref="Status" /> instance detailing
/// the state of the index.
///
/// </summary>
/// <param name="onlySegments">list of specific segment names to check
///
/// <p/>As this method checks every byte in the specified
/// segments, on a large index it can take quite a long
/// time to run.
///
/// <p/><b>WARNING</b>: make sure
/// you only call this when the index is not opened by any
/// writer.
/// </param>
public virtual Status CheckIndex_Renamed_Method(List <string> onlySegments, IState state)
{
System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
SegmentInfos sis = new SegmentInfos();
Status result = new Status();
result.dir = dir;
try
{
sis.Read(dir, state);
}
catch (System.Exception t)
{
Msg("ERROR: could not read any segments file in directory");
result.missingSegments = true;
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
return(result);
}
int numSegments = sis.Count;
var segmentsFileName = sis.GetCurrentSegmentFileName();
IndexInput input = null;
try
{
input = dir.OpenInput(segmentsFileName, state);
}
catch (System.Exception t)
{
Msg("ERROR: could not open segments file in directory");
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
result.cantOpenSegments = true;
return(result);
}
int format = 0;
try
{
format = input.ReadInt(state);
}
catch (System.Exception t)
{
Msg("ERROR: could not read segment file version in directory");
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
result.missingSegmentVersion = true;
return(result);
}
finally
{
if (input != null)
{
input.Close();
}
}
System.String sFormat = "";
bool skip = false;
if (format == SegmentInfos.FORMAT)
{
sFormat = "FORMAT [Lucene Pre-2.1]";
}
if (format == SegmentInfos.FORMAT_LOCKLESS)
{
sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
}
else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
{
sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
}
else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
}
else
{
if (format == SegmentInfos.FORMAT_CHECKSUM)
{
sFormat = "FORMAT_CHECKSUM [Lucene 2.4]";
}
else if (format == SegmentInfos.FORMAT_DEL_COUNT)
{
sFormat = "FORMAT_DEL_COUNT [Lucene 2.4]";
}
else if (format == SegmentInfos.FORMAT_HAS_PROX)
{
sFormat = "FORMAT_HAS_PROX [Lucene 2.4]";
}
else if (format == SegmentInfos.FORMAT_USER_DATA)
{
sFormat = "FORMAT_USER_DATA [Lucene 2.9]";
}
else if (format == SegmentInfos.FORMAT_DIAGNOSTICS)
{
sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
}
else if (format < SegmentInfos.CURRENT_FORMAT)
{
sFormat = "int=" + format + " [newer version of Lucene than this tool]";
skip = true;
}
else
{
sFormat = format + " [Lucene 1.3 or prior]";
}
}
result.segmentsFileName = segmentsFileName;
result.numSegments = numSegments;
result.segmentFormat = sFormat;
result.userData = sis.UserData;
System.String userDataString;
if (sis.UserData.Count > 0)
{
userDataString = " userData=" + CollectionsHelper.CollectionToString(sis.UserData);
}
else
{
userDataString = "";
}
Msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString);
if (onlySegments != null)
{
result.partial = true;
if (infoStream != null)
{
infoStream.Write("\nChecking only these segments:");
}
foreach (string s in onlySegments)
{
if (infoStream != null)
{
infoStream.Write(" " + s);
}
}
result.segmentsChecked.AddRange(onlySegments);
Msg(":");
}
if (skip)
{
Msg("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
result.toolOutOfDate = true;
return(result);
}
result.newSegments = (SegmentInfos)sis.Clone();
result.newSegments.Clear();
for (int i = 0; i < numSegments; i++)
{
SegmentInfo info = sis.Info(i);
if (onlySegments != null && !onlySegments.Contains(info.name))
{
continue;
}
var segInfoStat = new Status.SegmentInfoStatus();
result.segmentInfos.Add(segInfoStat);
Msg(" " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
segInfoStat.name = info.name;
segInfoStat.docCount = info.docCount;
int toLoseDocCount = info.docCount;
SegmentReader reader = null;
try
{
Msg(" compound=" + info.GetUseCompoundFile(state));
segInfoStat.compound = info.GetUseCompoundFile(state);
Msg(" hasProx=" + info.HasProx);
segInfoStat.hasProx = info.HasProx;
Msg(" numFiles=" + info.Files(state).Count);
segInfoStat.numFiles = info.Files(state).Count;
Msg(System.String.Format(nf, " size (MB)={0:f}", new System.Object[] { (info.SizeInBytes(state) / (1024.0 * 1024.0)) }));
segInfoStat.sizeMB = info.SizeInBytes(state) / (1024.0 * 1024.0);
IDictionary <string, string> diagnostics = info.Diagnostics;
segInfoStat.diagnostics = diagnostics;
if (diagnostics.Count > 0)
{
Msg(" diagnostics = " + CollectionsHelper.CollectionToString(diagnostics));
}
int docStoreOffset = info.DocStoreOffset;
if (docStoreOffset != -1)
{
Msg(" docStoreOffset=" + docStoreOffset);
segInfoStat.docStoreOffset = docStoreOffset;
Msg(" docStoreSegment=" + info.DocStoreSegment);
segInfoStat.docStoreSegment = info.DocStoreSegment;
Msg(" docStoreIsCompoundFile=" + info.DocStoreIsCompoundFile);
segInfoStat.docStoreCompoundFile = info.DocStoreIsCompoundFile;
}
System.String delFileName = info.GetDelFileName();
if (delFileName == null)
{
Msg(" no deletions");
segInfoStat.hasDeletions = false;
}
else
{
Msg(" has deletions [delFileName=" + delFileName + "]");
segInfoStat.hasDeletions = true;
segInfoStat.deletionsFileName = delFileName;
}
if (infoStream != null)
{
infoStream.Write(" test: open reader.........");
}
reader = SegmentReader.Get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, state);
segInfoStat.openReaderPassed = true;
int numDocs = reader.NumDocs();
toLoseDocCount = numDocs;
if (reader.HasDeletions)
{
if (reader.deletedDocs.Count() != info.GetDelCount(state))
{
throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
}
if (reader.deletedDocs.Count() > reader.MaxDoc)
{
throw new System.SystemException("too many deleted docs: MaxDoc=" + reader.MaxDoc + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
}
if (info.docCount - numDocs != info.GetDelCount(state))
{
throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs reader=" + (info.docCount - numDocs));
}
segInfoStat.numDeleted = info.docCount - numDocs;
Msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]");
}
else
{
if (info.GetDelCount(state) != 0)
{
throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount(state) + " vs reader=" + (info.docCount - numDocs));
}
Msg("OK");
}
if (reader.MaxDoc != info.docCount)
{
throw new System.SystemException("SegmentReader.MaxDoc " + reader.MaxDoc + " != SegmentInfos.docCount " + info.docCount);
}
// Test getFieldNames()
if (infoStream != null)
{
infoStream.Write(" test: fields..............");
}
ICollection <string> fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL);
Msg("OK [" + fieldNames.Count + " fields]");
segInfoStat.numFields = fieldNames.Count;
// Test Field Norms
segInfoStat.fieldNormStatus = TestFieldNorms(fieldNames, reader, state);
// Test the Term Index
segInfoStat.termIndexStatus = TestTermIndex(info, reader, state);
// Test Stored Fields
segInfoStat.storedFieldStatus = TestStoredFields(info, reader, nf, state);
// Test Term Vectors
segInfoStat.termVectorStatus = TestTermVectors(info, reader, nf, state);
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented properly
if (segInfoStat.fieldNormStatus.error != null)
{
throw new SystemException("Field Norm test failed");
}
else if (segInfoStat.termIndexStatus.error != null)
{
throw new SystemException("Term Index test failed");
}
else if (segInfoStat.storedFieldStatus.error != null)
{
throw new SystemException("Stored Field test failed");
}
else if (segInfoStat.termVectorStatus.error != null)
{
throw new System.SystemException("Term Vector test failed");
}
Msg("");
}
catch (System.Exception t)
{
Msg("FAILED");
const string comment = "fixIndex() would remove reference to this segment";
Msg(" WARNING: " + comment + "; full exception:");
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
Msg("");
result.totLoseDocCount += toLoseDocCount;
result.numBadSegments++;
continue;
}
finally
{
if (reader != null)
{
reader.Close();
}
}
// Keeper
result.newSegments.Add((SegmentInfo)info.Clone());
}
if (0 == result.numBadSegments)
{
result.clean = true;
Msg("No problems were detected with this index.\n");
}
else
{
Msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected");
}
return(result);
}
public virtual void TestManySubFiles()
{
Directory d = NewFSDirectory(CreateTempDir("CFSManySubFiles"));
int FILE_COUNT = AtLeast(500);
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
{
IndexOutput @out = d.CreateOutput("file." + fileIdx, NewIOContext(Random()));
@out.WriteByte((byte)(sbyte)fileIdx);
@out.Dispose();
}
CompoundFileDirectory cfd = new CompoundFileDirectory(d, "c.cfs", NewIOContext(Random()), true);
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
{
string fileName = "file." + fileIdx;
d.Copy(cfd, fileName, fileName, NewIOContext(Random()));
}
cfd.Dispose();
IndexInput[] ins = new IndexInput[FILE_COUNT];
CompoundFileDirectory cfr = new CompoundFileDirectory(d, "c.cfs", NewIOContext(Random()), false);
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
{
ins[fileIdx] = cfr.OpenInput("file." + fileIdx, NewIOContext(Random()));
}
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
{
Assert.AreEqual((byte)fileIdx, ins[fileIdx].ReadByte());
}
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++)
{
ins[fileIdx].Dispose();
}
cfr.Dispose();
d.Dispose();
}
private void AssertSameStreams(string msg, IndexInput expected, IndexInput actual, long seekTo)
{
if (seekTo >= 0 && seekTo < expected.Length())
{
expected.Seek(seekTo);
actual.Seek(seekTo);
AssertSameStreams(msg + ", seek(mid)", expected, actual);
}
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
string fileName = _name;
bool fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
long blobLength = blob.Properties.Length;
long.TryParse(blob.Metadata["CachedLength"], out blobLength);
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
//if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified))
// blobLastModifiedUTC = new DateTime(longLastModified, DateTimeKind.Local).ToUniversalTime();
if (cachedLength != blobLength)
fFileNeeded = true;
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
DateTime cachedLastModifiedUTC = DateTime.MinValue;
if (CacheDirectory is RAMDirectory) {
cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Local).ToUniversalTime();
}
else if (CacheDirectory is FSDirectory) {
cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Utc).ToUniversalTime().AddYears(1969);
}
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
fFileNeeded = true;
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
#if COMPRESSBLOBS
if (_azureDirectory.ShouldCompressFile(_name))
{
// then we will get it fresh into local deflatedName
// StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName));
MemoryStream deflatedStream = new MemoryStream();
// get the deflated blob
_blob.DownloadToStream(deflatedStream);
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length));
// seek back to begininng
deflatedStream.Seek(0, SeekOrigin.Begin);
// open output file for uncompressed contents
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// create decompressor
DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress);
byte[] bytes = new byte[65535];
int nRead = 0;
do
{
nRead = decompressor.Read(bytes, 0, 65535);
if (nRead > 0)
fileStream.Write(bytes, 0, nRead);
} while (nRead == 65535);
decompressor.Close(); // this should close the deflatedFileStream too
fileStream.Close();
}
else
#endif
{
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
fileStream.Close();
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
protected override void Dispose(bool disposing)
{
_fileMutex.WaitOne();
try
{
#if FULLDEBUG
Debug.WriteLine(String.Format("CLOSED READSTREAM local {0}", _name));
#endif
_indexInput.Dispose();
_indexInput = null;
_azureDirectory = null;
_blobContainer = null;
_blob = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
/// <summary> Construct a new SegmentInfo instance by reading a
/// previously saved SegmentInfo from input.
///
/// </summary>
/// <param name="dir">directory to load from
/// </param>
/// <param name="format">format of the segments info file
/// </param>
/// <param name="input">input handle to read segment info from
/// </param>
internal SegmentInfo(Directory dir, int format, IndexInput input, IState state)
{
this.dir = dir;
name = input.ReadString(state);
docCount = input.ReadInt(state);
if (format <= SegmentInfos.FORMAT_LOCKLESS)
{
delGen = input.ReadLong(state);
if (format <= SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
docStoreOffset = input.ReadInt(state);
if (docStoreOffset != -1)
{
docStoreSegment = input.ReadString(state);
docStoreIsCompoundFile = (1 == input.ReadByte(state));
}
else
{
docStoreSegment = name;
docStoreIsCompoundFile = false;
}
}
else
{
docStoreOffset = -1;
docStoreSegment = name;
docStoreIsCompoundFile = false;
}
if (format <= SegmentInfos.FORMAT_SINGLE_NORM_FILE)
{
hasSingleNormFile = (1 == input.ReadByte(state));
}
else
{
hasSingleNormFile = false;
}
int numNormGen = input.ReadInt(state);
if (numNormGen == NO)
{
normGen = null;
}
else
{
normGen = new long[numNormGen];
for (int j = 0; j < numNormGen; j++)
{
normGen[j] = input.ReadLong(state);
}
}
isCompoundFile = (sbyte)input.ReadByte(state);
preLockless = (isCompoundFile == CHECK_DIR);
if (format <= SegmentInfos.FORMAT_DEL_COUNT)
{
delCount = input.ReadInt(state);
System.Diagnostics.Debug.Assert(delCount <= docCount);
}
else
{
delCount = -1;
}
if (format <= SegmentInfos.FORMAT_HAS_PROX)
{
hasProx = input.ReadByte(state) == 1;
}
else
{
hasProx = true;
}
if (format <= SegmentInfos.FORMAT_DIAGNOSTICS)
{
diagnostics = input.ReadStringStringMap(state);
}
else
{
diagnostics = new Dictionary <string, string>();
}
}
else
{
delGen = CHECK_DIR;
normGen = null;
isCompoundFile = (sbyte)(CHECK_DIR);
preLockless = true;
hasSingleNormFile = false;
docStoreOffset = -1;
docStoreIsCompoundFile = false;
docStoreSegment = null;
delCount = -1;
hasProx = true;
diagnostics = new Dictionary <string, string>();
}
}
internal FaultyIndexInput(IndexInput delegate_Renamed)
{
this.delegate_Renamed = delegate_Renamed;
}
internal CSIndexInput(IndexInput @base, long fileOffset, long length, IState state) : this(@base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE, state)
{
}
internal CSIndexInput(IndexInput @base, long fileOffset, long length, int readBufferSize, IState state) : base(readBufferSize)
{
this.base_Renamed = (IndexInput)@base.Clone(state);
this.fileOffset = fileOffset;
this.length = length;
}
internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size, IState state)
{
bool success = false;
isOriginal = true;
try
{
fieldInfos = fn;
cloneableFieldsStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_EXTENSION, readBufferSize, state);
cloneableIndexStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION, readBufferSize, state);
// First version of fdx did not include a format
// header, but, the first int will always be 0 in that
// case
int firstInt = cloneableIndexStream.ReadInt(state);
format = firstInt == 0 ? 0 : firstInt;
if (format > FieldsWriter.FORMAT_CURRENT)
{
throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FieldsWriter.FORMAT_CURRENT + " or lower");
}
formatSize = format > FieldsWriter.FORMAT ? 4 : 0;
if (format < FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
{
cloneableFieldsStream.SetModifiedUTF8StringsMode();
}
fieldsStream = (IndexInput)cloneableFieldsStream.Clone(state);
long indexSize = cloneableIndexStream.Length(state) - formatSize;
if (docStoreOffset != -1)
{
// We read only a slice out of this shared fields file
this.docStoreOffset = docStoreOffset;
this.size = size;
// Verify the file is long enough to hold all of our
// docs
System.Diagnostics.Debug.Assert(((int)(indexSize / 8)) >= size + this.docStoreOffset, "indexSize=" + indexSize + " size=" + size + " docStoreOffset=" + docStoreOffset);
}
else
{
this.docStoreOffset = 0;
this.size = (int)(indexSize >> 3);
}
indexStream = (IndexInput)cloneableIndexStream.Clone(state);
numTotalDocs = (int)(indexSize >> 3);
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
Dispose();
}
}
}
public StreamInput(IndexInput input)
{
Input = input;
}
private SecureStoreIndexInput(SecureStoreIndexInput cloneInput)
{
_fileMutex = BlobMutexManager.GrabMutex(cloneInput._name);
_fileMutex.WaitOne();
try
{
_directory = cloneInput._directory;
_cache = cloneInput._cache;
_name = cloneInput._name;
_indexInput = cloneInput._indexInput.Clone() as IndexInput;
}
catch (Exception)
{
// sometimes we get access denied on the 2nd stream...but not always. I haven't tracked it down yet
// but this covers our tail until I do
LeoTrace.WriteLine(String.Format("Falling back to memory clone for {0}", cloneInput._name));
}
finally
{
_fileMutex.ReleaseMutex();
}
}
private void AssertSameSeekBehavior(string msg, IndexInput expected, IndexInput actual)
{
// seek to 0
long point = 0;
AssertSameStreams(msg + ", seek(0)", expected, actual, point);
// seek to middle
point = expected.Length() / 2l;
AssertSameStreams(msg + ", seek(mid)", expected, actual, point);
// seek to end - 2
point = expected.Length() - 2;
AssertSameStreams(msg + ", seek(end-2)", expected, actual, point);
// seek to end - 1
point = expected.Length() - 1;
AssertSameStreams(msg + ", seek(end-1)", expected, actual, point);
// seek to the end
point = expected.Length();
AssertSameStreams(msg + ", seek(end)", expected, actual, point);
// seek past end
point = expected.Length() + 1;
AssertSameStreams(msg + ", seek(end+1)", expected, actual, point);
}
protected override void Dispose(bool disposing)
{
_fileMutex.WaitOne();
try
{
_indexInput.Dispose();
_indexInput = null;
_directory = null;
_cache = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
private void AssertSameStreams(string msg, IndexInput expected, IndexInput test)
{
Assert.IsNotNull(expected, msg + " null expected");
Assert.IsNotNull(test, msg + " null test");
Assert.AreEqual(expected.Length(), test.Length(), msg + " length");
Assert.AreEqual(expected.FilePointer, test.FilePointer, msg + " position");
var expectedBuffer = new byte[512];
var testBuffer = new byte[expectedBuffer.Length];
long remainder = expected.Length() - expected.FilePointer;
while (remainder > 0)
{
int readLen = (int)Math.Min(remainder, expectedBuffer.Length);
expected.ReadBytes(expectedBuffer, 0, readLen);
test.ReadBytes(testBuffer, 0, readLen);
AssertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
remainder -= readLen;
}
}
internal SegmentTermEnum(IndexInput i, FieldInfos fis, bool isi, IState state)
{
input = i;
fieldInfos = fis;
isIndex = isi;
maxSkipLevels = 1; // use single-level skip lists for formats > -3
int firstInt = input.ReadInt(state);
if (firstInt >= 0)
{
// original-format file, without explicit format version number
format = 0;
size = firstInt;
// back-compatible settings
indexInterval = 128;
skipInterval = System.Int32.MaxValue; // switch off skipTo optimization
}
else
{
// we have a format version number
format = firstInt;
// check that it is a format we can understand
if (format < TermInfosWriter.FORMAT_CURRENT)
{
throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher");
}
size = input.ReadLong(state); // read the size
if (format == -1)
{
if (!isIndex)
{
indexInterval = input.ReadInt(state);
formatM1SkipInterval = input.ReadInt(state);
}
// switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in
// skipTo implementation of these versions
skipInterval = System.Int32.MaxValue;
}
else
{
indexInterval = input.ReadInt(state);
skipInterval = input.ReadInt(state);
if (format <= TermInfosWriter.FORMAT)
{
// this new format introduces multi-level skipping
maxSkipLevels = input.ReadInt(state);
}
}
System.Diagnostics.Debug.Assert(indexInterval > 0, "indexInterval=" + indexInterval + " is negative; must be > 0");
System.Diagnostics.Debug.Assert(skipInterval > 0, "skipInterval=" + skipInterval + " is negative; must be > 0");
}
if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
{
termBuffer.SetPreUTF8Strings();
scanBuffer.SetPreUTF8Strings();
prevBuffer.SetPreUTF8Strings();
}
}