/// <summary>Retrieve the length (in bytes) of the tvd and tvf
/// entries for the next numDocs starting with
/// startDocID. This is used for bulk copying when
/// merging segments, if the field numbers are
/// congruent. Once this returns, the tvf & tvd streams
/// are seeked to the startDocID.
/// </summary>
internal void RawDocs(int[] tvdLengths, int[] tvfLengths, int startDocID, int numDocs, IState state)
{
if (tvx == null)
{
for (int i = 0; i < tvdLengths.Length; i++)
{
tvdLengths[i] = 0;
}
for (int i = 0; i < tvfLengths.Length; i++)
{
tvfLengths[i] = 0;
}
return;
}
// SegmentMerger calls canReadRawDocs() first and should
// not call us if that returns false.
if (format < FORMAT_VERSION2)
{
throw new System.SystemException("cannot read raw docs with older term vector formats");
}
SeekTvx(startDocID, state);
long tvdPosition = tvx.ReadLong(state);
tvd.Seek(tvdPosition, state);
long tvfPosition = tvx.ReadLong(state);
tvf.Seek(tvfPosition, state);
long lastTvdPosition = tvdPosition;
long lastTvfPosition = tvfPosition;
int count = 0;
while (count < numDocs)
{
int docID = docStoreOffset + startDocID + count + 1;
System.Diagnostics.Debug.Assert(docID <= numTotalDocs);
if (docID < numTotalDocs)
{
tvdPosition = tvx.ReadLong(state);
tvfPosition = tvx.ReadLong(state);
}
else
{
tvdPosition = tvd.Length(state);
tvfPosition = tvf.Length(state);
System.Diagnostics.Debug.Assert(count == numDocs - 1);
}
tvdLengths[count] = (int)(tvdPosition - lastTvdPosition);
tvfLengths[count] = (int)(tvfPosition - lastTvfPosition);
count++;
lastTvdPosition = tvdPosition;
lastTvfPosition = tvfPosition;
}
}
private void SkipPayload(IState state)
{
if (needToLoadPayload && payloadLength > 0)
{
proxStream.Seek(proxStream.FilePointer(state) + payloadLength, state);
}
needToLoadPayload = false;
}
private void SeekTvx(int docNum, IState state)
{
if (format < FORMAT_VERSION2)
{
tvx.Seek((docNum + docStoreOffset) * 8L + FORMAT_SIZE, state);
}
else
{
tvx.Seek((docNum + docStoreOffset) * 16L + FORMAT_SIZE, state);
}
}
protected override MonotonicBlockPackedReader GetOrdIndexInstance(IndexInput data, FieldInfo field,
NumericEntry entry)
{
data.Seek(entry.Offset);
return new MonotonicBlockPackedReader((IndexInput)data.Clone(), entry.PackedIntsVersion, entry.BlockSize, entry.Count,
true);
}
protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field,
BinaryEntry bytes)
{
data.Seek(bytes.AddressesOffset);
return new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count,
true);
}
internal void Seek(long pointer, long p, Term t, TermInfo ti, IState state)
{
input.Seek(pointer, state);
position = p;
termBuffer.Set(t);
prevBuffer.Reset();
termInfo.Set(ti);
}
/// <summary>Expert: implements buffer refill. Reads bytes from the current
/// position in the input.
/// </summary>
/// <param name="b">the array to read bytes into
/// </param>
/// <param name="offset">the offset in the array to start storing bytes
/// </param>
/// <param name="len">the number of bytes to read
/// </param>
public override void ReadInternal(byte[] b, int offset, int len, IState state)
{
long start = FilePointer(state);
if (start + len > length)
{
throw new System.IO.IOException("read past EOF");
}
base_Renamed.Seek(fileOffset + start, state);
base_Renamed.ReadBytes(b, offset, len, false, state);
}
internal virtual void Seek(TermInfo ti, Term term, IState state)
{
count = 0;
FieldInfo fi = parent.core.fieldInfos.FieldInfo(term.Field);
currentFieldOmitTermFreqAndPositions = (fi != null) && fi.omitTermFreqAndPositions;
currentFieldStoresPayloads = (fi != null) && fi.storePayloads;
if (ti.IsEmpty)
{
df = 0;
}
else
{
df = ti.docFreq;
doc = 0;
freqBasePointer = ti.freqPointer;
proxBasePointer = ti.proxPointer;
skipPointer = freqBasePointer + ti.skipOffset;
freqStream.Seek(freqBasePointer, state);
haveSkipped = false;
}
}
/// <summary> Construct a FieldInfos object using the directory and the name of the file
/// IndexInput
/// </summary>
/// <param name="d">The directory to open the IndexInput from
/// </param>
/// <param name="name">The name of the file to open the IndexInput from in the Directory
/// </param>
/// <throws> IOException </throws>
public /*internal*/ FieldInfos(Directory d, String name, IState state)
{
IndexInput input = d.OpenInput(name, state);
try
{
try
{
Read(input, name, state);
}
catch (System.IO.IOException)
{
if (format == FORMAT_PRE)
{
// LUCENE-1623: FORMAT_PRE (before there was a
// format) may be 2.3.2 (pre-utf8) or 2.4.x (utf8)
// encoding; retry with input set to pre-utf8
input.Seek(0, state);
input.SetModifiedUTF8StringsMode();
byNumber.Clear();
byName.Clear();
bool rethrow = false;
try
{
Read(input, name, state);
}
catch (Exception)
{
// Ignore any new exception & set to throw original IOE
rethrow = true;
}
if (rethrow)
{
// Preserve stack trace
throw;
}
}
else
{
// The IOException cannot be caused by
// LUCENE-1623, so re-throw it
throw;
}
}
}
finally
{
input.Close();
}
}
public /*internal*/ Document Doc(int n, FieldSelector fieldSelector, IState state)
{
SeekIndex(n, state);
long position = indexStream.ReadLong(state);
fieldsStream.Seek(position, state);
var doc = new Document();
int numFields = fieldsStream.ReadVInt(state);
for (int i = 0; i < numFields; i++)
{
int fieldNumber = fieldsStream.ReadVInt(state);
FieldInfo fi = fieldInfos.FieldInfo(fieldNumber);
FieldSelectorResult acceptField = fieldSelector == null?FieldSelectorResult.LOAD:fieldSelector.Accept(fi.name);
byte bits = fieldsStream.ReadByte(state);
System.Diagnostics.Debug.Assert(bits <= FieldsWriter.FIELD_IS_COMPRESSED + FieldsWriter.FIELD_IS_TOKENIZED + FieldsWriter.FIELD_IS_BINARY);
bool compressed = (bits & FieldsWriter.FIELD_IS_COMPRESSED) != 0;
System.Diagnostics.Debug.Assert(
(!compressed || (format < FieldsWriter.FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS)),
"compressed fields are only allowed in indexes of version <= 2.9");
bool tokenize = (bits & FieldsWriter.FIELD_IS_TOKENIZED) != 0;
bool binary = (bits & FieldsWriter.FIELD_IS_BINARY) != 0;
//TODO: Find an alternative approach here if this list continues to grow beyond the
//list of 5 or 6 currently here. See Lucene 762 for discussion
if (acceptField.Equals(FieldSelectorResult.LOAD))
{
AddField(doc, fi, binary, compressed, tokenize, state);
}
else if (acceptField.Equals(FieldSelectorResult.LOAD_AND_BREAK))
{
AddField(doc, fi, binary, compressed, tokenize, state);
break; //Get out of this loop
}
else if (acceptField.Equals(FieldSelectorResult.LAZY_LOAD))
{
AddFieldLazy(doc, fi, binary, compressed, tokenize, state);
}
else if (acceptField.Equals(FieldSelectorResult.SIZE))
{
SkipField(binary, compressed, AddFieldSize(doc, fi, binary, compressed, state), state);
}
else if (acceptField.Equals(FieldSelectorResult.SIZE_AND_BREAK))
{
AddFieldSize(doc, fi, binary, compressed, state);
break;
}
else
{
SkipField(binary, compressed, state);
}
}
return(doc);
}
/// <summary>The value of the field as a String, or null. If null, the Reader value,
/// binary value, or TokenStream value is used. Exactly one of StringValue(),
/// ReaderValue(), GetBinaryValue(), and TokenStreamValue() must be set.
/// </summary>
public override string StringValue(IState state)
{
Enclosing_Instance.EnsureOpen();
if (internalIsBinary)
{
return(null);
}
if (fieldsData == null)
{
IndexInput localFieldsStream = GetFieldStream(state);
try
{
localFieldsStream.Seek(pointer, state);
if (isCompressed)
{
var b = new byte[toRead];
localFieldsStream.ReadBytes(b, 0, b.Length, state);
fieldsData =
System.Text.Encoding.GetEncoding("UTF-8").GetString(Enclosing_Instance.Uncompress(b));
}
else
{
if (Enclosing_Instance.format >= FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
{
var bytes = new byte[toRead];
localFieldsStream.ReadBytes(bytes, 0, toRead, state);
fieldsData = System.Text.Encoding.GetEncoding("UTF-8").GetString(bytes);
}
else
{
//read in chars b/c we already know the length we need to read
var chars = new char[toRead];
localFieldsStream.ReadChars(chars, 0, toRead, state);
fieldsData = new System.String(chars);
}
}
}
catch (System.IO.IOException e)
{
throw new FieldReaderException(e);
}
}
return((System.String)fieldsData);
}
/// <summary>
/// Sole constructor. </summary>
public BlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct)
{
this.ValueCount = valueCount;
BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
BlockMask = blockSize - 1;
int numBlocks = PackedInts.NumBlocks(valueCount, blockSize);
long[] minValues = null;
SubReaders = new PackedInts.Reader[numBlocks];
for (int i = 0; i < numBlocks; ++i)
{
int token = @in.ReadByte() & 0xFF;
int bitsPerValue = (int)((uint)token >> AbstractBlockPackedWriter.BPV_SHIFT);
if (bitsPerValue > 64)
{
throw new Exception("Corrupted");
}
if ((token & AbstractBlockPackedWriter.MIN_VALUE_EQUALS_0) == 0)
{
if (minValues == null)
{
minValues = new long[numBlocks];
}
minValues[i] = BlockPackedReaderIterator.ZigZagDecode(1L + BlockPackedReaderIterator.ReadVLong(@in));
}
if (bitsPerValue == 0)
{
SubReaders[i] = new PackedInts.NullReader(blockSize);
}
else
{
int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize);
if (direct)
{
long pointer = @in.FilePointer;
SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
@in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue));
}
else
{
SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
}
}
}
this.MinValues = minValues;
}
public override byte[] GetBinaryValue(byte[] result, IState state)
{
Enclosing_Instance.EnsureOpen();
if (internalIsBinary)
{
if (fieldsData == null)
{
// Allocate new buffer if result is null or too small
byte[] b;
if (result == null || result.Length < toRead)
{
b = new byte[toRead];
}
else
{
b = result;
}
IndexInput localFieldsStream = GetFieldStream(state);
// Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people
// since they are already handling this exception when getting the document
try
{
localFieldsStream.Seek(pointer, state);
localFieldsStream.ReadBytes(b, 0, toRead, state);
fieldsData = isCompressed ? Enclosing_Instance.Uncompress(b) : b;
}
catch (IOException e)
{
throw new FieldReaderException(e);
}
internalbinaryOffset = 0;
internalBinaryLength = toRead;
}
return((byte[])fieldsData);
}
return(null);
}
/// <summary>
/// Sole constructor. </summary>
public MonotonicBlockPackedReader(IndexInput @in, int packedIntsVersion, int blockSize, long valueCount, bool direct)
{
this.ValueCount = valueCount;
BlockShift = PackedInts.CheckBlockSize(blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
BlockMask = blockSize - 1;
int numBlocks = PackedInts.NumBlocks(valueCount, blockSize);
MinValues = new long[numBlocks];
Averages = new float[numBlocks];
SubReaders = new PackedInts.Reader[numBlocks];
for (int i = 0; i < numBlocks; ++i)
{
MinValues[i] = @in.ReadVLong();
Averages[i] = Number.IntBitsToFloat(@in.ReadInt());
int bitsPerValue = @in.ReadVInt();
if (bitsPerValue > 64)
{
throw new Exception("Corrupted");
}
if (bitsPerValue == 0)
{
SubReaders[i] = new PackedInts.NullReader(blockSize);
}
else
{
int size = (int)Math.Min(blockSize, valueCount - (long)i * blockSize);
if (direct)
{
long pointer = @in.FilePointer;
SubReaders[i] = PackedInts.GetDirectReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
@in.Seek(pointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion, size, bitsPerValue));
}
else
{
SubReaders[i] = PackedInts.GetReaderNoHeader(@in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
}
}
}
}
/// <summary>
/// Skip the next block of data.
/// </summary>
/// <param name="in"> the input where to read data </param>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
public void SkipBlock(IndexInput @in)
{
int numBits = @in.ReadByte();
if (numBits == ALL_VALUES_EQUAL)
{
@in.ReadVInt();
return;
}
Debug.Assert(numBits > 0 && numBits <= 32, numBits.ToString());
int encodedSize = EncodedSizes[numBits];
@in.Seek(@in.FilePointer + encodedSize);
}
private void RunReadBytes(IndexInput input, int bufferSize, System.Random r)
{
int pos = 0;
// gradually increasing size:
for (int size = 1; size < bufferSize * 10; size = size + size / 200 + 1)
{
CheckReadBytes(input, size, pos);
pos += size;
if (pos >= TEST_FILE_LENGTH)
{
// wrap
pos = 0;
input.Seek(0L);
}
}
// wildly fluctuating size:
for (long i = 0; i < 1000; i++)
{
int size = r.Next(10000);
CheckReadBytes(input, 1 + size, pos);
pos += 1 + size;
if (pos >= TEST_FILE_LENGTH)
{
// wrap
pos = 0;
input.Seek(0L);
}
}
// constant small size (7 bytes):
for (int i = 0; i < bufferSize; i++)
{
CheckReadBytes(input, 7, pos);
pos += 7;
if (pos >= TEST_FILE_LENGTH)
{
// wrap
pos = 0;
input.Seek(0L);
}
}
}
protected override void SeekDir(IndexInput input, long dirOffset)
{
input.Seek(input.Length() - sizeof(long)/8);
long offset = input.ReadLong();
input.Seek(offset);
}
public virtual void TestDirectInstantiation()
{
DirectoryInfo path = CreateTempDir("testDirectInstantiation");
byte[] largeBuffer = new byte[Random.Next(256 * 1024)], largeReadBuffer = new byte[largeBuffer.Length];
for (int i = 0; i < largeBuffer.Length; i++)
{
largeBuffer[i] = (byte)i; // automatically loops with modulo
}
var dirs = new FSDirectory[] { new SimpleFSDirectory(path, null), new NIOFSDirectory(path, null), new MMapDirectory(path, null) };
for (int i = 0; i < dirs.Length; i++)
{
FSDirectory dir = dirs[i];
dir.EnsureOpen();
string fname = "foo." + i;
string lockname = "foo" + i + ".lck";
IndexOutput @out = dir.CreateOutput(fname, NewIOContext(Random));
@out.WriteByte((byte)(sbyte)i);
@out.WriteBytes(largeBuffer, largeBuffer.Length);
@out.Dispose();
for (int j = 0; j < dirs.Length; j++)
{
FSDirectory d2 = dirs[j];
d2.EnsureOpen();
Assert.IsTrue(SlowFileExists(d2, fname));
Assert.AreEqual(1 + largeBuffer.Length, d2.FileLength(fname));
// LUCENENET specific - unmap hack not needed
//// don't do read tests if unmapping is not supported!
//if (d2 is MMapDirectory && !((MMapDirectory)d2).UseUnmap)
//{
// continue;
//}
IndexInput input = d2.OpenInput(fname, NewIOContext(Random));
Assert.AreEqual((byte)i, input.ReadByte());
// read array with buffering enabled
Arrays.Fill(largeReadBuffer, (byte)0);
input.ReadBytes(largeReadBuffer, 0, largeReadBuffer.Length, true);
Assert.AreEqual(largeBuffer, largeReadBuffer);
// read again without using buffer
input.Seek(1L);
Arrays.Fill(largeReadBuffer, (byte)0);
input.ReadBytes(largeReadBuffer, 0, largeReadBuffer.Length, false);
Assert.AreEqual(largeBuffer, largeReadBuffer);
input.Dispose();
}
// delete with a different dir
dirs[(i + 1) % dirs.Length].DeleteFile(fname);
for (int j = 0; j < dirs.Length; j++)
{
FSDirectory d2 = dirs[j];
Assert.IsFalse(SlowFileExists(d2, fname));
}
Lock @lock = dir.MakeLock(lockname);
Assert.IsTrue(@lock.Obtain());
for (int j = 0; j < dirs.Length; j++)
{
FSDirectory d2 = dirs[j];
Lock lock2 = d2.MakeLock(lockname);
try
{
Assert.IsFalse(lock2.Obtain(1));
}
#pragma warning disable 168
catch (LockObtainFailedException e)
#pragma warning restore 168
{
// OK
}
}
@lock.Dispose();
// now lock with different dir
@lock = dirs[(i + 1) % dirs.Length].MakeLock(lockname);
Assert.IsTrue(@lock.Obtain());
@lock.Dispose();
}
for (int i = 0; i < dirs.Length; i++)
{
FSDirectory dir = dirs[i];
dir.EnsureOpen();
dir.Dispose();
Assert.IsFalse(dir.IsOpen);
}
}
private void SeekIndex(int docID, IState state)
{
indexStream.Seek(formatSize + (docID + docStoreOffset) * 8L, state);
}
public override void SeekInternal(long pos)
{
//simOutage();
delegate_Renamed.Seek(pos, null);
}
public override void Seek(long pos)
{
EnsureOpen();
@delegate.Seek(pos);
}
private void AssertSameStreams(string msg, IndexInput expected, IndexInput actual, long seekTo)
{
if (seekTo >= 0 && seekTo < expected.Length())
{
expected.Seek(seekTo);
actual.Seek(seekTo);
AssertSameStreams(msg + ", seek(mid)", expected, actual);
}
}
/// <summary>
/// Returns (but does not validate) the checksum previously written by <seealso cref="#checkFooter"/>. </summary>
/// <returns> actual checksum value </returns>
/// <exception cref="IOException"> if the footer is invalid </exception>
public static long RetrieveChecksum(IndexInput @in)
{
@in.Seek(@in.Length() - FooterLength());
ValidateFooter(@in);
return @in.ReadLong();
}