public void Read(IndexInput input, FieldInfos fieldInfos)
{
this.term = null; // invalidate cache
int start = input.ReadVInt();
int length = input.ReadVInt();
int totalLength = start + length;
if (preUTF8Strings)
{
text.SetLength(totalLength);
input.ReadChars(text.result, start, length);
}
else
{
if (dirty)
{
// Fully convert all bytes since bytes is dirty
UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes);
bytes.SetLength(totalLength);
input.ReadBytes(bytes.result, start, length);
UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text);
dirty = false;
}
else
{
// Incrementally convert only the UTF8 bytes that are new:
bytes.SetLength(totalLength);
input.ReadBytes(bytes.result, start, length);
UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text);
}
}
this.field = fieldInfos.FieldName(input.ReadVInt());
}
private void ReadFile(Directory dir, System.String name)
{
IndexInput input = dir.OpenInput(name);
try
{
long size = dir.FileLength(name);
long bytesLeft = size;
while (bytesLeft > 0)
{
int numToRead;
if (bytesLeft < buffer.Length)
{
numToRead = (int)bytesLeft;
}
else
{
numToRead = buffer.Length;
}
input.ReadBytes(buffer, 0, numToRead, false);
bytesLeft -= numToRead;
}
// Don't do this in your real backups! This is just
// to force a backup to take a somewhat long time, to
// make sure we are exercising the fact that the
// IndexWriter should not delete this file even when I
// take my time reading it.
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1));
}
finally
{
input.Close();
}
}
public byte[] GetPayload(byte[] data, int offset)
{
if (!needToLoadPayload)
{
throw new System.IO.IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
}
// read payloads lazily
byte[] retArray;
int retOffset;
if (data == null || data.Length - offset < payloadLength)
{
// the array is too small to store the payload data,
// so we allocate a new one
retArray = new byte[payloadLength];
retOffset = 0;
}
else
{
retArray = data;
retOffset = offset;
}
proxStream.ReadBytes(retArray, retOffset, payloadLength);
needToLoadPayload = false;
return(retArray);
}
/// <summary>Read norms into a pre-allocated array. </summary>
public override void Norms(System.String field, byte[] bytes, int offset)
{
lock (this)
{
Norm norm = (Norm)norms[field];
if (norm == null)
{
Array.Copy(FakeNorms(), 0, bytes, offset, MaxDoc());
return;
}
if (norm.bytes != null)
{
// can copy from cache
Array.Copy(norm.bytes, 0, bytes, offset, MaxDoc());
return;
}
IndexInput normStream = (IndexInput)norm.in_Renamed.Clone();
try
{
// read from disk
normStream.Seek(0);
normStream.ReadBytes(bytes, offset, MaxDoc());
}
finally
{
normStream.Close();
}
}
}
/// <summary>The value of the field in Binary, or null. If null, the Reader value,
/// String value, or TokenStream value is used. Exactly one of stringValue(),
/// readerValue(), binaryValue(), and tokenStreamValue() must be set.
/// </summary>
public override byte[] BinaryValue()
{
Enclosing_Instance.EnsureOpen();
if (fieldsData == null)
{
byte[] b = new byte[toRead];
IndexInput localFieldsStream = GetFieldStream();
//Throw this IO Exception since IndexREader.document does so anyway, so probably not that big of a change for people
//since they are already handling this exception when getting the document
try
{
localFieldsStream.Seek(pointer);
localFieldsStream.ReadBytes(b, 0, b.Length);
if (isCompressed == true)
{
fieldsData = Enclosing_Instance.Uncompress(b);
}
else
{
fieldsData = b;
}
}
catch (System.IO.IOException e)
{
throw new FieldReaderException(e);
}
}
return(fieldsData is byte[] ? (byte[])fieldsData : null);
}
/// <summary>The value of the field as a String, or null. If null, the Reader value,
/// binary value, or TokenStream value is used. Exactly one of stringValue(),
/// readerValue(), binaryValue(), and tokenStreamValue() must be set.
/// </summary>
public override System.String StringValue()
{
Enclosing_Instance.EnsureOpen();
if (fieldsData == null)
{
IndexInput localFieldsStream = GetFieldStream();
try
{
localFieldsStream.Seek(pointer);
if (isCompressed)
{
byte[] b = new byte[toRead];
localFieldsStream.ReadBytes(b, 0, b.Length);
fieldsData = System.Text.Encoding.GetEncoding("UTF-8").GetString(Enclosing_Instance.Uncompress(b));
}
else
{
//read in chars b/c we already know the length we need to read
char[] chars = new char[toRead];
localFieldsStream.ReadChars(chars, 0, toRead);
fieldsData = new System.String(chars);
}
}
catch (System.IO.IOException e)
{
throw new FieldReaderException(e);
}
}
return(fieldsData is System.String ? (System.String)fieldsData : null);
}
/// <summary>The value of the field as a String, or null. If null, the Reader value,
/// binary value, or TokenStream value is used. Exactly one of stringValue(),
/// readerValue(), binaryValue(), and tokenStreamValue() must be set.
/// </summary>
public override System.String StringValue()
{
Enclosing_Instance.EnsureOpen();
if (isBinary)
{
return(null);
}
else
{
if (fieldsData == null)
{
IndexInput localFieldsStream = GetFieldStream();
try
{
localFieldsStream.Seek(pointer);
if (isCompressed)
{
byte[] b = new byte[toRead];
localFieldsStream.ReadBytes(b, 0, b.Length);
fieldsData = System.Text.Encoding.GetEncoding("UTF-8").GetString(Enclosing_Instance.Uncompress(b));
}
else
{
if (Enclosing_Instance.format >= FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
{
byte[] bytes = new byte[toRead];
localFieldsStream.ReadBytes(bytes, 0, toRead);
fieldsData = System.Text.Encoding.UTF8.GetString(bytes);
}
else
{
//read in chars b/c we already know the length we need to read
char[] chars = new char[toRead];
localFieldsStream.ReadChars(chars, 0, toRead);
fieldsData = new System.String(chars);
}
}
}
catch (System.IO.IOException e)
{
throw new FieldReaderException(e);
}
}
}
return((string)fieldsData);
}
// in merge mode we don't uncompress the data of a compressed field
private void AddFieldForMerge(Document doc, FieldInfo fi, bool binary, bool compressed, bool tokenize)
{
System.Object data;
if (binary || compressed)
{
int toRead = fieldsStream.ReadVInt();
byte[] b = new byte[toRead];
fieldsStream.ReadBytes(b, 0, b.Length);
data = b;
}
else
{
data = fieldsStream.ReadString();
}
doc.Add(new FieldForMerge(data, fi, binary, compressed, tokenize));
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp", null);
IndexInput is_Renamed = cr.OpenInput("f2", null);
is_Renamed.Seek(is_Renamed.Length(null) - 10, null);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10, null);
Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadByte(null), "Single byte read past end of file");
is_Renamed.Seek(is_Renamed.Length(null) - 10, null);
Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadBytes(b, 0, 50, null), "Block read past end of file");
is_Renamed.Close();
cr.Close();
}
/// <summary>Expert: implements buffer refill. Reads bytes from the current
/// position in the input.
/// </summary>
/// <param name="b">the array to read bytes into
/// </param>
/// <param name="offset">the offset in the array to start storing bytes
/// </param>
/// <param name="len">the number of bytes to read
/// </param>
public override void ReadInternal(byte[] b, int offset, int len)
{
long start = GetFilePointer();
if (start + len > length)
{
throw new System.IO.IOException("read past EOF");
}
base_Renamed.Seek(fileOffset + start);
base_Renamed.ReadBytes(b, offset, len, false);
}
private void AddField(Document doc, FieldInfo fi, bool binary, bool compressed, bool tokenize)
{
//we have a binary stored field, and it may be compressed
if (binary)
{
int toRead = fieldsStream.ReadVInt();
var b = new byte[toRead];
fieldsStream.ReadBytes(b, 0, b.Length);
doc.Add(compressed ? new Field(fi.name, Uncompress(b), Field.Store.YES) : new Field(fi.name, b, Field.Store.YES));
}
else
{
const Field.Store store = Field.Store.YES;
Field.Index index = FieldExtensions.ToIndex(fi.isIndexed, tokenize);
Field.TermVector termVector = FieldExtensions.ToTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
AbstractField f;
if (compressed)
{
int toRead = fieldsStream.ReadVInt();
var b = new byte[toRead];
fieldsStream.ReadBytes(b, 0, b.Length);
f = new Field(fi.name, false, System.Text.Encoding.GetEncoding("UTF-8").GetString(Uncompress(b)), store, index,
termVector)
{
OmitTermFreqAndPositions = fi.omitTermFreqAndPositions, OmitNorms = fi.omitNorms
};
}
else
{
f = new Field(fi.name, false, fieldsStream.ReadString(), store, index, termVector)
{
OmitTermFreqAndPositions = fi.omitTermFreqAndPositions, OmitNorms = fi.omitNorms
};
}
doc.Add(f);
}
}
public override byte[] GetBinaryValue(byte[] result)
{
Enclosing_Instance.EnsureOpen();
if (isBinary)
{
if (fieldsData == null)
{
// Allocate new buffer if result is null or too small
byte[] b;
if (result == null || result.Length < toRead)
{
b = new byte[toRead];
}
else
{
b = result;
}
IndexInput localFieldsStream = GetFieldStream();
// Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people
// since they are already handling this exception when getting the document
try
{
localFieldsStream.Seek(pointer);
localFieldsStream.ReadBytes(b, 0, toRead);
if (isCompressed == true)
{
fieldsData = Enclosing_Instance.Uncompress(b);
}
else
{
fieldsData = b;
}
}
catch (System.IO.IOException e)
{
throw new FieldReaderException(e);
}
binaryOffset = 0;
binaryLength = toRead;
}
return((byte[])fieldsData);
}
else
{
return(null);
}
}
/// <summary>Copy the contents of the file with specified extension into the
/// provided output stream. Use the provided buffer for moving data
/// to reduce memory allocation.
/// </summary>
private void CopyFile(FileEntry source, IndexOutput os, byte[] buffer)
{
IndexInput is_Renamed = null;
try
{
long startPtr = os.GetFilePointer();
is_Renamed = directory.OpenInput(source.file);
long length = is_Renamed.Length();
long remainder = length;
int chunk = buffer.Length;
while (remainder > 0)
{
int len = (int)System.Math.Min(chunk, remainder);
is_Renamed.ReadBytes(buffer, 0, len, false);
os.WriteBytes(buffer, len);
remainder -= len;
if (checkAbort != null)
{
// Roughly every 2 MB we will check if
// it's time to abort
checkAbort.Work(80);
}
}
// Verify that remainder is 0
if (remainder != 0)
{
throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")");
}
// Verify that the output length diff is equal to original file
long endPtr = os.GetFilePointer();
long diff = endPtr - startPtr;
if (diff != length)
{
throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length);
}
}
finally
{
if (is_Renamed != null)
{
is_Renamed.Close();
}
}
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
IndexInput is_Renamed = cr.OpenInput("f2");
is_Renamed.Seek(is_Renamed.Length() - 10);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10);
try
{
byte test = is_Renamed.ReadByte();
Assert.Fail("Single byte read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: single byte read past end of file: " + e);
}
is_Renamed.Seek(is_Renamed.Length() - 10);
try
{
is_Renamed.ReadBytes(b, 0, 50);
Assert.Fail("Block read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: block read past end of file: " + e);
}
is_Renamed.Close();
cr.Close();
}
/// <summary>Constructs a bit vector from the file <code>name</code> in Directory
/// <code>d</code>, as written by the {@link #write} method.
/// </summary>
public BitVector(Directory d, System.String name)
{
IndexInput input = d.OpenInput(name);
try
{
size = input.ReadInt(); // read size
count = input.ReadInt(); // read count
bits = new byte[(size >> 3) + 1]; // allocate bits
input.ReadBytes(bits, 0, bits.Length); // read bits
}
finally
{
input.Close();
}
}
public virtual void CopyFile(Directory dir, System.String src, System.String dest)
{
IndexInput in_Renamed = dir.OpenInput(src);
IndexOutput out_Renamed = dir.CreateOutput(dest);
byte[] b = new byte[1024];
long remainder = in_Renamed.Length();
while (remainder > 0)
{
int len = (int)System.Math.Min(b.Length, remainder);
in_Renamed.ReadBytes(b, 0, len);
out_Renamed.WriteBytes(b, len);
remainder -= len;
}
in_Renamed.Close();
out_Renamed.Close();
}
/** Copy numBytes from srcIn to destIn */
void copyBytes(IndexInput srcIn, IndexOutput destIn, long numBytes)
{
// TODO: we could do this more efficiently (save a copy)
// because it's always from a ByteSliceReader ->
// IndexOutput
while (numBytes > 0)
{
int chunk;
if (numBytes > 4096)
{
chunk = 4096;
}
else
{
chunk = (int)numBytes;
}
srcIn.ReadBytes(copyByteBuffer, 0, chunk);
destIn.WriteBytes(copyByteBuffer, 0, chunk);
numBytes -= chunk;
}
}
private void AssertSameStreams(System.String msg, IndexInput expected, IndexInput test)
{
Assert.IsNotNull(expected, msg + " null expected");
Assert.IsNotNull(test, msg + " null test");
Assert.AreEqual(expected.Length(), test.Length(), msg + " length");
Assert.AreEqual(expected.GetFilePointer(), test.GetFilePointer(), msg + " position");
byte[] expectedBuffer = new byte[512];
byte[] testBuffer = new byte[expectedBuffer.Length];
long remainder = expected.Length() - expected.GetFilePointer();
while (remainder > 0)
{
int readLen = (int)System.Math.Min(remainder, expectedBuffer.Length);
expected.ReadBytes(expectedBuffer, 0, readLen);
test.ReadBytes(testBuffer, 0, readLen);
AssertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
remainder -= readLen;
}
}
public static void Main(System.String[] args)
{
System.String filename = null;
bool extract = false;
for (int i = 0; i < args.Length; ++i)
{
if (args[i].Equals("-extract"))
{
extract = true;
}
else if (filename == null)
{
filename = args[i];
}
}
if (filename == null)
{
System.Console.Out.WriteLine("Usage: Lucene.Net.index.IndexReader [-extract] <cfsfile>");
return;
}
Directory dir = null;
CompoundFileReader cfr = null;
try
{
System.IO.FileInfo file = new System.IO.FileInfo(filename);
System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName;
filename = file.Name;
dir = FSDirectory.GetDirectory(dirname, false);
cfr = new CompoundFileReader(dir, filename);
System.String[] files = cfr.List();
System.Array.Sort(files); // sort the array of filename so that the output is more readable
for (int i = 0; i < files.Length; ++i)
{
long len = cfr.FileLength(files[i]);
if (extract)
{
System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory...");
IndexInput ii = cfr.OpenInput(files[i]);
System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create);
// read and write with a small buffer, which is more effectiv than reading byte by byte
byte[] buffer = new byte[1024];
int chunk = buffer.Length;
while (len > 0)
{
int bufLen = (int)System.Math.Min(chunk, len);
ii.ReadBytes(buffer, 0, bufLen);
byte[] byteArray = new byte[buffer.Length];
for (int index = 0; index < buffer.Length; index++)
{
byteArray[index] = (byte)buffer[index];
}
f.Write(byteArray, 0, bufLen);
len -= bufLen;
}
f.Close();
ii.Close();
}
else
{
System.Console.Out.WriteLine(files[i] + ": " + len + " bytes");
}
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
finally
{
try
{
if (dir != null)
{
dir.Close();
}
if (cfr != null)
{
cfr.Close();
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
}
}
private NumericDocValues LoadByteField(FieldInfo field, IndexInput input)
{
CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_START, Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
int valueSize = input.ReadInt();
if (valueSize != 1)
{
throw new CorruptIndexException("invalid valueSize: " + valueSize);
}
int maxDoc = State.SegmentInfo.DocCount;
var values = new byte[maxDoc];
input.ReadBytes(values, 0, values.Length);
RamBytesUsed_Renamed.AddAndGet(RamUsageEstimator.SizeOf(values));
return new NumericDocValuesAnonymousInnerClassHelper3(values);
}
internal SkipBuffer(IndexInput input, int length)
{
data = new byte[length];
pointer = input.GetFilePointer();
input.ReadBytes(data, 0, length);
}
internal SkipBuffer(IndexInput input, int length)
: base("SkipBuffer on " + input)
{
Data = new byte[length];
Pointer = input.FilePointer;
input.ReadBytes(Data, 0, length);
}
/** Copy numBytes from srcIn to destIn */
void copyBytes(IndexInput srcIn, IndexOutput destIn, long numBytes)
{
// TODO: we could do this more efficiently (save a copy)
// because it's always from a ByteSliceReader ->
// IndexOutput
while (numBytes > 0)
{
int chunk;
if (numBytes > 4096)
chunk = 4096;
else
chunk = (int)numBytes;
srcIn.ReadBytes(copyByteBuffer, 0, chunk);
destIn.WriteBytes(copyByteBuffer, 0, chunk);
numBytes -= chunk;
}
}
/// <summary>Read as a bit set </summary>
private void ReadBits(IndexInput input)
{
count = input.ReadInt(); // read count
bits = new byte[(size >> 3) + 1]; // allocate bits
input.ReadBytes(bits, 0, bits.Length);
}
/// <summary> </summary>
/// <param name="field">The field to read in
/// </param>
/// <param name="tvfPointer">The pointer within the tvf file where we should start reading
/// </param>
/// <param name="mapper">The mapper used to map the TermVector
/// </param>
/// <throws> IOException </throws>
private void ReadTermVector(System.String field, long tvfPointer, TermVectorMapper mapper)
{
// Now read the data from specified position
//We don't need to offset by the FORMAT here since the pointer already includes the offset
tvf.Seek(tvfPointer);
int numTerms = tvf.ReadVInt();
//System.out.println("Num Terms: " + numTerms);
// If no terms - return a constant empty termvector. However, this should never occur!
if (numTerms == 0)
{
return;
}
bool storePositions;
bool storeOffsets;
if (format >= FORMAT_VERSION)
{
byte bits = tvf.ReadByte();
storePositions = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
storeOffsets = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
}
else
{
tvf.ReadVInt();
storePositions = false;
storeOffsets = false;
}
mapper.SetExpectations(field, numTerms, storeOffsets, storePositions);
int start = 0;
int deltaLength = 0;
int totalLength = 0;
byte[] byteBuffer;
char[] charBuffer;
bool preUTF8 = format < FORMAT_UTF8_LENGTH_IN_BYTES;
// init the buffers
if (preUTF8)
{
charBuffer = new char[10];
byteBuffer = null;
}
else
{
charBuffer = null;
byteBuffer = new byte[20];
}
for (int i = 0; i < numTerms; i++)
{
start = tvf.ReadVInt();
deltaLength = tvf.ReadVInt();
totalLength = start + deltaLength;
System.String term;
if (preUTF8)
{
// Term stored as java chars
if (charBuffer.Length < totalLength)
{
char[] newCharBuffer = new char[(int)(1.5 * totalLength)];
Array.Copy(charBuffer, 0, newCharBuffer, 0, start);
charBuffer = newCharBuffer;
}
tvf.ReadChars(charBuffer, start, deltaLength);
term = new System.String(charBuffer, 0, totalLength);
}
else
{
// Term stored as utf8 bytes
if (byteBuffer.Length < totalLength)
{
byte[] newByteBuffer = new byte[(int)(1.5 * totalLength)];
Array.Copy(byteBuffer, 0, newByteBuffer, 0, start);
byteBuffer = newByteBuffer;
}
tvf.ReadBytes(byteBuffer, start, deltaLength);
term = System.Text.Encoding.UTF8.GetString(byteBuffer, 0, totalLength);
}
int freq = tvf.ReadVInt();
int[] positions = null;
if (storePositions)
{
//read in the positions
//does the mapper even care about positions?
if (mapper.IsIgnoringPositions() == false)
{
positions = new int[freq];
int prevPosition = 0;
for (int j = 0; j < freq; j++)
{
positions[j] = prevPosition + tvf.ReadVInt();
prevPosition = positions[j];
}
}
else
{
//we need to skip over the positions. Since these are VInts, I don't believe there is anyway to know for sure how far to skip
//
for (int j = 0; j < freq; j++)
{
tvf.ReadVInt();
}
}
}
TermVectorOffsetInfo[] offsets = null;
if (storeOffsets)
{
//does the mapper even care about offsets?
if (mapper.IsIgnoringOffsets() == false)
{
offsets = new TermVectorOffsetInfo[freq];
int prevOffset = 0;
for (int j = 0; j < freq; j++)
{
int startOffset = prevOffset + tvf.ReadVInt();
int endOffset = startOffset + tvf.ReadVInt();
offsets[j] = new TermVectorOffsetInfo(startOffset, endOffset);
prevOffset = endOffset;
}
}
else
{
for (int j = 0; j < freq; j++)
{
tvf.ReadVInt();
tvf.ReadVInt();
}
}
}
mapper.Map(term, freq, offsets, positions);
}
}
public void Read(IndexInput input, FieldInfos fieldInfos)
{
this.Term = null; // invalidate cache
NewSuffixStart = input.ReadVInt();
int length = input.ReadVInt();
int totalLength = NewSuffixStart + length;
Debug.Assert(totalLength <= ByteBlockPool.BYTE_BLOCK_SIZE - 2, "termLength=" + totalLength + ",resource=" + input);
if (Bytes.Bytes.Length < totalLength)
{
Bytes.Grow(totalLength);
}
Bytes.Length = totalLength;
input.ReadBytes(Bytes.Bytes, NewSuffixStart, length);
int fieldNumber = input.ReadVInt();
if (fieldNumber != CurrentFieldNumber)
{
CurrentFieldNumber = fieldNumber;
// NOTE: too much sneakiness here, seriously this is a negative vint?!
if (CurrentFieldNumber == -1)
{
Field = "";
}
else
{
Debug.Assert(fieldInfos.FieldInfo(CurrentFieldNumber) != null, CurrentFieldNumber.ToString());
Field = String.Intern(fieldInfos.FieldInfo(CurrentFieldNumber).Name);
}
}
else
{
Debug.Assert(Field.Equals(fieldInfos.FieldInfo(fieldNumber).Name), "currentFieldNumber=" + CurrentFieldNumber + " field=" + Field + " vs " + fieldInfos.FieldInfo(fieldNumber) == null ? "null" : fieldInfos.FieldInfo(fieldNumber).Name);
}
}
internal SkipBuffer(IndexInput input, int length)
{
data = new byte[length];
pointer = input.GetFilePointer();
input.ReadBytes(data, 0, length);
}
public void Read(IndexInput input, FieldInfos fieldInfos)
{
this.term = null; // invalidate cache
int start = input.ReadVInt();
int length = input.ReadVInt();
int totalLength = start + length;
if (preUTF8Strings)
{
text.SetLength(totalLength);
input.ReadChars(text.result, start, length);
}
else
{
if (dirty)
{
// Fully convert all bytes since bytes is dirty
UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes);
bytes.SetLength(totalLength);
input.ReadBytes(bytes.result, start, length);
UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text);
dirty = false;
}
else
{
// Incrementally convert only the UTF8 bytes that are new:
bytes.SetLength(totalLength);
input.ReadBytes(bytes.result, start, length);
UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text);
}
}
this.field = fieldInfos.FieldName(input.ReadVInt());
}
/// <summary>
/// Read this many bytes from in </summary>
public void Copy(IndexInput @in, long byteCount)
{
while (byteCount > 0)
{
int left = BlockSize - Upto;
if (left == 0)
{
if (CurrentBlock != null)
{
Blocks.Add(CurrentBlock);
BlockEnd.Add(Upto);
}
CurrentBlock = new byte[BlockSize];
Upto = 0;
left = BlockSize;
}
if (left < byteCount)
{
@in.ReadBytes(CurrentBlock, Upto, left, false);
Upto = BlockSize;
byteCount -= left;
}
else
{
@in.ReadBytes(CurrentBlock, Upto, (int)byteCount, false);
Upto += (int)byteCount;
break;
}
}
}
public /*internal*/ Document Doc(int n)
{
indexStream.Seek(n * 8L);
long position = indexStream.ReadLong();
fieldsStream.Seek(position);
Document doc = new Document();
int numFields = fieldsStream.ReadVInt();
for (int i = 0; i < numFields; i++)
{
int fieldNumber = fieldsStream.ReadVInt();
FieldInfo fi = fieldInfos.FieldInfo(fieldNumber);
byte bits = fieldsStream.ReadByte();
bool compressed = (bits & FieldsWriter.FIELD_IS_COMPRESSED) != 0;
bool tokenize = (bits & FieldsWriter.FIELD_IS_TOKENIZED) != 0;
if ((bits & FieldsWriter.FIELD_IS_BINARY) != 0)
{
byte[] b = new byte[fieldsStream.ReadVInt()];
fieldsStream.ReadBytes(b, 0, b.Length);
if (compressed)
{
doc.Add(new Field(fi.name, Uncompress(b), Field.Store.COMPRESS));
}
else
{
doc.Add(new Field(fi.name, b, Field.Store.YES));
}
}
else
{
Field.Index index;
Field.Store store = Field.Store.YES;
if (fi.isIndexed && tokenize)
{
index = Field.Index.TOKENIZED;
}
else if (fi.isIndexed && !tokenize)
{
index = Field.Index.UN_TOKENIZED;
}
else
{
index = Field.Index.NO;
}
Field.TermVector termVector = null;
if (fi.storeTermVector)
{
if (fi.storeOffsetWithTermVector)
{
if (fi.storePositionWithTermVector)
{
termVector = Field.TermVector.WITH_POSITIONS_OFFSETS;
}
else
{
termVector = Field.TermVector.WITH_OFFSETS;
}
}
else if (fi.storePositionWithTermVector)
{
termVector = Field.TermVector.WITH_POSITIONS;
}
else
{
termVector = Field.TermVector.YES;
}
}
else
{
termVector = Field.TermVector.NO;
}
if (compressed)
{
store = Field.Store.COMPRESS;
byte[] b = new byte[fieldsStream.ReadVInt()];
fieldsStream.ReadBytes(b, 0, b.Length);
Field f = new Field(fi.name, System.Text.Encoding.GetEncoding("UTF-8").GetString(Uncompress(b)), store, index, termVector);
f.SetOmitNorms(fi.omitNorms);
doc.Add(f);
}
else
{
Field f = new Field(fi.name, fieldsStream.ReadString(), store, index, termVector);
f.SetOmitNorms(fi.omitNorms);
doc.Add(f);
}
}
}
return(doc);
}
private void AssertSameStreams(System.String msg, IndexInput expected, IndexInput test)
{
Assert.IsNotNull(expected, msg + " null expected");
Assert.IsNotNull(test, msg + " null test");
Assert.AreEqual(expected.Length(), test.Length(), msg + " length");
Assert.AreEqual(expected.FilePointer, test.FilePointer, msg + " position");
byte[] expectedBuffer = new byte[512];
byte[] testBuffer = new byte[expectedBuffer.Length];
long remainder = expected.Length() - expected.FilePointer;
while (remainder > 0)
{
int readLen = (int) System.Math.Min(remainder, expectedBuffer.Length);
expected.ReadBytes(expectedBuffer, 0, readLen);
test.ReadBytes(testBuffer, 0, readLen);
AssertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
remainder -= readLen;
}
}
public override void ReadInternal(byte[] b, int offset, int length)
{
SimOutage();
delegate_Renamed.ReadBytes(b, offset, length);
}
/// <summary>Read as a bit set </summary>
private void ReadBits(IndexInput input)
{
count = input.ReadInt(); // read count
bits = new byte[(size >> 3) + 1]; // allocate bits
input.ReadBytes(bits, 0, bits.Length);
}
/// <summary>
/// Read as a bit set </summary>
private void ReadBits(IndexInput input)
{
Count_Renamed = input.ReadInt(); // read count
Bits = new byte[GetNumBytes(Size_Renamed)]; // allocate bits
input.ReadBytes(Bits, 0, Bits.Length);
}
