public virtual void Test()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER;
}
PagedBytes pb = new PagedBytes(15);
IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT);
long netBytes = 0;
long seed = Random.NextInt64();
long lastFP = 0;
Random r2 = new J2N.Randomizer(seed);
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt32(r2, 1, 32768);
byte[] bytes = new byte[numBytes];
r2.NextBytes(bytes);
dataOutput.WriteBytes(bytes, bytes.Length);
long fp = dataOutput.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
if (Debugging.AssertsEnabled)
{
Debugging.Assert(fp == lastFP + numBytes);
}
lastFP = fp;
netBytes += numBytes;
}
dataOutput.Dispose();
IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
pb.Copy(input, input.Length);
input.Dispose();
PagedBytes.Reader reader = pb.Freeze(true);
r2 = new J2N.Randomizer(seed);
netBytes = 0;
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt32(r2, 1, 32768);
var bytes = new byte[numBytes];
r2.NextBytes(bytes);
BytesRef expected = new BytesRef(bytes);
BytesRef actual = new BytesRef();
reader.FillSlice(actual, netBytes, numBytes);
Assert.AreEqual(expected, actual);
netBytes += numBytes;
}
dir.Dispose();
}
public virtual void Test()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
PagedBytes pb = new PagedBytes(15);
IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT);
long netBytes = 0;
long seed = Random().NextLong();
long lastFP = 0;
Random r2 = new Random((int)seed);
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt(r2, 1, 32768);
byte[] bytes = new byte[numBytes];
r2.NextBytes(bytes);
dataOutput.WriteBytes(bytes, bytes.Length);
long fp = dataOutput.FilePointer;
Debug.Assert(fp == lastFP + numBytes);
lastFP = fp;
netBytes += numBytes;
}
dataOutput.Dispose();
IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
pb.Copy(input, input.Length());
input.Dispose();
PagedBytes.Reader reader = pb.Freeze(true);
r2 = new Random((int)seed);
netBytes = 0;
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt(r2, 1, 32768);
var bytes = new byte[numBytes];
r2.NextBytes(bytes);
BytesRef expected = new BytesRef(bytes);
BytesRef actual = new BytesRef();
reader.FillSlice(actual, netBytes, numBytes);
Assert.AreEqual(expected, actual);
netBytes += numBytes;
}
dir.Dispose();
}
internal Reader(PagedBytes pagedBytes)
{
blocks = new byte[pagedBytes.blocks.Count][];
for (var i = 0; i < blocks.Length; i++)
{
blocks[i] = pagedBytes.blocks[i];
}
blockEnds = new int[blocks.Length];
for (int i = 0; i < blockEnds.Length; i++)
{
blockEnds[i] = pagedBytes.blockEnd[i];
}
blockBits = pagedBytes.blockBits;
blockMask = pagedBytes.blockMask;
blockSize = pagedBytes.blockSize;
}
internal Reader(PagedBytes pagedBytes)
{
Blocks = new sbyte[pagedBytes.Blocks.Count][];
for (int i = 0; i < Blocks.Length; i++)
{
Blocks[i] = pagedBytes.Blocks[i];
}
BlockEnds = new int[Blocks.Length];
for (int i = 0; i < BlockEnds.Length; i++)
{
BlockEnds[i] = pagedBytes.BlockEnd[i];
}
BlockBits = pagedBytes.BlockBits;
BlockMask = pagedBytes.BlockMask;
BlockSize = pagedBytes.BlockSize;
}
public virtual void Test()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
PagedBytes pb = new PagedBytes(15);
IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT);
long netBytes = 0;
long seed = Random().NextLong();
long lastFP = 0;
Random r2 = new Random((int)seed);
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt(r2, 1, 32768);
byte[] bytes = new byte[numBytes];
r2.NextBytes(bytes);
dataOutput.WriteBytes(bytes, bytes.Length);
long fp = dataOutput.FilePointer;
Debug.Assert(fp == lastFP + numBytes);
lastFP = fp;
netBytes += numBytes;
}
dataOutput.Dispose();
IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
pb.Copy(input, input.Length());
input.Dispose();
PagedBytes.Reader reader = pb.Freeze(true);
r2 = new Random((int)seed);
netBytes = 0;
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt(r2, 1, 32768);
var bytes = new byte[numBytes];
r2.NextBytes(bytes);
BytesRef expected = new BytesRef(bytes);
BytesRef actual = new BytesRef();
reader.FillSlice(actual, netBytes, numBytes);
Assert.AreEqual(expected, actual);
netBytes += numBytes;
}
dir.Dispose();
}
public virtual void TestOverflow() // memory hole
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
int blockBits = TestUtil.NextInt(Random(), 14, 28);
int blockSize = 1 << blockBits;
var arr = new byte[TestUtil.NextInt(Random(), blockSize / 2, blockSize * 2)];
for (int i = 0; i < arr.Length; ++i)
{
arr[i] = (byte)(sbyte)i;
}
long numBytes = (1L << 31) + TestUtil.NextInt(Random(), 1, blockSize * 3);
var p = new PagedBytes(blockBits);
var @out = dir.CreateOutput("foo", IOContext.DEFAULT);
for (long i = 0; i < numBytes;)
{
Assert.AreEqual(i, @out.FilePointer);
int len = (int)Math.Min(arr.Length, numBytes - i);
@out.WriteBytes(arr, len);
i += len;
}
Assert.AreEqual(numBytes, @out.FilePointer);
@out.Dispose();
IndexInput @in = dir.OpenInput("foo", IOContext.DEFAULT);
p.Copy(@in, numBytes);
PagedBytes.Reader reader = p.Freeze(Random().NextBoolean());
foreach (long offset in new long[] { 0L, int.MaxValue, numBytes - 1, TestUtil.NextLong(Random(), 1, numBytes - 2) })
{
BytesRef b = new BytesRef();
reader.FillSlice(b, offset, 1);
Assert.AreEqual(arr[(int)(offset % arr.Length)], b.Bytes[b.Offset]);
}
@in.Dispose();
dir.Dispose();
}
public PagedBytesDataOutput(PagedBytes pagedBytes)
{
this.outerInstance = pagedBytes;
}
internal PagedBytesDataInput(PagedBytes outerInstance)
{
this.outerInstance = outerInstance;
currentBlock = outerInstance.blocks[0];
}
public virtual void TestDataInputOutput()
{
Random random = Random();
for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
int blockBits = TestUtil.NextInt(random, 1, 20);
int blockSize = 1 << blockBits;
PagedBytes p = new PagedBytes(blockBits);
IndexOutput @out = dir.CreateOutput("foo", IOContext.DEFAULT);
int numBytes = TestUtil.NextInt(Random(), 2, 10000000);
byte[] answer = new byte[numBytes];
Random().NextBytes(answer);
int written = 0;
while (written < numBytes)
{
if (Random().Next(10) == 7)
{
@out.WriteByte(answer[written++]);
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - written);
@out.WriteBytes(answer, written, chunk);
written += chunk;
}
}
@out.Dispose();
IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
DataInput @in = (DataInput)input.Clone();
p.Copy(input, input.Length());
PagedBytes.Reader reader = p.Freeze(random.NextBoolean());
byte[] verify = new byte[numBytes];
int read = 0;
while (read < numBytes)
{
if (Random().Next(10) == 7)
{
verify[read++] = @in.ReadByte();
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - read);
@in.ReadBytes(verify, read, chunk);
read += chunk;
}
}
Assert.IsTrue(Arrays.Equals(answer, verify));
BytesRef slice = new BytesRef();
for (int iter2 = 0; iter2 < 100; iter2++)
{
int pos = random.Next(numBytes - 1);
int len = random.Next(Math.Min(blockSize + 1, numBytes - pos));
reader.FillSlice(slice, pos, len);
for (int byteUpto = 0; byteUpto < len; byteUpto++)
{
Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]);
}
}
input.Dispose();
dir.Dispose();
}
}
public virtual void TestOverflow()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
int blockBits = TestUtil.NextInt(Random(), 14, 28);
int blockSize = 1 << blockBits;
var arr = new byte[TestUtil.NextInt(Random(), blockSize / 2, blockSize * 2)];
for (int i = 0; i < arr.Length; ++i)
{
arr[i] = (byte)(sbyte)i;
}
long numBytes = (1L << 31) + TestUtil.NextInt(Random(), 1, blockSize * 3);
var p = new PagedBytes(blockBits);
var @out = dir.CreateOutput("foo", IOContext.DEFAULT);
for (long i = 0; i < numBytes; )
{
Assert.AreEqual(i, @out.FilePointer);
int len = (int)Math.Min(arr.Length, numBytes - i);
@out.WriteBytes(arr, len);
i += len;
}
Assert.AreEqual(numBytes, @out.FilePointer);
@out.Dispose();
IndexInput @in = dir.OpenInput("foo", IOContext.DEFAULT);
p.Copy(@in, numBytes);
PagedBytes.Reader reader = p.Freeze(Random().NextBoolean());
foreach (long offset in new long[] { 0L, int.MaxValue, numBytes - 1, TestUtil.NextLong(Random(), 1, numBytes - 2) })
{
BytesRef b = new BytesRef();
reader.FillSlice(b, offset, 1);
Assert.AreEqual(arr[(int)(offset % arr.Length)], b.Bytes[b.Offset]);
}
@in.Dispose();
dir.Dispose();
}
public PagedBytesDataOutput(PagedBytes outerInstance)
{
this.OuterInstance = outerInstance;
}
public BinaryDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance,
PagedBytes.Reader bytesReader, MonotonicBlockPackedReader addresses)
{
this.outerInstance = outerInstance;
this.bytesReader = bytesReader;
this.addresses = addresses;
}
public BinaryDocValuesAnonymousInnerClassHelper(MemoryDocValuesProducer outerInstance,
PagedBytes.Reader bytesReader, int fixedLength)
{
this.bytesReader = bytesReader;
this.fixedLength = fixedLength;
}
private BinaryDocValues LoadBinary(FieldInfo field)
{
BinaryEntry entry = binaries[field.Number];
data.Seek(entry.offset);
var bytes = new PagedBytes(16);
bytes.Copy(data, entry.numBytes);
var bytesReader = bytes.Freeze(true);
if (entry.minLength == entry.maxLength)
{
int fixedLength = entry.minLength;
ramBytesUsed.AddAndGet(bytes.RamBytesUsed());
return new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength);
}
else
{
data.Seek(data.FilePointer + entry.missingBytes);
var addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion,
entry.blockSize, maxDoc, false);
ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed());
return new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses);
}
}
internal Reader(PagedBytes pagedBytes)
{
Blocks = new byte[pagedBytes.Blocks.Count][];
for (var i = 0; i < Blocks.Length; i++)
{
Blocks[i] = pagedBytes.Blocks[i];
}
BlockEnds = new int[Blocks.Length];
for (int i = 0; i < BlockEnds.Length; i++)
{
BlockEnds[i] = pagedBytes.BlockEnd[i];
}
BlockBits = pagedBytes.BlockBits;
BlockMask = pagedBytes.BlockMask;
BlockSize = pagedBytes.BlockSize;
}
public PagedBytesDataOutput(PagedBytes outerInstance)
{
this.outerInstance = outerInstance;
}
public virtual void TestDataInputOutput2()
{
Random random = Random();
for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++)
{
int blockBits = TestUtil.NextInt(random, 1, 20);
int blockSize = 1 << blockBits;
PagedBytes p = new PagedBytes(blockBits);
DataOutput @out = p.DataOutput;
int numBytes = Random().Next(10000000);
byte[] answer = new byte[numBytes];
Random().NextBytes(answer);
int written = 0;
while (written < numBytes)
{
if (Random().Next(10) == 7)
{
@out.WriteByte(answer[written++]);
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - written);
@out.WriteBytes(answer, written, chunk);
written += chunk;
}
}
PagedBytes.Reader reader = p.Freeze(random.NextBoolean());
DataInput @in = p.DataInput;
byte[] verify = new byte[numBytes];
int read = 0;
while (read < numBytes)
{
if (Random().Next(10) == 7)
{
verify[read++] = @in.ReadByte();
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - read);
@in.ReadBytes(verify, read, chunk);
read += chunk;
}
}
Assert.IsTrue(Arrays.Equals(answer, verify));
BytesRef slice = new BytesRef();
for (int iter2 = 0; iter2 < 100; iter2++)
{
int pos = random.Next(numBytes - 1);
int len = random.Next(Math.Min(blockSize + 1, numBytes - pos));
reader.FillSlice(slice, pos, len);
for (int byteUpto = 0; byteUpto < len; byteUpto++)
{
Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]);
}
}
}
}
public virtual void TestDataInputOutput2()
{
Random random = Random();
for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++)
{
int blockBits = TestUtil.NextInt(random, 1, 20);
int blockSize = 1 << blockBits;
PagedBytes p = new PagedBytes(blockBits);
DataOutput @out = p.DataOutput;
int numBytes = Random().Next(10000000);
byte[] answer = new byte[numBytes];
Random().NextBytes(answer);
int written = 0;
while (written < numBytes)
{
if (Random().Next(10) == 7)
{
@out.WriteByte(answer[written++]);
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - written);
@out.WriteBytes(answer, written, chunk);
written += chunk;
}
}
PagedBytes.Reader reader = p.Freeze(random.NextBoolean());
DataInput @in = p.DataInput;
byte[] verify = new byte[numBytes];
int read = 0;
while (read < numBytes)
{
if (Random().Next(10) == 7)
{
verify[read++] = @in.ReadByte();
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - read);
@in.ReadBytes(verify, read, chunk);
read += chunk;
}
}
Assert.IsTrue(Arrays.Equals(answer, verify));
BytesRef slice = new BytesRef();
for (int iter2 = 0; iter2 < 100; iter2++)
{
int pos = random.Next(numBytes - 1);
int len = random.Next(Math.Min(blockSize + 1, numBytes - pos));
reader.FillSlice(slice, pos, len);
for (int byteUpto = 0; byteUpto < len; byteUpto++)
{
Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]);
}
}
}
}
public virtual void TestDataInputOutput()
{
Random random = Random();
for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
int blockBits = TestUtil.NextInt(random, 1, 20);
int blockSize = 1 << blockBits;
PagedBytes p = new PagedBytes(blockBits);
IndexOutput @out = dir.CreateOutput("foo", IOContext.DEFAULT);
int numBytes = TestUtil.NextInt(Random(), 2, 10000000);
byte[] answer = new byte[numBytes];
Random().NextBytes(answer);
int written = 0;
while (written < numBytes)
{
if (Random().Next(10) == 7)
{
@out.WriteByte(answer[written++]);
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - written);
@out.WriteBytes(answer, written, chunk);
written += chunk;
}
}
@out.Dispose();
IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
DataInput @in = (DataInput)input.Clone();
p.Copy(input, input.Length());
PagedBytes.Reader reader = p.Freeze(random.NextBoolean());
byte[] verify = new byte[numBytes];
int read = 0;
while (read < numBytes)
{
if (Random().Next(10) == 7)
{
verify[read++] = @in.ReadByte();
}
else
{
int chunk = Math.Min(Random().Next(1000), numBytes - read);
@in.ReadBytes(verify, read, chunk);
read += chunk;
}
}
Assert.IsTrue(Arrays.Equals(answer, verify));
BytesRef slice = new BytesRef();
for (int iter2 = 0; iter2 < 100; iter2++)
{
int pos = random.Next(numBytes - 1);
int len = random.Next(Math.Min(blockSize + 1, numBytes - pos));
reader.FillSlice(slice, pos, len);
for (int byteUpto = 0; byteUpto < len; byteUpto++)
{
Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]);
}
}
input.Dispose();
dir.Dispose();
}
}
internal PagedBytesDataInput(PagedBytes outerInstance)
{
this.OuterInstance = outerInstance;
CurrentBlock = outerInstance.Blocks[0];
}