public virtual void Test() { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER; } PagedBytes pb = new PagedBytes(15); IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT); long netBytes = 0; long seed = Random.NextInt64(); long lastFP = 0; Random r2 = new J2N.Randomizer(seed); while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt32(r2, 1, 32768); byte[] bytes = new byte[numBytes]; r2.NextBytes(bytes); dataOutput.WriteBytes(bytes, bytes.Length); long fp = dataOutput.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream if (Debugging.AssertsEnabled) { Debugging.Assert(fp == lastFP + numBytes); } lastFP = fp; netBytes += numBytes; } dataOutput.Dispose(); IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT); pb.Copy(input, input.Length); input.Dispose(); PagedBytes.Reader reader = pb.Freeze(true); r2 = new J2N.Randomizer(seed); netBytes = 0; while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt32(r2, 1, 32768); var bytes = new byte[numBytes]; r2.NextBytes(bytes); BytesRef expected = new BytesRef(bytes); BytesRef actual = new BytesRef(); reader.FillSlice(actual, netBytes, numBytes); Assert.AreEqual(expected, actual); netBytes += numBytes; } dir.Dispose(); }
public virtual void Test() { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER; } PagedBytes pb = new PagedBytes(15); IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT); long netBytes = 0; long seed = Random().NextLong(); long lastFP = 0; Random r2 = new Random((int)seed); while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt(r2, 1, 32768); byte[] bytes = new byte[numBytes]; r2.NextBytes(bytes); dataOutput.WriteBytes(bytes, bytes.Length); long fp = dataOutput.FilePointer; Debug.Assert(fp == lastFP + numBytes); lastFP = fp; netBytes += numBytes; } dataOutput.Dispose(); IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT); pb.Copy(input, input.Length()); input.Dispose(); PagedBytes.Reader reader = pb.Freeze(true); r2 = new Random((int)seed); netBytes = 0; while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt(r2, 1, 32768); var bytes = new byte[numBytes]; r2.NextBytes(bytes); BytesRef expected = new BytesRef(bytes); BytesRef actual = new BytesRef(); reader.FillSlice(actual, netBytes, numBytes); Assert.AreEqual(expected, actual); netBytes += numBytes; } dir.Dispose(); }
internal Reader(PagedBytes pagedBytes) { blocks = new byte[pagedBytes.blocks.Count][]; for (var i = 0; i < blocks.Length; i++) { blocks[i] = pagedBytes.blocks[i]; } blockEnds = new int[blocks.Length]; for (int i = 0; i < blockEnds.Length; i++) { blockEnds[i] = pagedBytes.blockEnd[i]; } blockBits = pagedBytes.blockBits; blockMask = pagedBytes.blockMask; blockSize = pagedBytes.blockSize; }
internal Reader(PagedBytes pagedBytes) { Blocks = new sbyte[pagedBytes.Blocks.Count][]; for (int i = 0; i < Blocks.Length; i++) { Blocks[i] = pagedBytes.Blocks[i]; } BlockEnds = new int[Blocks.Length]; for (int i = 0; i < BlockEnds.Length; i++) { BlockEnds[i] = pagedBytes.BlockEnd[i]; } BlockBits = pagedBytes.BlockBits; BlockMask = pagedBytes.BlockMask; BlockSize = pagedBytes.BlockSize; }
public virtual void TestOverflow() // memory hole { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER; } int blockBits = TestUtil.NextInt(Random(), 14, 28); int blockSize = 1 << blockBits; var arr = new byte[TestUtil.NextInt(Random(), blockSize / 2, blockSize * 2)]; for (int i = 0; i < arr.Length; ++i) { arr[i] = (byte)(sbyte)i; } long numBytes = (1L << 31) + TestUtil.NextInt(Random(), 1, blockSize * 3); var p = new PagedBytes(blockBits); var @out = dir.CreateOutput("foo", IOContext.DEFAULT); for (long i = 0; i < numBytes;) { Assert.AreEqual(i, @out.FilePointer); int len = (int)Math.Min(arr.Length, numBytes - i); @out.WriteBytes(arr, len); i += len; } Assert.AreEqual(numBytes, @out.FilePointer); @out.Dispose(); IndexInput @in = dir.OpenInput("foo", IOContext.DEFAULT); p.Copy(@in, numBytes); PagedBytes.Reader reader = p.Freeze(Random().NextBoolean()); foreach (long offset in new long[] { 0L, int.MaxValue, numBytes - 1, TestUtil.NextLong(Random(), 1, numBytes - 2) }) { BytesRef b = new BytesRef(); reader.FillSlice(b, offset, 1); Assert.AreEqual(arr[(int)(offset % arr.Length)], b.Bytes[b.Offset]); } @in.Dispose(); dir.Dispose(); }
public PagedBytesDataOutput(PagedBytes pagedBytes) { this.outerInstance = pagedBytes; }
internal PagedBytesDataInput(PagedBytes outerInstance) { this.outerInstance = outerInstance; currentBlock = outerInstance.blocks[0]; }
public virtual void TestDataInputOutput() { Random random = Random(); for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++) { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER; } int blockBits = TestUtil.NextInt(random, 1, 20); int blockSize = 1 << blockBits; PagedBytes p = new PagedBytes(blockBits); IndexOutput @out = dir.CreateOutput("foo", IOContext.DEFAULT); int numBytes = TestUtil.NextInt(Random(), 2, 10000000); byte[] answer = new byte[numBytes]; Random().NextBytes(answer); int written = 0; while (written < numBytes) { if (Random().Next(10) == 7) { @out.WriteByte(answer[written++]); } else { int chunk = Math.Min(Random().Next(1000), numBytes - written); @out.WriteBytes(answer, written, chunk); written += chunk; } } @out.Dispose(); IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT); DataInput @in = (DataInput)input.Clone(); p.Copy(input, input.Length()); PagedBytes.Reader reader = p.Freeze(random.NextBoolean()); byte[] verify = new byte[numBytes]; int read = 0; while (read < numBytes) { if (Random().Next(10) == 7) { verify[read++] = @in.ReadByte(); } else { int chunk = Math.Min(Random().Next(1000), numBytes - read); @in.ReadBytes(verify, read, chunk); read += chunk; } } Assert.IsTrue(Arrays.Equals(answer, verify)); BytesRef slice = new BytesRef(); for (int iter2 = 0; iter2 < 100; iter2++) { int pos = random.Next(numBytes - 1); int len = random.Next(Math.Min(blockSize + 1, numBytes - pos)); reader.FillSlice(slice, pos, len); for (int byteUpto = 0; byteUpto < len; byteUpto++) { Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]); } } input.Dispose(); dir.Dispose(); } }
public virtual void TestOverflow() { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER; } int blockBits = TestUtil.NextInt(Random(), 14, 28); int blockSize = 1 << blockBits; var arr = new byte[TestUtil.NextInt(Random(), blockSize / 2, blockSize * 2)]; for (int i = 0; i < arr.Length; ++i) { arr[i] = (byte)(sbyte)i; } long numBytes = (1L << 31) + TestUtil.NextInt(Random(), 1, blockSize * 3); var p = new PagedBytes(blockBits); var @out = dir.CreateOutput("foo", IOContext.DEFAULT); for (long i = 0; i < numBytes; ) { Assert.AreEqual(i, @out.FilePointer); int len = (int)Math.Min(arr.Length, numBytes - i); @out.WriteBytes(arr, len); i += len; } Assert.AreEqual(numBytes, @out.FilePointer); @out.Dispose(); IndexInput @in = dir.OpenInput("foo", IOContext.DEFAULT); p.Copy(@in, numBytes); PagedBytes.Reader reader = p.Freeze(Random().NextBoolean()); foreach (long offset in new long[] { 0L, int.MaxValue, numBytes - 1, TestUtil.NextLong(Random(), 1, numBytes - 2) }) { BytesRef b = new BytesRef(); reader.FillSlice(b, offset, 1); Assert.AreEqual(arr[(int)(offset % arr.Length)], b.Bytes[b.Offset]); } @in.Dispose(); dir.Dispose(); }
public PagedBytesDataOutput(PagedBytes outerInstance) { this.OuterInstance = outerInstance; }
public BinaryDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance, PagedBytes.Reader bytesReader, MonotonicBlockPackedReader addresses) { this.outerInstance = outerInstance; this.bytesReader = bytesReader; this.addresses = addresses; }
public BinaryDocValuesAnonymousInnerClassHelper(MemoryDocValuesProducer outerInstance, PagedBytes.Reader bytesReader, int fixedLength) { this.bytesReader = bytesReader; this.fixedLength = fixedLength; }
private BinaryDocValues LoadBinary(FieldInfo field) { BinaryEntry entry = binaries[field.Number]; data.Seek(entry.offset); var bytes = new PagedBytes(16); bytes.Copy(data, entry.numBytes); var bytesReader = bytes.Freeze(true); if (entry.minLength == entry.maxLength) { int fixedLength = entry.minLength; ramBytesUsed.AddAndGet(bytes.RamBytesUsed()); return new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength); } else { data.Seek(data.FilePointer + entry.missingBytes); var addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed()); return new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses); } }
internal Reader(PagedBytes pagedBytes) { Blocks = new byte[pagedBytes.Blocks.Count][]; for (var i = 0; i < Blocks.Length; i++) { Blocks[i] = pagedBytes.Blocks[i]; } BlockEnds = new int[Blocks.Length]; for (int i = 0; i < BlockEnds.Length; i++) { BlockEnds[i] = pagedBytes.BlockEnd[i]; } BlockBits = pagedBytes.BlockBits; BlockMask = pagedBytes.BlockMask; BlockSize = pagedBytes.BlockSize; }
public PagedBytesDataOutput(PagedBytes outerInstance) { this.outerInstance = outerInstance; }
public virtual void TestDataInputOutput2() { Random random = Random(); for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++) { int blockBits = TestUtil.NextInt(random, 1, 20); int blockSize = 1 << blockBits; PagedBytes p = new PagedBytes(blockBits); DataOutput @out = p.DataOutput; int numBytes = Random().Next(10000000); byte[] answer = new byte[numBytes]; Random().NextBytes(answer); int written = 0; while (written < numBytes) { if (Random().Next(10) == 7) { @out.WriteByte(answer[written++]); } else { int chunk = Math.Min(Random().Next(1000), numBytes - written); @out.WriteBytes(answer, written, chunk); written += chunk; } } PagedBytes.Reader reader = p.Freeze(random.NextBoolean()); DataInput @in = p.DataInput; byte[] verify = new byte[numBytes]; int read = 0; while (read < numBytes) { if (Random().Next(10) == 7) { verify[read++] = @in.ReadByte(); } else { int chunk = Math.Min(Random().Next(1000), numBytes - read); @in.ReadBytes(verify, read, chunk); read += chunk; } } Assert.IsTrue(Arrays.Equals(answer, verify)); BytesRef slice = new BytesRef(); for (int iter2 = 0; iter2 < 100; iter2++) { int pos = random.Next(numBytes - 1); int len = random.Next(Math.Min(blockSize + 1, numBytes - pos)); reader.FillSlice(slice, pos, len); for (int byteUpto = 0; byteUpto < len; byteUpto++) { Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]); } } } }
internal PagedBytesDataInput(PagedBytes outerInstance) { this.OuterInstance = outerInstance; CurrentBlock = outerInstance.Blocks[0]; }