private void AddPositions(DocsAndPositionsEnum @in, IndexOutput @out)
{
int freq = @in.Freq;
@out.WriteVInt32(freq);
int previousPosition = 0;
int previousEndOffset = 0;
for (int i = 0; i < freq; i++)
{
int pos = @in.NextPosition();
BytesRef payload = @in.GetPayload();
// The low-order bit of token is set only if there is a payload, the
// previous bits are the delta-encoded position.
int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1);
@out.WriteVInt32(token);
previousPosition = pos;
if (storeOffsets) // don't encode offsets if they are not stored
{
int startOffset = @in.StartOffset;
int endOffset = @in.EndOffset;
@out.WriteVInt32(startOffset - previousEndOffset);
@out.WriteVInt32(endOffset - startOffset);
previousEndOffset = endOffset;
}
if (payload != null)
{
@out.WriteVInt32(payload.Length);
@out.WriteBytes(payload.Bytes, payload.Offset, payload.Length);
}
}
}
/// <summary>
/// Write a block of data (<c>For</c> format).
/// </summary>
/// <param name="data"> The data to write. </param>
/// <param name="encoded"> A buffer to use to encode data. </param>
/// <param name="out"> The destination output. </param>
/// <exception cref="IOException"> If there is a low-level I/O error. </exception>
internal void WriteBlock(int[] data, byte[] encoded, IndexOutput @out)
{
if (IsAllEqual(data))
{
@out.WriteByte((byte)ALL_VALUES_EQUAL);
@out.WriteVInt32(data[0]);
return;
}
int numBits = BitsRequired(data);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numBits > 0 && numBits <= 32, numBits.ToString());
}
PackedInt32s.IEncoder encoder = encoders[numBits];
int iters = iterations[numBits];
if (Debugging.AssertsEnabled)
{
Debugging.Assert(iters * encoder.ByteValueCount >= Lucene41PostingsFormat.BLOCK_SIZE);
}
int encodedSize = encodedSizes[numBits];
if (Debugging.AssertsEnabled)
{
Debugging.Assert(iters * encoder.ByteBlockCount >= encodedSize);
}
@out.WriteByte((byte)numBits);
encoder.Encode(data, 0, encoded, 0, iters);
@out.WriteBytes(encoded, encodedSize);
}
/// <summary>
/// Add a new position & payload </summary>
public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset)
{
//if (DEBUG) System.out.println("SPW: addPos pos=" + position + " payload=" + (payload == null ? "null" : (payload.Length + " bytes")) + " proxFP=" + proxOut.getFilePointer());
Debug.Assert(IndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, "invalid indexOptions: " + IndexOptions);
Debug.Assert(ProxOut != null);
int delta = position - LastPosition;
Debug.Assert(delta >= 0, "position=" + position + " lastPosition=" + LastPosition); // not quite right (if pos=0 is repeated twice we don't catch it)
LastPosition = position;
int payloadLength = 0;
if (StorePayloads)
{
payloadLength = payload == null ? 0 : payload.Length;
if (payloadLength != LastPayloadLength)
{
LastPayloadLength = payloadLength;
ProxOut.WriteVInt((delta << 1) | 1);
ProxOut.WriteVInt(payloadLength);
}
else
{
ProxOut.WriteVInt(delta << 1);
}
}
else
{
ProxOut.WriteVInt(delta);
}
if (StoreOffsets)
{
// don't use startOffset - lastEndOffset, because this creates lots of negative vints for synonyms,
// and the numbers aren't that much smaller anyways.
int offsetDelta = startOffset - LastOffset;
int offsetLength = endOffset - startOffset;
Debug.Assert(offsetDelta >= 0 && offsetLength >= 0, "startOffset=" + startOffset + ",lastOffset=" + LastOffset + ",endOffset=" + endOffset);
if (offsetLength != LastOffsetLength)
{
ProxOut.WriteVInt(offsetDelta << 1 | 1);
ProxOut.WriteVInt(offsetLength);
}
else
{
ProxOut.WriteVInt(offsetDelta << 1);
}
LastOffset = startOffset;
LastOffsetLength = offsetLength;
}
if (payloadLength > 0)
{
ProxOut.WriteBytes(payload.Bytes, payload.Offset, payloadLength);
}
}
/// <exception cref="IOException"></exception>
private void CopyBytes(IndexOutput output, Stream input)
{
int numBytes;
while ((numBytes = input.Read(copyBuffer, 0, copyBuffer.Length)) > 0)
{
output.WriteBytes(copyBuffer, 0, numBytes);
}
}
/// <summary>
/// Add a new position & payload </summary>
public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset)
{
// if (DEBUG) {
// System.out.println("FPW.addPosition pos=" + position + " posBufferUpto=" + posBufferUpto + (fieldHasPayloads ? " payloadByteUpto=" + payloadByteUpto: ""));
// }
PosDeltaBuffer[PosBufferUpto] = position - LastPosition;
if (FieldHasPayloads)
{
if (payload == null || payload.Length == 0)
{
// no payload
PayloadLengthBuffer[PosBufferUpto] = 0;
}
else
{
PayloadLengthBuffer[PosBufferUpto] = payload.Length;
if (PayloadByteUpto + payload.Length > PayloadBytes.Length)
{
PayloadBytes = ArrayUtil.Grow(PayloadBytes, PayloadByteUpto + payload.Length);
}
Array.Copy(payload.Bytes, payload.Offset, PayloadBytes, PayloadByteUpto, payload.Length);
PayloadByteUpto += payload.Length;
}
}
if (FieldHasOffsets)
{
Debug.Assert(startOffset >= LastStartOffset);
Debug.Assert(endOffset >= startOffset);
OffsetStartDeltaBuffer[PosBufferUpto] = startOffset - LastStartOffset;
OffsetLengthBuffer[PosBufferUpto] = endOffset - startOffset;
LastStartOffset = startOffset;
}
PosBufferUpto++;
LastPosition = position;
if (PosBufferUpto == Lucene41PostingsFormat.BLOCK_SIZE)
{
// if (DEBUG) {
// System.out.println(" write pos bulk block @ fp=" + posOut.getFilePointer());
// }
ForUtil.WriteBlock(PosDeltaBuffer, Encoded, PosOut);
if (FieldHasPayloads)
{
ForUtil.WriteBlock(PayloadLengthBuffer, Encoded, PayOut);
PayOut.WriteVInt(PayloadByteUpto);
PayOut.WriteBytes(PayloadBytes, 0, PayloadByteUpto);
PayloadByteUpto = 0;
}
if (FieldHasOffsets)
{
ForUtil.WriteBlock(OffsetStartDeltaBuffer, Encoded, PayOut);
ForUtil.WriteBlock(OffsetLengthBuffer, Encoded, PayOut);
}
PosBufferUpto = 0;
}
}
private static void UnidirectionalSync(AzureDirectory sourceDirectory, Directory destinationDirectory)
{
var sourceFiles = sourceDirectory.ListAll();
var fileNameFilter = IndexFileNameFilter.Filter;
byte[] buffer = new byte[16384];
foreach (string sourceFile in sourceFiles)
{
// only copy file if it is accepted by Lucene's default filter
// and it does not already exist (except for segment map files, we always want those)
if (fileNameFilter.Accept(null, sourceFile) && (!destinationDirectory.FileExists(sourceFile) || sourceFile.StartsWith("segment")))
{
IndexOutput indexOutput = null;
IndexInput indexInput = null;
try
{
indexOutput = destinationDirectory.CreateOutput(sourceFile);
indexInput = sourceDirectory.OpenInput(sourceFile);
long length = indexInput.Length();
long position = 0;
while (position < length)
{
int bytesToRead = position + 16384L > length ? (int)(length - position) : 16384;
indexInput.ReadBytes(buffer, 0, bytesToRead);
indexOutput.WriteBytes(buffer, bytesToRead);
position += bytesToRead;
}
}
finally
{
try
{
indexOutput?.Dispose();
}
finally
{
indexInput?.Dispose();
}
}
}
}
// we'll remove old files from both AzureDirectory's cache directory, as well as our destination directory
// (only when older than 45 minutes - old files may still have active searches on them so we need a margin)
var referenceTimestamp = LuceneTimestampFromDateTime(DateTime.UtcNow.AddMinutes(-45));
// remove old files from AzureDirectory cache directory
RemoveOldFiles(sourceDirectory.CacheDirectory, sourceFiles, referenceTimestamp);
// remove old files from destination directory
RemoveOldFiles(destinationDirectory, sourceFiles, referenceTimestamp);
}
public override void WriteBytes(byte[] b, int offset, int length)
{
if (numWrote >= IO_SLEEP_THRESHOLD)
{
outerInstance.doSleep(rand, length);
numWrote = 0;
}
numWrote += length;
io.WriteBytes(b, offset, length);
}
private void AddVarSortedBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values, IEnumerable <long?> docToOrd)
{
field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_SORTED.Name);
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
/* values */
long startPos = data.FilePointer;
int valueCount = 0;
foreach (BytesRef v in values)
{
data.WriteBytes(v.Bytes, v.Offset, v.Length);
valueCount++;
}
/* addresses */
long maxAddress = data.FilePointer - startPos;
index.WriteLong(maxAddress);
Debug.Assert(valueCount != int.MaxValue); // unsupported by the 4.0 impl
PackedInts.Writer w = PackedInts.GetWriter(index, valueCount + 1, PackedInts.BitsRequired(maxAddress), PackedInts.DEFAULT);
long currentPosition = 0;
foreach (BytesRef v in values)
{
w.Add(currentPosition);
currentPosition += v.Length;
}
// write sentinel
Debug.Assert(currentPosition == maxAddress);
w.Add(currentPosition);
w.Finish();
/* ordinals */
int maxDoc = State.SegmentInfo.DocCount;
Debug.Assert(valueCount > 0);
PackedInts.Writer ords = PackedInts.GetWriter(index, maxDoc, PackedInts.BitsRequired(valueCount - 1), PackedInts.DEFAULT);
foreach (long n in docToOrd)
{
ords.Add((long)n);
}
ords.Finish();
}
protected internal override void FlushBuffer(byte[] b, int offset, int len)
{
RateLimiter.Pause(len);
if (BufferedDelegate != null)
{
BufferedDelegate.FlushBuffer(b, offset, len);
}
else
{
@delegate.WriteBytes(b, offset, len);
}
}
public virtual void TestRawIndexInputRead()
{
Random random = Random;
RAMDirectory dir = new RAMDirectory();
IndexOutput os = dir.CreateOutput("foo", NewIOContext(random));
os.WriteBytes(READ_TEST_BYTES, READ_TEST_BYTES.Length);
os.Dispose();
IndexInput @is = dir.OpenInput("foo", NewIOContext(random));
CheckReads(@is, typeof(IOException));
@is.Dispose();
os = dir.CreateOutput("bar", NewIOContext(random));
os.WriteBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.Length);
os.Dispose();
@is = dir.OpenInput("bar", NewIOContext(random));
CheckRandomReads(@is);
@is.Dispose();
dir.Dispose();
}
public override void WriteBytes(byte[] b, int offset, int length)
{
long before = Time.NanoTime();
// TODO: sometimes, write only half the bytes, then
// sleep, then 2nd half, then sleep, so we sometimes
// interrupt having only written not all bytes
@delegate.WriteBytes(b, offset, length);
timeElapsed += Time.NanoTime() - before;
pendingBytes += length;
Sleep(GetDelay(false));
}
public virtual void Test()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER;
}
PagedBytes pb = new PagedBytes(15);
IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT);
long netBytes = 0;
long seed = Random.NextInt64();
long lastFP = 0;
Random r2 = new J2N.Randomizer(seed);
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt32(r2, 1, 32768);
byte[] bytes = new byte[numBytes];
r2.NextBytes(bytes);
dataOutput.WriteBytes(bytes, bytes.Length);
long fp = dataOutput.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
if (Debugging.AssertsEnabled)
{
Debugging.Assert(fp == lastFP + numBytes);
}
lastFP = fp;
netBytes += numBytes;
}
dataOutput.Dispose();
IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
pb.Copy(input, input.Length);
input.Dispose();
PagedBytes.Reader reader = pb.Freeze(true);
r2 = new J2N.Randomizer(seed);
netBytes = 0;
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt32(r2, 1, 32768);
var bytes = new byte[numBytes];
r2.NextBytes(bytes);
BytesRef expected = new BytesRef(bytes);
BytesRef actual = new BytesRef();
reader.FillSlice(actual, netBytes, numBytes);
Assert.AreEqual(expected, actual);
netBytes += numBytes;
}
dir.Dispose();
}
// NOTE: 4.0 file format docs are crazy/wrong here...
private void AddVarStraightBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values)
{
field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_VAR_STRAIGHT.ToString());
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
/* values */
long startPos = data.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
foreach (BytesRef v in values)
{
if (v != null)
{
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
}
/* addresses */
long maxAddress = data.Position - startPos; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
index.WriteVInt64(maxAddress);
int maxDoc = state.SegmentInfo.DocCount;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(maxDoc != int.MaxValue); // unsupported by the 4.0 impl
}
PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc + 1, PackedInt32s.BitsRequired(maxAddress), PackedInt32s.DEFAULT);
long currentPosition = 0;
foreach (BytesRef v in values)
{
w.Add(currentPosition);
if (v != null)
{
currentPosition += v.Length;
}
}
// write sentinel
if (Debugging.AssertsEnabled)
{
Debugging.Assert(currentPosition == maxAddress);
}
w.Add(currentPosition);
w.Finish();
}
private void AddFixedDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values, int length)
{
field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_FIXED_DEREF.ToString());
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
// deduplicate
JCG.SortedSet <BytesRef> dictionary = new JCG.SortedSet <BytesRef>();
foreach (BytesRef v in values)
{
dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v));
}
/* values */
data.WriteInt32(length);
foreach (BytesRef v in dictionary)
{
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
/* ordinals */
int valueCount = dictionary.Count;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(valueCount > 0);
}
index.WriteInt32(valueCount);
int maxDoc = state.SegmentInfo.DocCount;
PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc, PackedInt32s.BitsRequired(valueCount - 1), PackedInt32s.DEFAULT);
BytesRef brefDummy;
foreach (BytesRef v in values)
{
brefDummy = v;
if (v == null)
{
brefDummy = new BytesRef();
}
//int ord = dictionary.HeadSet(brefDummy).Size();
int ord = dictionary.Count(@ref => @ref.CompareTo(brefDummy) < 0);
w.Add(ord);
}
w.Finish();
}
public long WriteTo(IndexOutput @out)
{
long size = 0;
while (true)
{
if (limit + bufferOffset == endIndex)
{
System.Diagnostics.Debug.Assert(endIndex - bufferOffset >= upto);
@out.WriteBytes(buffer, upto, limit - upto);
size += limit - upto;
break;
}
else
{
@out.WriteBytes(buffer, upto, limit - upto);
size += limit - upto;
NextSlice();
}
}
return(size);
}
public virtual void Test()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
PagedBytes pb = new PagedBytes(15);
IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT);
long netBytes = 0;
long seed = Random().NextLong();
long lastFP = 0;
Random r2 = new Random((int)seed);
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt(r2, 1, 32768);
byte[] bytes = new byte[numBytes];
r2.NextBytes(bytes);
dataOutput.WriteBytes(bytes, bytes.Length);
long fp = dataOutput.FilePointer;
Debug.Assert(fp == lastFP + numBytes);
lastFP = fp;
netBytes += numBytes;
}
dataOutput.Dispose();
IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
pb.Copy(input, input.Length());
input.Dispose();
PagedBytes.Reader reader = pb.Freeze(true);
r2 = new Random((int)seed);
netBytes = 0;
while (netBytes < 1.1 * int.MaxValue)
{
int numBytes = TestUtil.NextInt(r2, 1, 32768);
var bytes = new byte[numBytes];
r2.NextBytes(bytes);
BytesRef expected = new BytesRef(bytes);
BytesRef actual = new BytesRef();
reader.FillSlice(actual, netBytes, numBytes);
Assert.AreEqual(expected, actual);
netBytes += numBytes;
}
dir.Dispose();
}
/// <summary>Copy the contents of the file with specified extension into the
/// provided output stream. Use the provided buffer for moving data
/// to reduce memory allocation.
/// </summary>
private void CopyFile(FileEntry source, IndexOutput os, byte[] buffer)
{
IndexInput isRenamed = null;
try
{
long startPtr = os.FilePointer;
isRenamed = directory.OpenInput(source.file);
long length = isRenamed.Length();
long remainder = length;
int chunk = buffer.Length;
while (remainder > 0)
{
var len = (int)Math.Min(chunk, remainder);
isRenamed.ReadBytes(buffer, 0, len, false);
os.WriteBytes(buffer, len);
remainder -= len;
if (checkAbort != null)
{
// Roughly every 2 MB we will check if
// it's time to abort
checkAbort.Work(80);
}
}
// Verify that remainder is 0
if (remainder != 0)
{
throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")");
}
// Verify that the output length diff is equal to original file
long endPtr = os.FilePointer;
long diff = endPtr - startPtr;
if (diff != length)
{
throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length);
}
}
finally
{
if (isRenamed != null)
{
isRenamed.Close();
}
}
}
private void AddFixedStraightBytesField(FieldInfo field, IndexOutput output, IEnumerable <BytesRef> values, int length)
{
field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_FIXED_STRAIGHT.Name);
CodecUtil.WriteHeader(output, Lucene40DocValuesFormat.BYTES_FIXED_STRAIGHT_CODEC_NAME, Lucene40DocValuesFormat.BYTES_FIXED_STRAIGHT_VERSION_CURRENT);
output.WriteInt(length);
foreach (BytesRef v in values)
{
if (v != null)
{
output.WriteBytes(v.Bytes, v.Offset, v.Length);
}
}
}
public virtual void CopyFile(Directory dir, string src, string dest)
{
IndexInput @in = dir.OpenInput(src, NewIOContext(Random));
IndexOutput @out = dir.CreateOutput(dest, NewIOContext(Random));
var b = new byte[1024];
long remainder = @in.Length;
while (remainder > 0)
{
int len = (int)Math.Min(b.Length, remainder);
@in.ReadBytes(b, 0, len);
@out.WriteBytes(b, len);
remainder -= len;
}
@in.Dispose();
@out.Dispose();
}
// NOTE: 4.0 file format docs are crazy/wrong here...
private void AddVarStraightBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values)
{
field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_STRAIGHT.Name);
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
/* values */
long startPos = data.FilePointer;
foreach (BytesRef v in values)
{
if (v != null)
{
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
}
/* addresses */
long maxAddress = data.FilePointer - startPos;
index.WriteVLong(maxAddress);
int maxDoc = State.SegmentInfo.DocCount;
Debug.Assert(maxDoc != int.MaxValue); // unsupported by the 4.0 impl
PackedInts.Writer w = PackedInts.GetWriter(index, maxDoc + 1, PackedInts.BitsRequired(maxAddress), PackedInts.DEFAULT);
long currentPosition = 0;
foreach (BytesRef v in values)
{
w.Add(currentPosition);
if (v != null)
{
currentPosition += v.Length;
}
}
// write sentinel
Debug.Assert(currentPosition == maxAddress);
w.Add(currentPosition);
w.Finish();
}
/// <summary>
/// Copy the current contents of this buffer to the named output.
/// </summary>
public virtual void WriteTo(IndexOutput other)
{
Flush();
long num1 = file.Length;
long num2 = 0L;
int num3 = 0;
long num4;
for (; num2 < num1; num2 = num4)
{
int length = 1024;
num4 = num2 + length;
if (num4 > num1)
{
length = (int)(num1 - num2);
}
other.WriteBytes(file.GetBuffer(num3++), length);
}
}
public virtual void TestOverflow() // memory hole
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
int blockBits = TestUtil.NextInt(Random(), 14, 28);
int blockSize = 1 << blockBits;
sbyte[] arr = new sbyte[TestUtil.NextInt(Random(), blockSize / 2, blockSize * 2)];
for (int i = 0; i < arr.Length; ++i)
{
arr[i] = (sbyte)i;
}
long numBytes = (1L << 31) + TestUtil.NextInt(Random(), 1, blockSize * 3);
PagedBytes p = new PagedBytes(blockBits);
IndexOutput @out = dir.CreateOutput("foo", IOContext.DEFAULT);
for (long i = 0; i < numBytes;)
{
Assert.AreEqual(i, @out.FilePointer);
int len = (int)Math.Min(arr.Length, numBytes - i);
@out.WriteBytes(arr, len);
i += len;
}
Assert.AreEqual(numBytes, @out.FilePointer);
@out.Dispose();
IndexInput @in = dir.OpenInput("foo", IOContext.DEFAULT);
p.Copy(@in, numBytes);
PagedBytes.Reader reader = p.Freeze(Random().NextBoolean());
foreach (long offset in new long[] { 0L, int.MaxValue, numBytes - 1, TestUtil.NextLong(Random(), 1, numBytes - 2) })
{
BytesRef b = new BytesRef();
reader.FillSlice(b, offset, 1);
Assert.AreEqual(arr[(int)(offset % arr.Length)], b.Bytes[b.Offset]);
}
@in.Dispose();
dir.Dispose();
}
private void AddVarDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values)
{
field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_DEREF.Name);
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
// deduplicate
SortedSet <BytesRef> dictionary = new SortedSet <BytesRef>();
foreach (BytesRef v in values)
{
dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v));
}
/* values */
long startPosition = data.FilePointer;
long currentAddress = 0;
Dictionary <BytesRef, long> valueToAddress = new Dictionary <BytesRef, long>();
foreach (BytesRef v in dictionary)
{
currentAddress = data.FilePointer - startPosition;
valueToAddress[v] = currentAddress;
WriteVShort(data, v.Length);
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
/* ordinals */
long totalBytes = data.FilePointer - startPosition;
index.WriteLong(totalBytes);
int maxDoc = State.SegmentInfo.DocCount;
PackedInts.Writer w = PackedInts.GetWriter(index, maxDoc, PackedInts.BitsRequired(currentAddress), PackedInts.DEFAULT);
foreach (BytesRef v in values)
{
w.Add(valueToAddress[v == null ? new BytesRef() : v]);
}
w.Finish();
}
/// <summary>
/// Add a new position & payload. </summary>
public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
}
int delta = position - lastPosition;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(delta >= 0, "position={0} lastPosition={1}", position, lastPosition); // not quite right (if pos=0 is repeated twice we don't catch it)
}
lastPosition = position;
if (storePayloads)
{
int payloadLength = payload == null ? 0 : payload.Length;
if (payloadLength != lastPayloadLength)
{
lastPayloadLength = payloadLength;
// TODO: explore whether we get better compression
// by not storing payloadLength into prox stream?
posOut.Write((delta << 1) | 1);
posOut.Write(payloadLength);
}
else
{
posOut.Write(delta << 1);
}
if (payloadLength > 0)
{
payloadOut.WriteBytes(payload.Bytes, payload.Offset, payloadLength);
}
}
else
{
posOut.Write(delta);
}
lastPosition = position;
}
private void AddVarDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values)
{
field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_VAR_DEREF.ToString());
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
// deduplicate
JCG.SortedSet <BytesRef> dictionary = new JCG.SortedSet <BytesRef>();
foreach (BytesRef v in values)
{
dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v));
}
/* values */
long startPosition = data.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
long currentAddress = 0;
Dictionary <BytesRef, long> valueToAddress = new Dictionary <BytesRef, long>();
foreach (BytesRef v in dictionary)
{
currentAddress = data.Position - startPosition; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
valueToAddress[v] = currentAddress;
WriteVInt16(data, v.Length);
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
/* ordinals */
long totalBytes = data.Position - startPosition; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
index.WriteInt64(totalBytes);
int maxDoc = state.SegmentInfo.DocCount;
PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc, PackedInt32s.BitsRequired(currentAddress), PackedInt32s.DEFAULT);
foreach (BytesRef v in values)
{
w.Add(valueToAddress[v ?? new BytesRef()]);
}
w.Finish();
}
private int LastFieldNumber = -1; // only for assert
public PreFlexRWNormsConsumer(Directory directory, string segment, IOContext context)
{
string normsFileName = IndexFileNames.SegmentFileName(segment, "", NORMS_EXTENSION);
bool success = false;
IndexOutput output = null;
try
{
output = directory.CreateOutput(normsFileName, context);
output.WriteBytes(NORMS_HEADER, 0, NORMS_HEADER.Length);
@out = output;
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
public virtual void TestLargeWrites()
{
IndexOutput os = Dir.CreateOutput("testBufferStart.txt", NewIOContext(Random()));
var largeBuf = new byte[2048];
for (int i = 0; i < largeBuf.Length; i++)
{
largeBuf[i] = (byte)unchecked((sbyte)(new Random(1).NextDouble() * 256));
}
long currentPos = os.FilePointer;
os.WriteBytes(largeBuf, largeBuf.Length);
try
{
Assert.AreEqual(currentPos + largeBuf.Length, os.FilePointer);
}
finally
{
os.Dispose();
}
}
/// <summary>
/// Add a new position & payload </summary>
public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset)
{
Debug.Assert(INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
int delta = position - LAST_POSITION;
Debug.Assert(delta >= 0, "position=" + position + " lastPosition=" + LAST_POSITION);
// not quite right (if pos=0 is repeated twice we don't catch it)
LAST_POSITION = position;
if (STORE_PAYLOADS)
{
int payloadLength = payload == null ? 0 : payload.Length;
if (payloadLength != LAST_PAYLOAD_LENGTH)
{
LAST_PAYLOAD_LENGTH = payloadLength;
// TODO: explore whether we get better compression
// by not storing payloadLength into prox stream?
POS_OUT.Write((delta << 1) | 1);
POS_OUT.Write(payloadLength);
}
else
{
POS_OUT.Write(delta << 1);
}
if (payloadLength > 0 && payload != null)
{
PAYLOAD_OUT.WriteBytes(payload.Bytes, payload.Offset, payloadLength);
}
}
else
{
POS_OUT.Write(delta);
}
LAST_POSITION = position;
}
public virtual void TestLargeWrites()
{
IndexOutput os = dir.CreateOutput("testBufferStart.txt", NewIOContext(Random));
var largeBuf = new byte[2048];
for (int i = 0; i < largeBuf.Length; i++)
{
largeBuf[i] = (byte)(new Random(1).NextDouble() * 256);
}
long currentPos = os.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
os.WriteBytes(largeBuf, largeBuf.Length);
try
{
Assert.AreEqual(currentPos + largeBuf.Length, os.Position); // LUCENENET specific: Renamed from getFilePointer() to match FileStream
}
finally
{
os.Dispose();
}
}
public virtual void TestLargeWrites()
{
IndexOutput os = dir.CreateOutput("testBufferStart.txt", NewIOContext(Random));
var largeBuf = new byte[2048];
for (int i = 0; i < largeBuf.Length; i++)
{
largeBuf[i] = (byte)(Random.NextDouble() * 256); // LUCENENET: Using Random, since Math.random() doesn't exist in .NET, and it seems to make sense to make this repeatable.
}
long currentPos = os.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream
os.WriteBytes(largeBuf, largeBuf.Length);
try
{
Assert.AreEqual(currentPos + largeBuf.Length, os.Position); // LUCENENET specific: Renamed from getFilePointer() to match FileStream
}
finally
{
os.Dispose();
}
}
