private void AddPositions(DocsAndPositionsEnum @in, IndexOutput @out) { int freq = @in.Freq; @out.WriteVInt32(freq); int previousPosition = 0; int previousEndOffset = 0; for (int i = 0; i < freq; i++) { int pos = @in.NextPosition(); BytesRef payload = @in.GetPayload(); // The low-order bit of token is set only if there is a payload, the // previous bits are the delta-encoded position. int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1); @out.WriteVInt32(token); previousPosition = pos; if (storeOffsets) // don't encode offsets if they are not stored { int startOffset = @in.StartOffset; int endOffset = @in.EndOffset; @out.WriteVInt32(startOffset - previousEndOffset); @out.WriteVInt32(endOffset - startOffset); previousEndOffset = endOffset; } if (payload != null) { @out.WriteVInt32(payload.Length); @out.WriteBytes(payload.Bytes, payload.Offset, payload.Length); } } }
/// <summary> /// Write a block of data (<c>For</c> format). /// </summary> /// <param name="data"> The data to write. </param> /// <param name="encoded"> A buffer to use to encode data. </param> /// <param name="out"> The destination output. </param> /// <exception cref="IOException"> If there is a low-level I/O error. </exception> internal void WriteBlock(int[] data, byte[] encoded, IndexOutput @out) { if (IsAllEqual(data)) { @out.WriteByte((byte)ALL_VALUES_EQUAL); @out.WriteVInt32(data[0]); return; } int numBits = BitsRequired(data); if (Debugging.AssertsEnabled) { Debugging.Assert(numBits > 0 && numBits <= 32, numBits.ToString()); } PackedInt32s.IEncoder encoder = encoders[numBits]; int iters = iterations[numBits]; if (Debugging.AssertsEnabled) { Debugging.Assert(iters * encoder.ByteValueCount >= Lucene41PostingsFormat.BLOCK_SIZE); } int encodedSize = encodedSizes[numBits]; if (Debugging.AssertsEnabled) { Debugging.Assert(iters * encoder.ByteBlockCount >= encodedSize); } @out.WriteByte((byte)numBits); encoder.Encode(data, 0, encoded, 0, iters); @out.WriteBytes(encoded, encodedSize); }
/// <summary> /// Add a new position & payload </summary> public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset) { //if (DEBUG) System.out.println("SPW: addPos pos=" + position + " payload=" + (payload == null ? "null" : (payload.Length + " bytes")) + " proxFP=" + proxOut.getFilePointer()); Debug.Assert(IndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, "invalid indexOptions: " + IndexOptions); Debug.Assert(ProxOut != null); int delta = position - LastPosition; Debug.Assert(delta >= 0, "position=" + position + " lastPosition=" + LastPosition); // not quite right (if pos=0 is repeated twice we don't catch it) LastPosition = position; int payloadLength = 0; if (StorePayloads) { payloadLength = payload == null ? 0 : payload.Length; if (payloadLength != LastPayloadLength) { LastPayloadLength = payloadLength; ProxOut.WriteVInt((delta << 1) | 1); ProxOut.WriteVInt(payloadLength); } else { ProxOut.WriteVInt(delta << 1); } } else { ProxOut.WriteVInt(delta); } if (StoreOffsets) { // don't use startOffset - lastEndOffset, because this creates lots of negative vints for synonyms, // and the numbers aren't that much smaller anyways. int offsetDelta = startOffset - LastOffset; int offsetLength = endOffset - startOffset; Debug.Assert(offsetDelta >= 0 && offsetLength >= 0, "startOffset=" + startOffset + ",lastOffset=" + LastOffset + ",endOffset=" + endOffset); if (offsetLength != LastOffsetLength) { ProxOut.WriteVInt(offsetDelta << 1 | 1); ProxOut.WriteVInt(offsetLength); } else { ProxOut.WriteVInt(offsetDelta << 1); } LastOffset = startOffset; LastOffsetLength = offsetLength; } if (payloadLength > 0) { ProxOut.WriteBytes(payload.Bytes, payload.Offset, payloadLength); } }
/// <exception cref="IOException"></exception> private void CopyBytes(IndexOutput output, Stream input) { int numBytes; while ((numBytes = input.Read(copyBuffer, 0, copyBuffer.Length)) > 0) { output.WriteBytes(copyBuffer, 0, numBytes); } }
/// <summary> /// Add a new position & payload </summary> public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset) { // if (DEBUG) { // System.out.println("FPW.addPosition pos=" + position + " posBufferUpto=" + posBufferUpto + (fieldHasPayloads ? " payloadByteUpto=" + payloadByteUpto: "")); // } PosDeltaBuffer[PosBufferUpto] = position - LastPosition; if (FieldHasPayloads) { if (payload == null || payload.Length == 0) { // no payload PayloadLengthBuffer[PosBufferUpto] = 0; } else { PayloadLengthBuffer[PosBufferUpto] = payload.Length; if (PayloadByteUpto + payload.Length > PayloadBytes.Length) { PayloadBytes = ArrayUtil.Grow(PayloadBytes, PayloadByteUpto + payload.Length); } Array.Copy(payload.Bytes, payload.Offset, PayloadBytes, PayloadByteUpto, payload.Length); PayloadByteUpto += payload.Length; } } if (FieldHasOffsets) { Debug.Assert(startOffset >= LastStartOffset); Debug.Assert(endOffset >= startOffset); OffsetStartDeltaBuffer[PosBufferUpto] = startOffset - LastStartOffset; OffsetLengthBuffer[PosBufferUpto] = endOffset - startOffset; LastStartOffset = startOffset; } PosBufferUpto++; LastPosition = position; if (PosBufferUpto == Lucene41PostingsFormat.BLOCK_SIZE) { // if (DEBUG) { // System.out.println(" write pos bulk block @ fp=" + posOut.getFilePointer()); // } ForUtil.WriteBlock(PosDeltaBuffer, Encoded, PosOut); if (FieldHasPayloads) { ForUtil.WriteBlock(PayloadLengthBuffer, Encoded, PayOut); PayOut.WriteVInt(PayloadByteUpto); PayOut.WriteBytes(PayloadBytes, 0, PayloadByteUpto); PayloadByteUpto = 0; } if (FieldHasOffsets) { ForUtil.WriteBlock(OffsetStartDeltaBuffer, Encoded, PayOut); ForUtil.WriteBlock(OffsetLengthBuffer, Encoded, PayOut); } PosBufferUpto = 0; } }
private static void UnidirectionalSync(AzureDirectory sourceDirectory, Directory destinationDirectory) { var sourceFiles = sourceDirectory.ListAll(); var fileNameFilter = IndexFileNameFilter.Filter; byte[] buffer = new byte[16384]; foreach (string sourceFile in sourceFiles) { // only copy file if it is accepted by Lucene's default filter // and it does not already exist (except for segment map files, we always want those) if (fileNameFilter.Accept(null, sourceFile) && (!destinationDirectory.FileExists(sourceFile) || sourceFile.StartsWith("segment"))) { IndexOutput indexOutput = null; IndexInput indexInput = null; try { indexOutput = destinationDirectory.CreateOutput(sourceFile); indexInput = sourceDirectory.OpenInput(sourceFile); long length = indexInput.Length(); long position = 0; while (position < length) { int bytesToRead = position + 16384L > length ? (int)(length - position) : 16384; indexInput.ReadBytes(buffer, 0, bytesToRead); indexOutput.WriteBytes(buffer, bytesToRead); position += bytesToRead; } } finally { try { indexOutput?.Dispose(); } finally { indexInput?.Dispose(); } } } } // we'll remove old files from both AzureDirectory's cache directory, as well as our destination directory // (only when older than 45 minutes - old files may still have active searches on them so we need a margin) var referenceTimestamp = LuceneTimestampFromDateTime(DateTime.UtcNow.AddMinutes(-45)); // remove old files from AzureDirectory cache directory RemoveOldFiles(sourceDirectory.CacheDirectory, sourceFiles, referenceTimestamp); // remove old files from destination directory RemoveOldFiles(destinationDirectory, sourceFiles, referenceTimestamp); }
public override void WriteBytes(byte[] b, int offset, int length) { if (numWrote >= IO_SLEEP_THRESHOLD) { outerInstance.doSleep(rand, length); numWrote = 0; } numWrote += length; io.WriteBytes(b, offset, length); }
private void AddVarSortedBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values, IEnumerable <long?> docToOrd) { field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_SORTED.Name); CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT); CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT); /* values */ long startPos = data.FilePointer; int valueCount = 0; foreach (BytesRef v in values) { data.WriteBytes(v.Bytes, v.Offset, v.Length); valueCount++; } /* addresses */ long maxAddress = data.FilePointer - startPos; index.WriteLong(maxAddress); Debug.Assert(valueCount != int.MaxValue); // unsupported by the 4.0 impl PackedInts.Writer w = PackedInts.GetWriter(index, valueCount + 1, PackedInts.BitsRequired(maxAddress), PackedInts.DEFAULT); long currentPosition = 0; foreach (BytesRef v in values) { w.Add(currentPosition); currentPosition += v.Length; } // write sentinel Debug.Assert(currentPosition == maxAddress); w.Add(currentPosition); w.Finish(); /* ordinals */ int maxDoc = State.SegmentInfo.DocCount; Debug.Assert(valueCount > 0); PackedInts.Writer ords = PackedInts.GetWriter(index, maxDoc, PackedInts.BitsRequired(valueCount - 1), PackedInts.DEFAULT); foreach (long n in docToOrd) { ords.Add((long)n); } ords.Finish(); }
protected internal override void FlushBuffer(byte[] b, int offset, int len) { RateLimiter.Pause(len); if (BufferedDelegate != null) { BufferedDelegate.FlushBuffer(b, offset, len); } else { @delegate.WriteBytes(b, offset, len); } }
public virtual void TestRawIndexInputRead() { Random random = Random; RAMDirectory dir = new RAMDirectory(); IndexOutput os = dir.CreateOutput("foo", NewIOContext(random)); os.WriteBytes(READ_TEST_BYTES, READ_TEST_BYTES.Length); os.Dispose(); IndexInput @is = dir.OpenInput("foo", NewIOContext(random)); CheckReads(@is, typeof(IOException)); @is.Dispose(); os = dir.CreateOutput("bar", NewIOContext(random)); os.WriteBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.Length); os.Dispose(); @is = dir.OpenInput("bar", NewIOContext(random)); CheckRandomReads(@is); @is.Dispose(); dir.Dispose(); }
public override void WriteBytes(byte[] b, int offset, int length) { long before = Time.NanoTime(); // TODO: sometimes, write only half the bytes, then // sleep, then 2nd half, then sleep, so we sometimes // interrupt having only written not all bytes @delegate.WriteBytes(b, offset, length); timeElapsed += Time.NanoTime() - before; pendingBytes += length; Sleep(GetDelay(false)); }
public virtual void Test() { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER; } PagedBytes pb = new PagedBytes(15); IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT); long netBytes = 0; long seed = Random.NextInt64(); long lastFP = 0; Random r2 = new J2N.Randomizer(seed); while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt32(r2, 1, 32768); byte[] bytes = new byte[numBytes]; r2.NextBytes(bytes); dataOutput.WriteBytes(bytes, bytes.Length); long fp = dataOutput.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream if (Debugging.AssertsEnabled) { Debugging.Assert(fp == lastFP + numBytes); } lastFP = fp; netBytes += numBytes; } dataOutput.Dispose(); IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT); pb.Copy(input, input.Length); input.Dispose(); PagedBytes.Reader reader = pb.Freeze(true); r2 = new J2N.Randomizer(seed); netBytes = 0; while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt32(r2, 1, 32768); var bytes = new byte[numBytes]; r2.NextBytes(bytes); BytesRef expected = new BytesRef(bytes); BytesRef actual = new BytesRef(); reader.FillSlice(actual, netBytes, numBytes); Assert.AreEqual(expected, actual); netBytes += numBytes; } dir.Dispose(); }
// NOTE: 4.0 file format docs are crazy/wrong here... private void AddVarStraightBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values) { field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_VAR_STRAIGHT.ToString()); CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT); CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT); /* values */ long startPos = data.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream foreach (BytesRef v in values) { if (v != null) { data.WriteBytes(v.Bytes, v.Offset, v.Length); } } /* addresses */ long maxAddress = data.Position - startPos; // LUCENENET specific: Renamed from getFilePointer() to match FileStream index.WriteVInt64(maxAddress); int maxDoc = state.SegmentInfo.DocCount; if (Debugging.AssertsEnabled) { Debugging.Assert(maxDoc != int.MaxValue); // unsupported by the 4.0 impl } PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc + 1, PackedInt32s.BitsRequired(maxAddress), PackedInt32s.DEFAULT); long currentPosition = 0; foreach (BytesRef v in values) { w.Add(currentPosition); if (v != null) { currentPosition += v.Length; } } // write sentinel if (Debugging.AssertsEnabled) { Debugging.Assert(currentPosition == maxAddress); } w.Add(currentPosition); w.Finish(); }
private void AddFixedDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values, int length) { field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_FIXED_DEREF.ToString()); CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT); CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT); // deduplicate JCG.SortedSet <BytesRef> dictionary = new JCG.SortedSet <BytesRef>(); foreach (BytesRef v in values) { dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v)); } /* values */ data.WriteInt32(length); foreach (BytesRef v in dictionary) { data.WriteBytes(v.Bytes, v.Offset, v.Length); } /* ordinals */ int valueCount = dictionary.Count; if (Debugging.AssertsEnabled) { Debugging.Assert(valueCount > 0); } index.WriteInt32(valueCount); int maxDoc = state.SegmentInfo.DocCount; PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc, PackedInt32s.BitsRequired(valueCount - 1), PackedInt32s.DEFAULT); BytesRef brefDummy; foreach (BytesRef v in values) { brefDummy = v; if (v == null) { brefDummy = new BytesRef(); } //int ord = dictionary.HeadSet(brefDummy).Size(); int ord = dictionary.Count(@ref => @ref.CompareTo(brefDummy) < 0); w.Add(ord); } w.Finish(); }
public long WriteTo(IndexOutput @out) { long size = 0; while (true) { if (limit + bufferOffset == endIndex) { System.Diagnostics.Debug.Assert(endIndex - bufferOffset >= upto); @out.WriteBytes(buffer, upto, limit - upto); size += limit - upto; break; } else { @out.WriteBytes(buffer, upto, limit - upto); size += limit - upto; NextSlice(); } } return(size); }
public virtual void Test() { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("test2BPagedBytes")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER; } PagedBytes pb = new PagedBytes(15); IndexOutput dataOutput = dir.CreateOutput("foo", IOContext.DEFAULT); long netBytes = 0; long seed = Random().NextLong(); long lastFP = 0; Random r2 = new Random((int)seed); while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt(r2, 1, 32768); byte[] bytes = new byte[numBytes]; r2.NextBytes(bytes); dataOutput.WriteBytes(bytes, bytes.Length); long fp = dataOutput.FilePointer; Debug.Assert(fp == lastFP + numBytes); lastFP = fp; netBytes += numBytes; } dataOutput.Dispose(); IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT); pb.Copy(input, input.Length()); input.Dispose(); PagedBytes.Reader reader = pb.Freeze(true); r2 = new Random((int)seed); netBytes = 0; while (netBytes < 1.1 * int.MaxValue) { int numBytes = TestUtil.NextInt(r2, 1, 32768); var bytes = new byte[numBytes]; r2.NextBytes(bytes); BytesRef expected = new BytesRef(bytes); BytesRef actual = new BytesRef(); reader.FillSlice(actual, netBytes, numBytes); Assert.AreEqual(expected, actual); netBytes += numBytes; } dir.Dispose(); }
/// <summary>Copy the contents of the file with specified extension into the /// provided output stream. Use the provided buffer for moving data /// to reduce memory allocation. /// </summary> private void CopyFile(FileEntry source, IndexOutput os, byte[] buffer) { IndexInput isRenamed = null; try { long startPtr = os.FilePointer; isRenamed = directory.OpenInput(source.file); long length = isRenamed.Length(); long remainder = length; int chunk = buffer.Length; while (remainder > 0) { var len = (int)Math.Min(chunk, remainder); isRenamed.ReadBytes(buffer, 0, len, false); os.WriteBytes(buffer, len); remainder -= len; if (checkAbort != null) { // Roughly every 2 MB we will check if // it's time to abort checkAbort.Work(80); } } // Verify that remainder is 0 if (remainder != 0) { throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")"); } // Verify that the output length diff is equal to original file long endPtr = os.FilePointer; long diff = endPtr - startPtr; if (diff != length) { throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length); } } finally { if (isRenamed != null) { isRenamed.Close(); } } }
private void AddFixedStraightBytesField(FieldInfo field, IndexOutput output, IEnumerable <BytesRef> values, int length) { field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_FIXED_STRAIGHT.Name); CodecUtil.WriteHeader(output, Lucene40DocValuesFormat.BYTES_FIXED_STRAIGHT_CODEC_NAME, Lucene40DocValuesFormat.BYTES_FIXED_STRAIGHT_VERSION_CURRENT); output.WriteInt(length); foreach (BytesRef v in values) { if (v != null) { output.WriteBytes(v.Bytes, v.Offset, v.Length); } } }
public virtual void CopyFile(Directory dir, string src, string dest) { IndexInput @in = dir.OpenInput(src, NewIOContext(Random)); IndexOutput @out = dir.CreateOutput(dest, NewIOContext(Random)); var b = new byte[1024]; long remainder = @in.Length; while (remainder > 0) { int len = (int)Math.Min(b.Length, remainder); @in.ReadBytes(b, 0, len); @out.WriteBytes(b, len); remainder -= len; } @in.Dispose(); @out.Dispose(); }
// NOTE: 4.0 file format docs are crazy/wrong here... private void AddVarStraightBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values) { field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_STRAIGHT.Name); CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT); CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT); /* values */ long startPos = data.FilePointer; foreach (BytesRef v in values) { if (v != null) { data.WriteBytes(v.Bytes, v.Offset, v.Length); } } /* addresses */ long maxAddress = data.FilePointer - startPos; index.WriteVLong(maxAddress); int maxDoc = State.SegmentInfo.DocCount; Debug.Assert(maxDoc != int.MaxValue); // unsupported by the 4.0 impl PackedInts.Writer w = PackedInts.GetWriter(index, maxDoc + 1, PackedInts.BitsRequired(maxAddress), PackedInts.DEFAULT); long currentPosition = 0; foreach (BytesRef v in values) { w.Add(currentPosition); if (v != null) { currentPosition += v.Length; } } // write sentinel Debug.Assert(currentPosition == maxAddress); w.Add(currentPosition); w.Finish(); }
/// <summary> /// Copy the current contents of this buffer to the named output. /// </summary> public virtual void WriteTo(IndexOutput other) { Flush(); long num1 = file.Length; long num2 = 0L; int num3 = 0; long num4; for (; num2 < num1; num2 = num4) { int length = 1024; num4 = num2 + length; if (num4 > num1) { length = (int)(num1 - num2); } other.WriteBytes(file.GetBuffer(num3++), length); } }
public virtual void TestOverflow() // memory hole { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow")); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER; } int blockBits = TestUtil.NextInt(Random(), 14, 28); int blockSize = 1 << blockBits; sbyte[] arr = new sbyte[TestUtil.NextInt(Random(), blockSize / 2, blockSize * 2)]; for (int i = 0; i < arr.Length; ++i) { arr[i] = (sbyte)i; } long numBytes = (1L << 31) + TestUtil.NextInt(Random(), 1, blockSize * 3); PagedBytes p = new PagedBytes(blockBits); IndexOutput @out = dir.CreateOutput("foo", IOContext.DEFAULT); for (long i = 0; i < numBytes;) { Assert.AreEqual(i, @out.FilePointer); int len = (int)Math.Min(arr.Length, numBytes - i); @out.WriteBytes(arr, len); i += len; } Assert.AreEqual(numBytes, @out.FilePointer); @out.Dispose(); IndexInput @in = dir.OpenInput("foo", IOContext.DEFAULT); p.Copy(@in, numBytes); PagedBytes.Reader reader = p.Freeze(Random().NextBoolean()); foreach (long offset in new long[] { 0L, int.MaxValue, numBytes - 1, TestUtil.NextLong(Random(), 1, numBytes - 2) }) { BytesRef b = new BytesRef(); reader.FillSlice(b, offset, 1); Assert.AreEqual(arr[(int)(offset % arr.Length)], b.Bytes[b.Offset]); } @in.Dispose(); dir.Dispose(); }
private void AddVarDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values) { field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_DEREF.Name); CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT); CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT); // deduplicate SortedSet <BytesRef> dictionary = new SortedSet <BytesRef>(); foreach (BytesRef v in values) { dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v)); } /* values */ long startPosition = data.FilePointer; long currentAddress = 0; Dictionary <BytesRef, long> valueToAddress = new Dictionary <BytesRef, long>(); foreach (BytesRef v in dictionary) { currentAddress = data.FilePointer - startPosition; valueToAddress[v] = currentAddress; WriteVShort(data, v.Length); data.WriteBytes(v.Bytes, v.Offset, v.Length); } /* ordinals */ long totalBytes = data.FilePointer - startPosition; index.WriteLong(totalBytes); int maxDoc = State.SegmentInfo.DocCount; PackedInts.Writer w = PackedInts.GetWriter(index, maxDoc, PackedInts.BitsRequired(currentAddress), PackedInts.DEFAULT); foreach (BytesRef v in values) { w.Add(valueToAddress[v == null ? new BytesRef() : v]); } w.Finish(); }
/// <summary> /// Add a new position & payload. </summary> public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset) { if (Debugging.AssertsEnabled) { Debugging.Assert(indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); } int delta = position - lastPosition; if (Debugging.AssertsEnabled) { Debugging.Assert(delta >= 0, "position={0} lastPosition={1}", position, lastPosition); // not quite right (if pos=0 is repeated twice we don't catch it) } lastPosition = position; if (storePayloads) { int payloadLength = payload == null ? 0 : payload.Length; if (payloadLength != lastPayloadLength) { lastPayloadLength = payloadLength; // TODO: explore whether we get better compression // by not storing payloadLength into prox stream? posOut.Write((delta << 1) | 1); posOut.Write(payloadLength); } else { posOut.Write(delta << 1); } if (payloadLength > 0) { payloadOut.WriteBytes(payload.Bytes, payload.Offset, payloadLength); } } else { posOut.Write(delta); } lastPosition = position; }
private void AddVarDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values) { field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_VAR_DEREF.ToString()); CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT); CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT); // deduplicate JCG.SortedSet <BytesRef> dictionary = new JCG.SortedSet <BytesRef>(); foreach (BytesRef v in values) { dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v)); } /* values */ long startPosition = data.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream long currentAddress = 0; Dictionary <BytesRef, long> valueToAddress = new Dictionary <BytesRef, long>(); foreach (BytesRef v in dictionary) { currentAddress = data.Position - startPosition; // LUCENENET specific: Renamed from getFilePointer() to match FileStream valueToAddress[v] = currentAddress; WriteVInt16(data, v.Length); data.WriteBytes(v.Bytes, v.Offset, v.Length); } /* ordinals */ long totalBytes = data.Position - startPosition; // LUCENENET specific: Renamed from getFilePointer() to match FileStream index.WriteInt64(totalBytes); int maxDoc = state.SegmentInfo.DocCount; PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc, PackedInt32s.BitsRequired(currentAddress), PackedInt32s.DEFAULT); foreach (BytesRef v in values) { w.Add(valueToAddress[v ?? new BytesRef()]); } w.Finish(); }
private int LastFieldNumber = -1; // only for assert public PreFlexRWNormsConsumer(Directory directory, string segment, IOContext context) { string normsFileName = IndexFileNames.SegmentFileName(segment, "", NORMS_EXTENSION); bool success = false; IndexOutput output = null; try { output = directory.CreateOutput(normsFileName, context); output.WriteBytes(NORMS_HEADER, 0, NORMS_HEADER.Length); @out = output; success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(output); } } }
public virtual void TestLargeWrites() { IndexOutput os = Dir.CreateOutput("testBufferStart.txt", NewIOContext(Random())); var largeBuf = new byte[2048]; for (int i = 0; i < largeBuf.Length; i++) { largeBuf[i] = (byte)unchecked((sbyte)(new Random(1).NextDouble() * 256)); } long currentPos = os.FilePointer; os.WriteBytes(largeBuf, largeBuf.Length); try { Assert.AreEqual(currentPos + largeBuf.Length, os.FilePointer); } finally { os.Dispose(); } }
/// <summary> /// Add a new position & payload </summary> public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset) { Debug.Assert(INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); int delta = position - LAST_POSITION; Debug.Assert(delta >= 0, "position=" + position + " lastPosition=" + LAST_POSITION); // not quite right (if pos=0 is repeated twice we don't catch it) LAST_POSITION = position; if (STORE_PAYLOADS) { int payloadLength = payload == null ? 0 : payload.Length; if (payloadLength != LAST_PAYLOAD_LENGTH) { LAST_PAYLOAD_LENGTH = payloadLength; // TODO: explore whether we get better compression // by not storing payloadLength into prox stream? POS_OUT.Write((delta << 1) | 1); POS_OUT.Write(payloadLength); } else { POS_OUT.Write(delta << 1); } if (payloadLength > 0 && payload != null) { PAYLOAD_OUT.WriteBytes(payload.Bytes, payload.Offset, payloadLength); } } else { POS_OUT.Write(delta); } LAST_POSITION = position; }
public virtual void TestLargeWrites() { IndexOutput os = dir.CreateOutput("testBufferStart.txt", NewIOContext(Random)); var largeBuf = new byte[2048]; for (int i = 0; i < largeBuf.Length; i++) { largeBuf[i] = (byte)(new Random(1).NextDouble() * 256); } long currentPos = os.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream os.WriteBytes(largeBuf, largeBuf.Length); try { Assert.AreEqual(currentPos + largeBuf.Length, os.Position); // LUCENENET specific: Renamed from getFilePointer() to match FileStream } finally { os.Dispose(); } }
public virtual void TestLargeWrites() { IndexOutput os = dir.CreateOutput("testBufferStart.txt", NewIOContext(Random)); var largeBuf = new byte[2048]; for (int i = 0; i < largeBuf.Length; i++) { largeBuf[i] = (byte)(Random.NextDouble() * 256); // LUCENENET: Using Random, since Math.random() doesn't exist in .NET, and it seems to make sense to make this repeatable. } long currentPos = os.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream os.WriteBytes(largeBuf, largeBuf.Length); try { Assert.AreEqual(currentPos + largeBuf.Length, os.Position); // LUCENENET specific: Renamed from getFilePointer() to match FileStream } finally { os.Dispose(); } }