public long WriteTo(DataOutput @out) { long size = 0; while (true) { if (limit + BufferOffset == EndIndex) { Debug.Assert(EndIndex - BufferOffset >= upto); @out.WriteBytes(buffer, upto, limit - upto); size += limit - upto; break; } else { @out.WriteBytes(buffer, upto, limit - upto); size += limit - upto; NextSlice(); } } return(size); }
protected void WriteValues(int bitsRequired) { PackedInt32s.IEncoder encoder = PackedInt32s.GetEncoder(PackedInt32s.Format.PACKED, PackedInt32s.VERSION_CURRENT, bitsRequired); int iterations = m_values.Length / encoder.ByteValueCount; int blockSize = encoder.ByteBlockCount * iterations; if (m_blocks == null || m_blocks.Length < blockSize) { m_blocks = new byte[blockSize]; } if (m_off < m_values.Length) { Arrays.Fill(m_values, m_off, m_values.Length, 0L); } encoder.Encode(m_values, 0, m_blocks, 0, iterations); int blockCount = (int)PackedInt32s.Format.PACKED.ByteCount(PackedInt32s.VERSION_CURRENT, m_off, bitsRequired); m_out.WriteBytes(m_blocks, blockCount); }
protected internal void WriteValues(int bitsRequired) { PackedInts.Encoder encoder = PackedInts.GetEncoder(PackedInts.Format.PACKED, PackedInts.VERSION_CURRENT, bitsRequired); int iterations = Values.Length / encoder.ByteValueCount(); int blockSize = encoder.ByteBlockCount() * iterations; if (Blocks == null || Blocks.Length < blockSize) { Blocks = new byte[blockSize]; } if (Off < Values.Length) { Arrays.Fill(Values, Off, Values.Length, 0L); } encoder.Encode(Values, 0, Blocks, 0, iterations); int blockCount = (int)PackedInts.Format.PACKED.ByteCount(PackedInts.VERSION_CURRENT, Off, bitsRequired); @out.WriteBytes(Blocks, blockCount); }
public virtual void AddValue(int docID, BytesRef value) { if (docID < addedValues) { throw new ArgumentOutOfRangeException(nameof(docID), "DocValuesField \"" + fieldInfo.Name + "\" appears more than once in this document (only one value is allowed per field)"); // LUCENENET specific - changed from IllegalArgumentException to ArgumentOutOfRangeException (.NET convention) } if (value is null) { throw new ArgumentNullException("field=\"" + fieldInfo.Name + "\": null value not allowed"); // LUCENENET specific - changed from IllegalArgumentException to ArgumentNullException (.NET convention) } if (value.Length > MAX_LENGTH) { throw new ArgumentException("DocValuesField \"" + fieldInfo.Name + "\" is too large, must be <= " + MAX_LENGTH); } // Fill in any holes: while (addedValues < docID) { addedValues++; lengths.Add(0); } addedValues++; lengths.Add(value.Length); try { bytesOut.WriteBytes(value.Bytes, value.Offset, value.Length); } catch (Exception ioe) when(ioe.IsIOException()) { // Should never happen! throw RuntimeException.Create(ioe); } docsWithField = FixedBitSet.EnsureCapacity(docsWithField, docID); docsWithField.Set(docID); UpdateBytesUsed(); }
public virtual void AddValue(int docID, BytesRef value) { if (docID < addedValues) { throw new System.ArgumentException("DocValuesField \"" + fieldInfo.Name + "\" appears more than once in this document (only one value is allowed per field)"); } if (value == null) { throw new System.ArgumentException("field=\"" + fieldInfo.Name + "\": null value not allowed"); } if (value.Length > MAX_LENGTH) { throw new System.ArgumentException("DocValuesField \"" + fieldInfo.Name + "\" is too large, must be <= " + MAX_LENGTH); } // Fill in any holes: while (addedValues < docID) { addedValues++; lengths.Add(0); } addedValues++; lengths.Add(value.Length); try { bytesOut.WriteBytes(value.Bytes, value.Offset, value.Length); } catch (System.IO.IOException ioe) { // Should never happen! throw new Exception(ioe.ToString(), ioe); } docsWithField = FixedBitSet.EnsureCapacity(docsWithField, docID); docsWithField.Set(docID); UpdateBytesUsed(); }
public override void Compress(byte[] bytes, int off, int len, DataOutput output) { byte[] resultArray = null; using (MemoryStream compressionMemoryStream = new MemoryStream()) { using (DeflateStream deflateStream = new DeflateStream(compressionMemoryStream, compressionLevel)) { deflateStream.Write(bytes, off, len); } resultArray = compressionMemoryStream.ToArray(); } if (resultArray.Length == 0) { Debug.Assert(len == 0, len.ToString()); output.WriteVInt32(0); return; } else { output.WriteVInt32(resultArray.Length); output.WriteBytes(resultArray, resultArray.Length); } }
public override void Write(BytesRef prefix, DataOutput @out) { Debug.Assert(prefix != null); @out.WriteVInt32(prefix.Length); @out.WriteBytes(prefix.Bytes, prefix.Offset, prefix.Length); }
/// <summary> /// Writes all of our bytes to the target <seealso cref="DataOutput"/>. </summary> public virtual void WriteTo(DataOutput @out) { foreach (byte[] block in Blocks) { @out.WriteBytes(block, 0, block.Length); } }
public virtual void TestDataInputOutput2() { Random random = Random(); for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++) { int blockBits = TestUtil.NextInt(random, 1, 20); int blockSize = 1 << blockBits; PagedBytes p = new PagedBytes(blockBits); DataOutput @out = p.DataOutput; int numBytes = Random().Next(10000000); byte[] answer = new byte[numBytes]; Random().NextBytes(answer); int written = 0; while (written < numBytes) { if (Random().Next(10) == 7) { @out.WriteByte(answer[written++]); } else { int chunk = Math.Min(Random().Next(1000), numBytes - written); @out.WriteBytes(answer, written, chunk); written += chunk; } } PagedBytes.Reader reader = p.Freeze(random.NextBoolean()); DataInput @in = p.DataInput; byte[] verify = new byte[numBytes]; int read = 0; while (read < numBytes) { if (Random().Next(10) == 7) { verify[read++] = @in.ReadByte(); } else { int chunk = Math.Min(Random().Next(1000), numBytes - read); @in.ReadBytes(verify, read, chunk); read += chunk; } } Assert.IsTrue(Arrays.Equals(answer, verify)); BytesRef slice = new BytesRef(); for (int iter2 = 0; iter2 < 100; iter2++) { int pos = random.Next(numBytes - 1); int len = random.Next(Math.Min(blockSize + 1, numBytes - pos)); reader.FillSlice(slice, pos, len); for (int byteUpto = 0; byteUpto < len; byteUpto++) { Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]); } } } }
public override void Compress(byte[] bytes, int off, int len, DataOutput output) { byte[] resultArray = null; using (MemoryStream compressionMemoryStream = new MemoryStream()) { using (DeflateStream deflateStream = new DeflateStream(compressionMemoryStream, compressionLevel)) { deflateStream.Write(bytes, off, len); } resultArray = compressionMemoryStream.ToArray(); } if (resultArray.Length == 0) { Debug.Assert(len == 0, len.ToString()); output.WriteVInt(0); return; } else { output.WriteVInt(resultArray.Length); output.WriteBytes(resultArray, resultArray.Length); } }
public override void Compress(sbyte[] bytes, int off, int len, DataOutput @out) { @out.WriteBytes(bytes, off, len); }
private static void EncodeLiterals(sbyte[] bytes, int token, int anchor, int literalLen, DataOutput @out) { @out.WriteByte((sbyte)token); // encode literal length if (literalLen >= 0x0F) { EncodeLen(literalLen - 0x0F, @out); } // encode literals @out.WriteBytes(bytes, anchor, literalLen); }
public void ToData(DataOutput output) { output.WriteBytes(arr); output.WriteObject(name); output.WriteInt32(identifire); }
public override void Compress(sbyte[] bytes, int off, int len, DataOutput @out) { Compressor.Reset(); Compressor.SetInput((byte[])(Array)bytes, off, len); Compressor.Finish(); if (Compressor.NeedsInput) { // no output Debug.Assert(len == 0, len.ToString()); @out.WriteVInt(0); return; } int totalCount = 0; for (; ; ) { int count = Compressor.Deflate(Compressed, totalCount, Compressed.Length - totalCount); totalCount += count; Debug.Assert(totalCount <= Compressed.Length); if (Compressor.IsFinished) { break; } else { Compressed = ArrayUtil.Grow(Compressed); } } @out.WriteVInt(totalCount); @out.WriteBytes(Compressed, totalCount); }
public override void EncodeTerm(long[] empty, DataOutput output, FieldInfo fieldInfo, BlockTermState state, bool abs) { var _state = (PulsingTermState) state; Debug.Assert(empty.Length == 0); _absolute = _absolute || abs; if (_state.BYTES == null) { _wrappedPostingsWriter.EncodeTerm(_longs, _buffer, fieldInfo, _state.WRAPPED_STATE, _absolute); for (var i = 0; i < _longsSize; i++) { output.WriteVLong(_longs[i]); } _buffer.WriteTo(output); _buffer.Reset(); _absolute = false; } else { output.WriteVInt(_state.BYTES.Length); output.WriteBytes(_state.BYTES, 0, _state.BYTES.Length); _absolute = _absolute || abs; } }