public void BufferedReaderWriter_Basics() { Random r = new Random(5); byte[] content = new byte[1024]; r.NextBytes(content); byte[] result = new byte[128]; using (BufferedReader reader = BufferedReader.FromArray(content, 0, content.Length)) using (BufferedWriter writer = BufferedWriter.ToArray(content)) { while (!reader.EndOfStream) { int length = reader.EnsureSpace(150); writer.EnsureSpace(length); Buffer.BlockCopy(reader.Buffer, reader.Index, writer.Buffer, writer.Index, length); reader.Index += length; writer.Index += length; } result = writer.Buffer; } Assert.IsTrue(result.Length >= content.Length); for (int i = 0; i < content.Length; ++i) { Assert.AreEqual(content[i], result[i], $"@{i:n0}, expect: {content[i]}, actual: {result[i]}"); } }
private void WriteStringLength(BionToken marker, int stringLength) { int lengthOfLength; int markerAdjustment; if (stringLength <= MaxOneByteLength) { lengthOfLength = 1; markerAdjustment = -2; } else if (stringLength <= MaxTwoByteLength) { lengthOfLength = 2; markerAdjustment = -1; } else { lengthOfLength = 5; markerAdjustment = 0; } _writer.EnsureSpace(lengthOfLength + 1); int index = _writer.Index++; NumberConverter.WriteSevenBitFixed(_writer, (ulong)stringLength, lengthOfLength); _writer.Buffer[index] = (byte)((int)marker + markerAdjustment); }
private void WriteBlock(int[] array, int index, int endIndex) { int length = endIndex - index; if (length == 0) { return; } // Choose how to encode IntBlock block = IntBlock.Plan(array, index, endIndex); Stats?.Add(block); // Write the block markers block.Write(_writer); _writer.EnsureSpace(block.TotalBytes); // Write adjustment bits if (block.BitsPerAdjustment > 0) { int bitsLeftToWrite = 0; ulong toWrite = 0; for (int i = 0; i < block.AdjustmentCount; ++i) { // Calculate adjustment (zero once out of real values) ulong adjustment = unchecked ((uint)(i >= block.Count ? 0 : array[index + i] - (block.Base + block.Slope * i))); // Add new value to bits left to write at top of long toWrite += adjustment << (64 - block.BitsPerAdjustment - bitsLeftToWrite); // Write all whole bytes built bitsLeftToWrite += block.BitsPerAdjustment; while (bitsLeftToWrite >= 8) { // Write top byte byte nextByte = (byte)(toWrite >> 56); _writer.Buffer[_writer.Index++] = nextByte; // Shift to next byte toWrite = toWrite << 8; bitsLeftToWrite -= 8; } } } // Clear buffer; it's been written _bufferCount = 0; }
public void RewriteOptimized(BufferedWriter writer, string containerIndexPath = null, string searchIndexPath = null) { int[] map = _compressor.OptimizeIndex(); using (BufferedReader inner = BufferedReader.FromArray(_reader.Buffer, 0, 0)) using (ContainerIndex containerIndex = (containerIndexPath == null ? null : ContainerIndex.OpenWrite(containerIndexPath))) using (SearchIndexWriter indexWriter = (searchIndexPath == null ? null : new SearchIndexWriter(searchIndexPath, map.Length, 128 * 1024))) { long last = 0; while (this.Read()) { int length = (int)(this.BytesRead - last); writer.EnsureSpace(length); if (LengthLookup[(byte)_currentMarker] >= 0) { // Everything but compressed text: write bytes out Buffer.BlockCopy(_reader.Buffer, _reader.Index - length, writer.Buffer, writer.Index, length); writer.Index += length; } else { writer.Buffer[writer.Index++] = (byte)_currentMarker; // Compressed Test: Rewrite the text segment inner.ReadSlice(_reader.Buffer, _reader.Index - _currentLength, _reader.Index - 1); _compressor.RewriteOptimized(map, inner, writer, indexWriter); writer.Buffer[writer.Index++] = (byte)BionMarker.EndValue; } if ((byte)_currentMarker >= (byte)BionMarker.EndArray) { if ((byte)_currentMarker >= (byte)BionMarker.StartArray) { containerIndex?.Start(writer.BytesWritten - 1); } else { containerIndex?.End(writer.BytesWritten); } } last = this.BytesRead; } } }
public void Expand(BufferedReader reader, BufferedWriter writer) { while (!reader.EndOfStream) { int count = NumberConverter.ReadSixBitTerminatedBlock(reader, _block); for (int i = 0; i < count; ++i) { int wordIndex = (int)_block[i]; String8 word = _words[wordIndex].Word; writer.EnsureSpace(word.Length); word.CopyTo(writer.Buffer, writer.Index); writer.Index += word.Length; } } }
public void Write(BufferedWriter writer) { writer.EnsureSpace(this.TotalBytes); // Write count (if needed) if (this.Count != IntBlock.BlockSize) { writer.Buffer[writer.Index++] = CountMarker; writer.Buffer[writer.Index++] = this.Count; } // Write base, if needed WriteComponent(writer, this.Base, BaseMarker); // Write slope, if needed WriteComponent(writer, this.Slope, SlopeMarker); // Write adjustment marker writer.Buffer[writer.Index++] = (byte)(AdjustmentMarker + this.BitsPerAdjustment); }