public void BufferedReaderWriter_Basics()
{
Random r = new Random(5);
byte[] content = new byte[1024];
r.NextBytes(content);
byte[] result = new byte[128];
using (BufferedReader reader = BufferedReader.FromArray(content, 0, content.Length))
using (BufferedWriter writer = BufferedWriter.ToArray(content))
{
while (!reader.EndOfStream)
{
int length = reader.EnsureSpace(150);
writer.EnsureSpace(length);
Buffer.BlockCopy(reader.Buffer, reader.Index, writer.Buffer, writer.Index, length);
reader.Index += length;
writer.Index += length;
}
result = writer.Buffer;
}
Assert.IsTrue(result.Length >= content.Length);
for (int i = 0; i < content.Length; ++i)
{
Assert.AreEqual(content[i], result[i], $"@{i:n0}, expect: {content[i]}, actual: {result[i]}");
}
}
private void WriteStringLength(BionToken marker, int stringLength)
{
int lengthOfLength;
int markerAdjustment;
if (stringLength <= MaxOneByteLength)
{
lengthOfLength = 1;
markerAdjustment = -2;
}
else if (stringLength <= MaxTwoByteLength)
{
lengthOfLength = 2;
markerAdjustment = -1;
}
else
{
lengthOfLength = 5;
markerAdjustment = 0;
}
_writer.EnsureSpace(lengthOfLength + 1);
int index = _writer.Index++;
NumberConverter.WriteSevenBitFixed(_writer, (ulong)stringLength, lengthOfLength);
_writer.Buffer[index] = (byte)((int)marker + markerAdjustment);
}
private void WriteBlock(int[] array, int index, int endIndex)
{
int length = endIndex - index;
if (length == 0)
{
return;
}
// Choose how to encode
IntBlock block = IntBlock.Plan(array, index, endIndex);
Stats?.Add(block);
// Write the block markers
block.Write(_writer);
_writer.EnsureSpace(block.TotalBytes);
// Write adjustment bits
if (block.BitsPerAdjustment > 0)
{
int bitsLeftToWrite = 0;
ulong toWrite = 0;
for (int i = 0; i < block.AdjustmentCount; ++i)
{
// Calculate adjustment (zero once out of real values)
ulong adjustment = unchecked ((uint)(i >= block.Count ? 0 : array[index + i] - (block.Base + block.Slope * i)));
// Add new value to bits left to write at top of long
toWrite += adjustment << (64 - block.BitsPerAdjustment - bitsLeftToWrite);
// Write all whole bytes built
bitsLeftToWrite += block.BitsPerAdjustment;
while (bitsLeftToWrite >= 8)
{
// Write top byte
byte nextByte = (byte)(toWrite >> 56);
_writer.Buffer[_writer.Index++] = nextByte;
// Shift to next byte
toWrite = toWrite << 8;
bitsLeftToWrite -= 8;
}
}
}
// Clear buffer; it's been written
_bufferCount = 0;
}
public void RewriteOptimized(BufferedWriter writer, string containerIndexPath = null, string searchIndexPath = null)
{
int[] map = _compressor.OptimizeIndex();
using (BufferedReader inner = BufferedReader.FromArray(_reader.Buffer, 0, 0))
using (ContainerIndex containerIndex = (containerIndexPath == null ? null : ContainerIndex.OpenWrite(containerIndexPath)))
using (SearchIndexWriter indexWriter = (searchIndexPath == null ? null : new SearchIndexWriter(searchIndexPath, map.Length, 128 * 1024)))
{
long last = 0;
while (this.Read())
{
int length = (int)(this.BytesRead - last);
writer.EnsureSpace(length);
if (LengthLookup[(byte)_currentMarker] >= 0)
{
// Everything but compressed text: write bytes out
Buffer.BlockCopy(_reader.Buffer, _reader.Index - length, writer.Buffer, writer.Index, length);
writer.Index += length;
}
else
{
writer.Buffer[writer.Index++] = (byte)_currentMarker;
// Compressed Test: Rewrite the text segment
inner.ReadSlice(_reader.Buffer, _reader.Index - _currentLength, _reader.Index - 1);
_compressor.RewriteOptimized(map, inner, writer, indexWriter);
writer.Buffer[writer.Index++] = (byte)BionMarker.EndValue;
}
if ((byte)_currentMarker >= (byte)BionMarker.EndArray)
{
if ((byte)_currentMarker >= (byte)BionMarker.StartArray)
{
containerIndex?.Start(writer.BytesWritten - 1);
}
else
{
containerIndex?.End(writer.BytesWritten);
}
}
last = this.BytesRead;
}
}
}
public void Expand(BufferedReader reader, BufferedWriter writer)
{
while (!reader.EndOfStream)
{
int count = NumberConverter.ReadSixBitTerminatedBlock(reader, _block);
for (int i = 0; i < count; ++i)
{
int wordIndex = (int)_block[i];
String8 word = _words[wordIndex].Word;
writer.EnsureSpace(word.Length);
word.CopyTo(writer.Buffer, writer.Index);
writer.Index += word.Length;
}
}
}
public void Write(BufferedWriter writer)
{
writer.EnsureSpace(this.TotalBytes);
// Write count (if needed)
if (this.Count != IntBlock.BlockSize)
{
writer.Buffer[writer.Index++] = CountMarker;
writer.Buffer[writer.Index++] = this.Count;
}
// Write base, if needed
WriteComponent(writer, this.Base, BaseMarker);
// Write slope, if needed
WriteComponent(writer, this.Slope, SlopeMarker);
// Write adjustment marker
writer.Buffer[writer.Index++] = (byte)(AdjustmentMarker + this.BitsPerAdjustment);
}
