public bool Read()
{
_reader.EnsureSpace(20);
// Check for end (after reading one thing at the root depth)
if (_reader.EndOfStream)
{
TokenType = BionToken.None;
return(false);
}
// Read the marker
byte marker = _reader.Buffer[_reader.Index++];
// Identify the token type and length
_currentMarker = (BionMarker)marker;
_currentLength = LengthLookup[marker];
TokenType = TokenLookup[marker];
if (_currentLength < 0)
{
_currentLength = LengthIncludingTerminator();
_currentDecodedString = null;
_currentCompressedStringStart = _reader.Index;
_reader.Index += _currentLength;
}
else if (_currentLength > 0)
{
// Read value (non-string) or length (string)
_currentDecodedNumber = NumberConverter.ReadSevenBitExplicit(_reader, _currentLength);
// Read value (string)
if (marker >= 0xF0)
{
_currentCompressedStringStart = -1;
_currentDecodedString = null;
_currentLength = (int)_currentDecodedNumber;
_reader.EnsureSpace(_currentLength);
_currentString = String8.Reference(_reader.Buffer, _reader.Index, _currentLength);
_reader.Index += _currentLength;
}
}
else
{
// Adjust depth (length 0 tokens only)
_currentDepth += DepthLookup[marker];
}
if (TokenType == BionToken.None)
{
throw new BionSyntaxException($"Invalid Bion Token 0x{_currentMarker:X2} @{BytesRead:n0}.");
}
return(true);
}
public void BufferedReaderWriter_Basics()
{
Random r = new Random(5);
byte[] content = new byte[1024];
r.NextBytes(content);
byte[] result = new byte[128];
using (BufferedReader reader = BufferedReader.FromArray(content, 0, content.Length))
using (BufferedWriter writer = BufferedWriter.ToArray(content))
{
while (!reader.EndOfStream)
{
int length = reader.EnsureSpace(150);
writer.EnsureSpace(length);
Buffer.BlockCopy(reader.Buffer, reader.Index, writer.Buffer, writer.Index, length);
reader.Index += length;
writer.Index += length;
}
result = writer.Buffer;
}
Assert.IsTrue(result.Length >= content.Length);
for (int i = 0; i < content.Length; ++i)
{
Assert.AreEqual(content[i], result[i], $"@{i:n0}, expect: {content[i]}, actual: {result[i]}");
}
}
public static void ReadBufferedReader(string filePath, int bufferSizeBytes)
{
byte[] buffer = new byte[bufferSizeBytes];
long totalSize = 0;
using (new ConsoleWatch($"ReadBufferedReader(\"{filePath}\", {bufferSizeBytes})", () => $"Done; {totalSize:n0} bytes"))
{
using (BufferedReader reader = new BufferedReader(File.OpenRead(filePath), buffer))
{
while (!reader.EndOfStream)
{
reader.EnsureSpace(buffer.Length);
reader.Index = reader.Length;
}
totalSize = reader.BytesRead;
}
}
}
private static string SplitAndJoin(string content)
{
StringBuilder result = new StringBuilder();
byte[] buffer = new byte[content.Length];
using (BufferedReader reader = BufferedReader.FromString(content, ref buffer))
{
bool isWord = WordSplitter.IsLetterOrDigit(reader.Buffer[reader.Index]);
int length = 0;
while (!reader.EndOfStream)
{
// Read the next word
length = WordSplitter.NextWordLength(reader, isWord);
String8 word = String8.Reference(reader.Buffer, reader.Index, length);
// Set state to read next word
reader.Index += length;
isWord = !isWord;
if (reader.Index < reader.Length || reader.EndOfStream)
{
// If this is word is definitely complete, write it
if (result.Length > 0)
{
result.Append("|");
}
result.Append(word.ToString());
}
else if (!reader.EndOfStream)
{
// Reset state to re-read this word
reader.Index -= length;
isWord = !isWord;
// If end of buffer but not stream, request more
reader.EnsureSpace(length * 2);
}
}
}
return(result.ToString());
}
public int Page(ref long[] matches)
{
int lengthLeft = (int)(IndexEndOffset - CurrentOffset);
int lengthToGet = Math.Min(lengthLeft, matches.Length * 10);
// Read the match bytes
Reader.Seek(CurrentOffset, SeekOrigin.Begin);
Reader.EnsureSpace(lengthToGet, lengthLeft);
// Decode to non-relative longs
int count = 0;
while (Reader.BytesRead < IndexEndOffset && count < matches.Length)
{
long value = LastValue + (long)NumberConverter.ReadSevenBitTerminated(Reader);
matches[count++] = value << SearchIndexWriter.Shift;
LastValue = value;
}
CurrentOffset = Reader.BytesRead;
return(count);
}
public void Compress(BufferedReader reader, BufferedWriter writer)
{
if (reader.EndOfStream)
{
return;
}
bool isWord = WordSplitter.IsLetterOrDigit(reader.Buffer[reader.Index]);
int length = 0;
while (!reader.EndOfStream)
{
// Read the next word
length = WordSplitter.NextWordLength(reader, isWord);
String8 word = String8.Reference(reader.Buffer, reader.Index, length);
// Set state to read next word
reader.Index += length;
isWord = !isWord;
if (reader.Index < reader.Length || reader.EndOfStream)
{
// If this is word is definitely complete, write it
int wordIndex = _words.FindOrAdd(word);
NumberConverter.WriteSixBitTerminated(writer, (ulong)wordIndex);
}
else if (!reader.EndOfStream)
{
// Reset state to re-read this word
reader.Index -= length;
isWord = !isWord;
// If end of buffer but not stream, request more
reader.EnsureSpace(length * 2);
}
}
}
public static IntBlock Read(BufferedReader reader)
{
// Read enough for a max length block
reader.EnsureSpace(513);
// Component defaults
byte count = IntBlock.BlockSize;
int baseV = 0;
int slope = 0;
// Read components
byte marker = reader.Buffer[reader.Index++];
while (marker < IntBlock.AdjustmentMarker)
{
if (marker == IntBlock.CountMarker)
{
count = reader.Buffer[reader.Index++];
}
else if (marker < IntBlock.SlopeMarker)
{
baseV = ReadComponent(reader, marker, IntBlock.BaseMarker);
}
else
{
slope = ReadComponent(reader, marker, IntBlock.SlopeMarker);
}
marker = reader.Buffer[reader.Index++];
}
byte adjustmentMarker = marker;
byte adjustmentBitLength = (byte)(marker - IntBlock.AdjustmentMarker);
return(new IntBlock(count, baseV, slope, adjustmentBitLength));
}