public static int NextWordLength(BufferedReader reader, bool isWord)
{
int length = 1;
while (reader.Index + length < reader.Length && WordSplitter.IsLetterOrDigit(reader.Buffer[reader.Index + length]) == isWord)
{
length++;
}
return(length);
}
public void Compress(BufferedReader reader, BufferedWriter writer)
{
if (reader.EndOfStream)
{
return;
}
bool isWord = WordSplitter.IsLetterOrDigit(reader.Buffer[reader.Index]);
int length = 0;
while (!reader.EndOfStream)
{
// Read the next word
length = WordSplitter.NextWordLength(reader, isWord);
String8 word = String8.Reference(reader.Buffer, reader.Index, length);
// Set state to read next word
reader.Index += length;
isWord = !isWord;
if (reader.Index < reader.Length || reader.EndOfStream)
{
// If this is word is definitely complete, write it
int wordIndex = _words.FindOrAdd(word);
NumberConverter.WriteSixBitTerminated(writer, (ulong)wordIndex);
}
else if (!reader.EndOfStream)
{
// Reset state to re-read this word
reader.Index -= length;
isWord = !isWord;
// If end of buffer but not stream, request more
reader.EnsureSpace(length * 2);
}
}
}