public void FlushBuffer()
{
var encodedSize = GroupVarint.Encode(buffer, 0, bufferIndex, flushBuffer, 0);
// Write data
persistentStorage.WriteAll(persistentStorage.Length, flushBuffer, 0, encodedSize);
totalSize += encodedSize;
bufferIndex = 0;
}
public PostingListAddress EndList()
{
if (bufferIndex > 0)
{
buffer[deltaSelectorIndex] = deltaSelector;
var encodedSize = GroupVarint.Encode(buffer, 0, bufferIndex, flushBuffer, 0);
persistentStorage.WriteAll(persistentStorage.Length, flushBuffer, 0, encodedSize);
totalSize += encodedSize;
}
// Write length of the list
persistentStorage.WriteAll(listStart + sizeof(long), BitConverter.GetBytes(totalSize), 0, sizeof(int));
var listEnd = persistentStorage.Length;
if (listEnd - listStart != totalSize + sizeof(long) + sizeof(int))
{
throw new InvalidOperationException();
}
return(new PostingListAddress(listStart));
}
public void AddOccurrence(Occurrence occurrence)
{
if (first)
{
checked
{
buffer[bufferIndex++] = (int)occurrence.DocumentId;
buffer[bufferIndex++] = (int)occurrence.FieldId;
buffer[bufferIndex++] = (int)occurrence.TokenId;
}
previous = occurrence;
first = false;
deltaSelectorIndex = bufferIndex;
bufferIndex++;
}
else
{
int n;
if (previous.DocumentId == occurrence.DocumentId)
{
if (previous.FieldId == occurrence.FieldId)
{
n = 1;
checked
{
buffer[bufferIndex++] = (int)occurrence.TokenId - (int)previous.TokenId;
}
// NOTE: Removed zero value as it will lead to extra trailing occurrences
// because the last deltaSelector might have unsed bits,
// i.e. when deltaSelectorOffset < 32
// if (previous.TokenId == occurrence.TokenId)
// {
// n = 0;
// }
// else
// {
// n = 1;
// checked
// {
// buffer[bufferIndex++] = (int)occurrence.TokenId - (int)previous.TokenId;
// }
// }
}
else
{
n = 2;
checked
{
buffer[bufferIndex++] = (int)occurrence.FieldId - (int)previous.FieldId;
buffer[bufferIndex++] = (int)occurrence.TokenId;
}
}
}
else
{
n = 3;
checked
{
buffer[bufferIndex++] = (int)occurrence.DocumentId - (int)previous.DocumentId;
buffer[bufferIndex++] = (int)occurrence.FieldId;
buffer[bufferIndex++] = (int)occurrence.TokenId;
}
}
previous = occurrence;
deltaSelector |= (n << deltaSelectorOffset);
deltaSelectorOffset += 2;
if (deltaSelectorOffset == 32)
{
buffer[deltaSelectorIndex] = deltaSelector;
deltaSelector = 0;
deltaSelectorOffset = 0;
if (remainingBlocks == 1)
{
// Write data
var toKeep = bufferIndex % 4;
var toEncode = bufferIndex - toKeep;
var encodedSize = GroupVarint.Encode(buffer, 0, toEncode, flushBuffer, 0);
persistentStorage.WriteAll(persistentStorage.Length, flushBuffer, 0, encodedSize);
totalSize += encodedSize;
// Copy not encoded bytes (0,1,2,3)
Array.Copy(buffer, bufferIndex - toKeep, buffer, 0, toKeep);
bufferIndex = toKeep;
deltaSelectorIndex = bufferIndex;
bufferIndex++;
remainingBlocks = BlocksInMemory;
}
else
{
// Reserve space for delta selector
deltaSelectorIndex = bufferIndex;
bufferIndex++;
remainingBlocks--;
}
}
}
}
