private ulong NextNumber()
{
if (indxInBuffer >= dataInBuffer)
{
var toRead = buffer.Length;
if (listEndOffset - readOffset < toRead)
{
checked
{
toRead = (int)(listEndOffset - readOffset);
}
}
if (toRead == 0)
{
isEof = true;
return(0);
}
persistentStorage.ReadAll(readOffset, buffer, 0, toRead);
for (int i = toRead; i < buffer.Length; ++i)
{
buffer[i] = 0;
}
readOffset += toRead;
dataInBuffer = toRead;
indxInBuffer = 0;
}
indxInBuffer += VarInt.ReadVUInt64(buffer, indxInBuffer, out var result);
return(result);
}
public TextReader GetText(long textStart)
{
var result = new StringBuilder();
var buffer = new byte[4096];
var offset = textStart;
var chunkSize = 0;
persistentStorage.ReadAll(offset, buffer, 0, sizeof(int));
offset += sizeof(int);
chunkSize = BitConverter.ToInt32(buffer, 0);
while (chunkSize > 0)
{
if (buffer.Length < chunkSize)
{
buffer = new byte[4096 * ((chunkSize + 4095) / 4096)];
}
persistentStorage.ReadAll(offset, buffer, 0, chunkSize);
offset += chunkSize;
result.Append(Encoding.UTF8.GetString(buffer, 0, chunkSize));
persistentStorage.ReadAll(offset, buffer, 0, sizeof(int));
offset += sizeof(int);
chunkSize = BitConverter.ToInt32(buffer, 0);
}
return(new StringReader(result.ToString()));
}
private bool EnsureBuffer(int size)
{
if (isEof)
{
return(size <= (dataInBuffer - indxInBuffer));
}
if (size > buffer.Length)
{
throw new Exception("Resize buffer");
}
if (indxInBuffer + size >= dataInBuffer)
{
Array.Copy(buffer, indxInBuffer, buffer, 0, dataInBuffer - indxInBuffer);
dataInBuffer = dataInBuffer - indxInBuffer;
indxInBuffer = 0;
var toRead = (int)Math.Min(buffer.Length - dataInBuffer, (listEndOffset - readOffset));
if (toRead == 0)
{
isEof = true;
}
else
{
persistentStorage.ReadAll(readOffset, buffer, dataInBuffer, toRead);
readOffset += toRead;
dataInBuffer += toRead;
}
return(size <= dataInBuffer);
}
return(true);
}
// TODO: Need unit test for this
public IPostingList GetBasic(PostingListAddress address)
{
var offset = address.Offset;
var buffer = new byte[sizeof(long) + sizeof(int)];
var occurrences = new List <Occurrence>();
while (true)
{
persistentStorage.ReadAll(offset, buffer, 0, buffer.Length);
long continuationOffset = BitConverter.ToInt64(buffer, 0);
int length = BitConverter.ToInt32(buffer, sizeof(long));
var dataBuffer = new byte[length];
persistentStorage.ReadAll(offset + sizeof(long) + sizeof(int), dataBuffer, 0, dataBuffer.Length);
ParseBufferTo(dataBuffer, occurrences);
if (continuationOffset == 0)
{
break;
}
else
{
offset = continuationOffset;
}
}
return(new PostingListArray(occurrences.ToArray()));
}
private (long indexOffset, long dataOffset, int dataSize) FindKeyRecord(TKey key)
{
if (key == null)
{
throw new ArgumentNullException(nameof(key));
}
var index = (long)((ulong)GetKeyHash(key) % (ulong)capacity);
var offset = headerSize + index * IndexRecordSize;
dataStorage.ReadAll(offset, valueBuffer, 0, IndexRecordSize);
var dataOffset = BitConverter.ToInt64(valueBuffer, 0);
var dataSize = BitConverter.ToInt32(valueBuffer, sizeof(long));
return(offset, dataOffset, dataSize);
}
public void ReadHeader(IPersistentStorage persistentStorage)
{
persistentStorage.ReadAll(0, headerData, 0, headerData.Length);
if (Text != HeaderText)
{
throw new InvalidOperationException("Header text mismatch");
}
}
public TValue this[long index]
{
get
{
var offset = linearIndex[index];
if (offset == 0)
{
throw new ArgumentOutOfRangeException(nameof(index));
}
dataStorage.ReadAll(offset, sizeBuffer, 0, sizeBuffer.Length);
int dataSize = BitConverter.ToInt32(sizeBuffer, 0);
//TODO: Improve serializer, no need to reallocate
if (valueBuffer == null || valueBuffer.Length != dataSize)
{
valueBuffer = new byte[dataSize];
}
dataStorage.ReadAll(offset + sizeBuffer.Length, valueBuffer, 0, valueBuffer.Length);
return(valueSerializer.GetValue(valueBuffer));
}
set
{
var offset = linearIndex[index];
if (offset == 0)
{
offset = dataStorage.Length;
linearIndex[index] = dataStorage.Length;
}
var data = valueSerializer.GetBytes(value);
var dataSize = BitConverter.GetBytes(data.Length);
dataStorage.WriteAll(offset, dataSize, 0, dataSize.Length);
dataStorage.WriteAll(offset + dataSize.Length, data, 0, data.Length);
}
}
public PersistentDictionaryFst(IPersistentStorage storage,
int maxTokenLength,
ITextEncoding encoding)
{
this.maxTokenByteLength = encoding.GetMaxEncodedLength(maxTokenLength);
this.storage = storage;
this.encoding = encoding;
if (storage.Length > 0)
{
var buffer = new byte[storage.Length];
storage.ReadAll(0, buffer, 0, buffer.Length);
this.fst = FST <int> .FromBytesCompressed(buffer, FSTVarIntOutput.Instance);
}
}
public void UpdateNextList(PostingListAddress address, PostingListAddress nextList)
{
var buffer = new byte[sizeof(long)];
var offset = address.Offset;
while (true)
{
persistentStorage.ReadAll(offset, buffer, 0, buffer.Length);
long continuationOffset = BitConverter.ToInt64(buffer, 0);
if (continuationOffset == 0)
{
persistentStorage.WriteAll(offset, BitConverter.GetBytes(nextList.Offset), 0, sizeof(long));
break;
}
else
{
offset = continuationOffset;
}
}
}
public void ReadAll(long fileOffset, byte[] buffer, int offset, int count)
{
instance.ReadAll(fileOffset, buffer, offset, count);
}
