public Buffer(Lmdb.BufferDto dto, string folder)
{
_fileName = dto.GetBufferFileName();
FullPath = Path.Combine(folder, _fileName);
_startPos = dto.GetBufferStartPos();
_maxBytes = dto.GetBufferMaxBytes();
_pos = dto.GetBufferPos();
_records = dto.GetBufferRecords();
}
public void ReadAll(Action <ReadPos, ArraySegment <byte>, BoundedStream> handler, long offset = 0, long recordCount = long.MaxValue)
{
if (ByteSize == 0)
{
return;
}
// interested in all _chunks, where offset comes before the end
var filteredChunks = _chunks.Where(c => offset < (c.GetChunkStartPos() + c.GetUncompressedByteSize())).ToList();
var keyBuffer = new byte[_maxKeySize];
//Console.WriteLine("{0} _chunks to read", filteredChunks.Count);
if (filteredChunks.Any())
{
var buffer = new byte[filteredChunks.Max(c => c.GetUncompressedByteSize())];
foreach (var chunk in filteredChunks)
{
var path = Path.Combine(_folder, chunk.GetChunkFileName());
var bufferSize = 16 * 1024;
// allocate all at once
using (var mem = new MemoryWithDisposeShield(buffer)) {
using (var f = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite,
bufferSize)) {
using (var aes = new AesCryptoServiceProvider()) {
var iv = new byte[16];
f.Read(iv, 0, 16);
aes.IV = iv;
aes.Key = _key;
aes.Mode = CipherMode.CBC;
aes.Padding = PaddingMode.PKCS7;
using (var decryptor = aes.CreateDecryptor())
using (var crypto = new CryptoStream(f, decryptor, CryptoStreamMode.Read))
using (var lz = new LZ4Stream(crypto, CompressionMode.Decompress)) {
// uncompress all in one go
lz.CopyTo(mem);
}
}
}
// rewind
mem.Seek(0, SeekOrigin.Begin);
var localPos = 0;
// fast forward seek bytes
if (offset > chunk.GetChunkStartPos())
{
var seek = (int)(offset - chunk.GetChunkStartPos());
mem.Seek(seek, SeekOrigin.Current);
localPos += seek;
}
while (localPos < chunk.GetUncompressedByteSize())
{
// compute absolute position
var pos = new ReadPos(localPos + chunk.GetChunkStartPos());
// read key
var keySize = ReadKeySize(mem, ref localPos);
mem.Read(keyBuffer, 0, keySize);
localPos += keySize;
var keySegment = new ArraySegment <byte>(keyBuffer, 0, keySize);
// read value
var valueSize = ReadValueSize(mem, ref localPos);
using (var b = new BoundedStream(mem, valueSize)) {
handler(pos, keySegment, b);
}
localPos += valueSize;
if (--recordCount == 0)
{
return;
}
}
}
}
}
if (_buffer == null)
{
return;
}
var bufferStartPos = _buffer.GetBufferStartPos();
//Console.WriteLine("Tail starts {0}, {1} offset (or {2})", bufferStartPos, offset, offset - bufferStartPos);
var bufferPath = Path.Combine(_folder, _buffer.GetBufferFileName());
using (var f = File.OpenRead(bufferPath)) {
var localPos = 0;
if (offset > bufferStartPos)
{
var seek = (int)(offset - bufferStartPos);
if (seek < 0)
{
var msg = string.Format("Tail starts at {0} trying to read at offset {1}", bufferStartPos, offset);
throw new IOException(msg);
}
f.Seek(seek, SeekOrigin.Begin);
localPos += seek;
}
var bytesInBuffer = _buffer.GetBufferPos();
while (localPos < bytesInBuffer)
{
// record position
var rp = new ReadPos(localPos + bufferStartPos);
// read key
var keySize = ReadKeySize(f, ref localPos);
f.Read(keyBuffer, 0, keySize);
localPos += keySize;
var keySegment = new ArraySegment <byte>(keyBuffer, 0, keySize);
// read value
int valueSize = ReadValueSize(f, ref localPos);
using (var b = new BoundedStream(f, valueSize)) {
handler(rp, keySegment, b);
}
localPos += valueSize;
if (--recordCount == 0)
{
return;
}
}
}
}
