public Buffer(Lmdb.BufferDto dto, string folder) { _fileName = dto.GetBufferFileName(); FullPath = Path.Combine(folder, _fileName); _startPos = dto.GetBufferStartPos(); _maxBytes = dto.GetBufferMaxBytes(); _pos = dto.GetBufferPos(); _records = dto.GetBufferRecords(); }
public void ReadAll(Action <ReadPos, ArraySegment <byte>, BoundedStream> handler, long offset = 0, long recordCount = long.MaxValue) { if (ByteSize == 0) { return; } // interested in all _chunks, where offset comes before the end var filteredChunks = _chunks.Where(c => offset < (c.GetChunkStartPos() + c.GetUncompressedByteSize())).ToList(); var keyBuffer = new byte[_maxKeySize]; //Console.WriteLine("{0} _chunks to read", filteredChunks.Count); if (filteredChunks.Any()) { var buffer = new byte[filteredChunks.Max(c => c.GetUncompressedByteSize())]; foreach (var chunk in filteredChunks) { var path = Path.Combine(_folder, chunk.GetChunkFileName()); var bufferSize = 16 * 1024; // allocate all at once using (var mem = new MemoryWithDisposeShield(buffer)) { using (var f = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, bufferSize)) { using (var aes = new AesCryptoServiceProvider()) { var iv = new byte[16]; f.Read(iv, 0, 16); aes.IV = iv; aes.Key = _key; aes.Mode = CipherMode.CBC; aes.Padding = PaddingMode.PKCS7; using (var decryptor = aes.CreateDecryptor()) using (var crypto = new CryptoStream(f, decryptor, CryptoStreamMode.Read)) using (var lz = new LZ4Stream(crypto, CompressionMode.Decompress)) { // uncompress all in one go lz.CopyTo(mem); } } } // rewind mem.Seek(0, SeekOrigin.Begin); var localPos = 0; // fast forward seek bytes if (offset > chunk.GetChunkStartPos()) { var seek = (int)(offset - chunk.GetChunkStartPos()); mem.Seek(seek, SeekOrigin.Current); localPos += seek; } while (localPos < chunk.GetUncompressedByteSize()) { // compute absolute position var pos = new ReadPos(localPos + chunk.GetChunkStartPos()); // read key var keySize = ReadKeySize(mem, ref localPos); mem.Read(keyBuffer, 0, keySize); localPos += keySize; var keySegment = new ArraySegment <byte>(keyBuffer, 0, keySize); // read value var valueSize = ReadValueSize(mem, ref localPos); using (var b = new BoundedStream(mem, valueSize)) { handler(pos, keySegment, b); } localPos += valueSize; if (--recordCount == 0) { return; } } } } } if (_buffer == null) { return; } var bufferStartPos = _buffer.GetBufferStartPos(); //Console.WriteLine("Tail starts {0}, {1} offset (or {2})", bufferStartPos, offset, offset - bufferStartPos); var bufferPath = Path.Combine(_folder, _buffer.GetBufferFileName()); using (var f = File.OpenRead(bufferPath)) { var localPos = 0; if (offset > bufferStartPos) { var seek = (int)(offset - bufferStartPos); if (seek < 0) { var msg = string.Format("Tail starts at {0} trying to read at offset {1}", bufferStartPos, offset); throw new IOException(msg); } f.Seek(seek, SeekOrigin.Begin); localPos += seek; } var bytesInBuffer = _buffer.GetBufferPos(); while (localPos < bytesInBuffer) { // record position var rp = new ReadPos(localPos + bufferStartPos); // read key var keySize = ReadKeySize(f, ref localPos); f.Read(keyBuffer, 0, keySize); localPos += keySize; var keySegment = new ArraySegment <byte>(keyBuffer, 0, keySize); // read value int valueSize = ReadValueSize(f, ref localPos); using (var b = new BoundedStream(f, valueSize)) { handler(rp, keySegment, b); } localPos += valueSize; if (--recordCount == 0) { return; } } } }