internal static SectorPtr Unpack(byte[] data, int offset) { SectorPtr result; result.Ptr = PackUnpack.UnpackInt64LE(data, offset); result.Checksum = PackUnpack.UnpackUInt32LE(data, offset + 8); return(result); }
public void ReadToEnd(IEventStoreObserver observer) { var overflowWriter = default(ByteBufferWriter); var bufferBlock = new byte[FirstReadAhead + MaxBlockSize]; var bufferStartPosition = NextReadPosition & SectorMask; var bufferFullLength = 0; var bufferReadOffset = (int)(NextReadPosition - bufferStartPosition); var currentReadAhead = FirstReadAhead; var buf = ByteBuffer.NewSync(bufferBlock, bufferFullLength, currentReadAhead); var bufReadLength = (int)File.Read(buf, bufferStartPosition); bufferFullLength += bufReadLength; while (true) { if (bufferStartPosition + (ulong)bufferReadOffset + HeaderSize > File.MaxFileSize) { KnownAsFinished = true; return; } if (bufferReadOffset == bufferFullLength) { break; } if (bufferReadOffset + HeaderSize > bufferFullLength) { for (var i = bufferReadOffset; i < bufferFullLength; i++) { if (bufferBlock[i] != 0) { SetCorrupted(); return; } } break; } var blockCheckSum = PackUnpack.UnpackUInt32LE(bufferBlock, bufferReadOffset); bufferReadOffset += 4; var blockLen = PackUnpack.UnpackUInt32LE(bufferBlock, bufferReadOffset); if (blockCheckSum == 0 && blockLen == 0) { bufferReadOffset -= 4; break; } var blockType = (BlockType)(blockLen & 0xff); blockLen >>= 8; if (blockType == BlockType.LastBlock && blockLen == 0) { if (Checksum.CalcFletcher32(bufferBlock, (uint)bufferReadOffset, 4) != blockCheckSum) { SetCorrupted(); return; } KnownAsFinished = true; return; } if (blockLen == 0 && blockType != (BlockType.FirstBlock | BlockType.LastBlock)) { SetCorrupted(); return; } if (blockLen + HeaderSize > MaxBlockSize) { SetCorrupted(); return; } bufferReadOffset += 4; var bufferLenToFill = (uint)(bufferReadOffset + (int)blockLen + FirstReadAhead) & SectorMaskUInt; if (bufferLenToFill > bufferBlock.Length) { bufferLenToFill = (uint)bufferBlock.Length; } buf = ByteBuffer.NewSync(bufferBlock, bufferFullLength, (int)(bufferLenToFill - bufferFullLength)); if (buf.Length > 0) { bufferLenToFill = (uint)(bufferReadOffset + (int)blockLen + currentReadAhead) & SectorMaskUInt; if (bufferLenToFill > bufferBlock.Length) { bufferLenToFill = (uint)bufferBlock.Length; } if (bufferStartPosition + bufferLenToFill > File.MaxFileSize) { bufferLenToFill = (uint)(File.MaxFileSize - bufferStartPosition); } buf = ByteBuffer.NewSync(bufferBlock, bufferFullLength, (int)(bufferLenToFill - bufferFullLength)); if (buf.Length > 0) { if (currentReadAhead * 4 < MaxBlockSize) { currentReadAhead = currentReadAhead * 2; } bufReadLength = (int)File.Read(buf, bufferStartPosition + (ulong)bufferFullLength); bufferFullLength += bufReadLength; } } if (bufferReadOffset + (int)blockLen > bufferFullLength) { SetCorrupted(); return; } if (Checksum.CalcFletcher32(bufferBlock, (uint)bufferReadOffset - 4, blockLen + 4) != blockCheckSum) { SetCorrupted(); return; } var blockTypeBlock = blockType & (BlockType.FirstBlock | BlockType.MiddleBlock | BlockType.LastBlock); var stopReadingRequested = false; if (blockTypeBlock == (BlockType.FirstBlock | BlockType.LastBlock)) { stopReadingRequested = Process(blockType, ByteBuffer.NewSync(bufferBlock, bufferReadOffset, (int)blockLen), observer); } else { if (blockTypeBlock == BlockType.FirstBlock) { overflowWriter = new ByteBufferWriter(); } else if (blockTypeBlock == BlockType.MiddleBlock || blockTypeBlock == BlockType.LastBlock) { if (overflowWriter == null) { SetCorrupted(); return; } } else { SetCorrupted(); return; } overflowWriter.WriteBlock(ByteBuffer.NewSync(bufferBlock, bufferReadOffset, (int)blockLen)); if (blockTypeBlock == BlockType.LastBlock) { stopReadingRequested = Process(blockType, overflowWriter.Data, observer); overflowWriter = null; } } bufferReadOffset += (int)blockLen; if (overflowWriter == null) { NextReadPosition = bufferStartPosition + (ulong)bufferReadOffset; } if (stopReadingRequested) { return; } var nextBufferStartPosition = (bufferStartPosition + (ulong)bufferReadOffset) & SectorMask; var bufferMoveDistance = (int)(nextBufferStartPosition - bufferStartPosition); if (bufferMoveDistance <= 0) { continue; } Array.Copy(bufferBlock, bufferMoveDistance, bufferBlock, 0, bufferFullLength - bufferMoveDistance); bufferStartPosition = nextBufferStartPosition; bufferFullLength -= bufferMoveDistance; bufferReadOffset -= bufferMoveDistance; } if (overflowWriter != null) { // It is not corrupted here just unfinished, but definitely not appendable EndBufferPosition = ulong.MaxValue; return; } EndBufferLen = (uint)(bufferReadOffset - (bufferReadOffset & SectorMaskUInt)); EndBufferPosition = bufferStartPosition + (ulong)bufferReadOffset - EndBufferLen; Array.Copy(bufferBlock, bufferReadOffset - EndBufferLen, EndBuffer, 0, EndBufferLen); }
/// <summary> /// Reads and inserts all key value pairs into current prefix from stream /// </summary> /// <param name="transaction">transaction where to import all data</param> /// <param name="stream">where to read it from</param> public static void Import(IKeyValueDBTransaction transaction, Stream stream) { if (transaction == null) { throw new ArgumentNullException(nameof(transaction)); } if (stream == null) { throw new ArgumentNullException(nameof(stream)); } if (!stream.CanRead) { throw new ArgumentException("stream must be readable", nameof(stream)); } var tempbuf = new byte[4096]; var tempbuf2 = new byte[4096]; if (stream.Read(tempbuf, 0, 16) != 16) { throw new EndOfStreamException(); } if (tempbuf[0] != 'B' || tempbuf[1] != 'T' || tempbuf[2] != 'D' || tempbuf[3] != 'B' || tempbuf[4] != 'E' || tempbuf[5] != 'X' || tempbuf[6] != 'P' || tempbuf[7] != '2') { throw new BTDBException("Invalid header (it should Start with BTDBEXP2)"); } var keyValuePairs = PackUnpack.UnpackInt64LE(tempbuf, 8); if (keyValuePairs < 0) { throw new BTDBException("Negative number of key value pairs"); } for (var kv = 0; kv < keyValuePairs; kv++) { if (stream.Read(tempbuf, 0, 4) != 4) { throw new EndOfStreamException(); } var keySize = PackUnpack.UnpackInt32LE(tempbuf, 0); if (keySize < 0) { throw new BTDBException("Negative key size"); } if (keySize > tempbuf.Length) { tempbuf = new byte[keySize]; } if (stream.Read(tempbuf, 0, keySize) != keySize) { throw new EndOfStreamException(); } if (stream.Read(tempbuf2, 0, 4) != 4) { throw new EndOfStreamException(); } var valueSize = PackUnpack.UnpackInt32LE(tempbuf2, 0); if (valueSize < 0) { throw new BTDBException("Negative value size"); } if (valueSize > tempbuf2.Length) { tempbuf2 = new byte[valueSize]; } if (stream.Read(tempbuf2, 0, valueSize) != valueSize) { throw new EndOfStreamException(); } transaction.CreateOrUpdateKeyValue(tempbuf.AsSpan(0, keySize), tempbuf2.AsSpan(0, valueSize)); } if (stream.Read(tempbuf, 0, 8) == 8) { transaction.SetCommitUlong(PackUnpack.UnpackUInt64LE(tempbuf, 0)); if (stream.Read(tempbuf, 0, 4) == 4) { var ulongCount = PackUnpack.UnpackUInt32LE(tempbuf, 0); for (var i = 0u; i < ulongCount; i++) { if (stream.Read(tempbuf, 0, 8) != 8) { throw new EndOfStreamException(); } transaction.SetUlong(i, PackUnpack.UnpackUInt64LE(tempbuf, 0)); } } } }