public bool Open(IPositionLessStream positionLessStream, bool dispose) { lock (_log) { _log.WriteUInt8((byte)KVReplayOperation.Open); ulong size = positionLessStream.GetSize(); _log.WriteVUInt64(size); ulong pos = 0; var buf = new byte[4096]; while (pos < size) { var read = positionLessStream.Read(buf, 0, buf.Length, pos); // Next 2 conditions should not happen or file is mutated when it should not if (read == 0) { break; } if ((ulong)read > size - pos) { read = (int)(size - pos); } _log.WriteBlock(buf, 0, read); pos += (ulong)read; } while (pos < size) { _log.WriteUInt8(0); pos++; } _log.FlushBuffer(); } return(_db.Open(positionLessStream, dispose)); }
public LoggingPositionLessStream(IPositionLessStream positionLessStream, bool dispose, Action<string> logAction) { _positionLessStream = positionLessStream; _dispose = dispose; _logAction = logAction; Log("LoggingStream created Stream:{0} Dispose:{1}", _positionLessStream.ToString(), _dispose); }
public bool Open(IPositionLessStream positionLessStream, bool dispose) { lock (_log) { _log.WriteUInt8((byte)KVReplayOperation.Open); ulong size = positionLessStream.GetSize(); _log.WriteVUInt64(size); ulong pos = 0; var buf = new byte[4096]; while (pos < size) { var read = positionLessStream.Read(buf, 0, buf.Length, pos); // Next 2 conditions should not happen or file is mutated when it should not if (read == 0) break; if ((ulong)read > size - pos) read = (int)(size - pos); _log.WriteBlock(buf, 0, read); pos += (ulong)read; } while (pos < size) { _log.WriteUInt8(0); pos++; } _log.FlushBuffer(); } return _db.Open(positionLessStream, dispose); }
public LoggingPositionLessStream(IPositionLessStream positionLessStream, bool dispose, Action <string> logAction) { _positionLessStream = positionLessStream; _dispose = dispose; _logAction = logAction; Log("LoggingStream created Stream:{0} Dispose:{1}", _positionLessStream.ToString(), _dispose); }
public PositionLessStreamWriter(IPositionLessStream stream) { _stream = stream; Buf = new byte[4096]; End = Buf.Length; _ofs = 0; _stream.SetSize(0); }
public PositionLessStreamReader(IPositionLessStream stream) { _stream = stream; _valueSize = _stream.GetSize(); _ofs = 0; Buf = new byte[4096]; FillBuffer(); }
public PositionLessStreamReader(IPositionLessStream stream) { _stream = stream; _valueSize = _stream.GetSize(); _ofs = 0; Buf = new byte[8192]; FillBuffer(); }
public void Dispose() { Log("LoggingStream disposed"); if (_dispose && _positionLessStream != null) { _positionLessStream.Dispose(); _positionLessStream = null; } }
public PositionLessStreamReader(IPositionLessStream stream, int bufferSize) { if (bufferSize <= 0) { throw new ArgumentOutOfRangeException(nameof(bufferSize)); } _stream = stream; _valueSize = _stream.GetSize(); _ofs = 0; Buf = new byte[bufferSize]; FillBuffer(); }
public PositionLessStreamWriter(IPositionLessStream stream, Action?onDispose, bool atEnd = false) { _stream = stream; _onDispose = onDispose ?? DisposeStream; _buf = new byte[BufLength]; _pos = 0; if (atEnd) { _ofs = _stream.GetSize(); } else { _ofs = 0; _stream.SetSize(0); } }
public PositionLessStreamWriter(IPositionLessStream stream, Action onDispose, bool atEnd = false) { _stream = stream; if (onDispose == null) onDispose = DisposeStream; _onDispose = onDispose; Buf = new byte[8192]; End = Buf.Length; if (atEnd) { _ofs = _stream.GetSize(); } else { _ofs = 0; _stream.SetSize(0); } }
public PositionLessStreamWriter(IPositionLessStream stream, Action onDispose, bool atEnd = false) { _stream = stream; if (onDispose == null) { onDispose = DisposeStream; } _onDispose = onDispose; Buf = new byte[8192]; End = Buf.Length; if (atEnd) { _ofs = _stream.GetSize(); } else { _ofs = 0; _stream.SetSize(0); } }
public PositionLessStreamWriter(IPositionLessStream stream, bool atEnd = false) : this(stream, null, atEnd) { }
public PositionLessStreamReader(IPositionLessStream stream) : this(stream, 8192) { }
public void Replay() { while (!_reader.Eof) { var operation = (KVReplayOperation)_reader.ReadUInt8(); Console.WriteLine(operation); uint tri; IKeyValueDBTransaction tr; int valOfs; int valLen; byte[] valBuf; int keyLen; int keyOfs; byte[] keyBuf; switch (operation) { case KVReplayOperation.Open: var initialStreamSize = _reader.ReadVUInt64(); _dbstream = new MemoryPositionLessStream(); var buf = new byte[4096]; ulong pos = 0; while (pos < initialStreamSize) { var r = (int)Math.Min((ulong)buf.Length, initialStreamSize - pos); _reader.ReadBlock(buf, 0, r); _dbstream.Write(buf, 0, r, pos); pos += (ulong)r; } _db.Open(_dbstream, true); break; case KVReplayOperation.KeyValueDBDispose: _db.Dispose(); break; case KVReplayOperation.StartTransaction: tr = _db.StartTransaction(); tri = _reader.ReadVUInt32(); _trs[tri] = tr; break; case KVReplayOperation.StartWritingTransaction: tr = _db.StartWritingTransaction().Result; tri = _reader.ReadVUInt32(); _trs[tri] = tr; break; case KVReplayOperation.CalculateStats: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.CalculateStats(); break; case KVReplayOperation.SetKeyPrefix: tri = _reader.ReadVUInt32(); tr = _trs[tri]; int prefixLen = _reader.ReadVInt32(); int prefixOfs = _reader.ReadVInt32(); var prefixBuf = new byte[prefixOfs + prefixLen]; _reader.ReadBlock(prefixBuf, prefixOfs, prefixLen); tr.SetKeyPrefix(prefixBuf, prefixOfs, prefixLen); break; case KVReplayOperation.FindKey: tri = _reader.ReadVUInt32(); tr = _trs[tri]; keyLen = _reader.ReadVInt32(); keyOfs = _reader.ReadVInt32(); keyBuf = new byte[keyOfs + keyLen]; _reader.ReadBlock(keyBuf, keyOfs, keyLen); var strategy = (FindKeyStrategy)_reader.ReadVUInt32(); tr.FindKey(keyBuf, keyOfs, keyLen, strategy); break; case KVReplayOperation.GetKeyIndex: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.GetKeyIndex(); break; case KVReplayOperation.SetValue: tri = _reader.ReadVUInt32(); tr = _trs[tri]; valOfs = _reader.ReadVInt32(); valLen = _reader.ReadVInt32(); valBuf = new byte[valOfs + valLen]; _reader.ReadBlock(valBuf, valOfs, valLen); tr.SetValue(valBuf, valOfs, valLen); break; case KVReplayOperation.SetValueSize: tri = _reader.ReadVUInt32(); tr = _trs[tri]; var valueSize = _reader.ReadVInt64(); tr.SetValueSize(valueSize); break; case KVReplayOperation.WriteValue: tri = _reader.ReadVUInt32(); tr = _trs[tri]; var ofs = _reader.ReadVInt64(); valOfs = _reader.ReadVInt32(); valLen = _reader.ReadVInt32(); valBuf = new byte[valOfs + valLen]; _reader.ReadBlock(valBuf, valOfs, valLen); tr.WriteValue(ofs, valLen, valBuf, valOfs); break; case KVReplayOperation.CreateOrUpdateKeyValue: tri = _reader.ReadVUInt32(); tr = _trs[tri]; keyLen = _reader.ReadVInt32(); keyOfs = _reader.ReadVInt32(); keyBuf = new byte[keyOfs + keyLen]; _reader.ReadBlock(keyBuf, keyOfs, keyLen); valLen = _reader.ReadVInt32(); valOfs = _reader.ReadVInt32(); valBuf = new byte[valOfs + valLen]; _reader.ReadBlock(valBuf, valOfs, valLen); tr.CreateOrUpdateKeyValue(keyBuf, keyOfs, keyLen, valBuf, valOfs, valLen); break; case KVReplayOperation.Commit: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.Commit(); break; case KVReplayOperation.TransactionDispose: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.Dispose(); _trs.Remove(tri); break; case KVReplayOperation.EraseCurrent: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.EraseCurrent(); break; case KVReplayOperation.EraseAll: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.EraseAll(); break; case KVReplayOperation.FindFirstKey: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.FindFirstKey(); break; case KVReplayOperation.FindPreviousKey: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.FindPreviousKey(); break; case KVReplayOperation.FindNextKey: tri = _reader.ReadVUInt32(); tr = _trs[tri]; tr.FindNextKey(); break; default: Console.WriteLine(string.Format("Unimplemented operation {0}({1})", operation, (byte)operation)); throw new NotSupportedException(string.Format("Unimplemented operation {0}({1})", operation, (byte)operation)); } } Console.WriteLine("Finish"); }
public bool Open(IPositionLessStream positionLessStream, bool dispose) { if (positionLessStream == null) throw new ArgumentNullException("positionLessStream"); _positionLessStream = positionLessStream; _disposeStream = dispose; _spaceSoonReusable = null; _freeSpaceAllocatorOptimizer.GlobalInvalidate(); _wasAnyCommits = false; bool newDB = false; if (positionLessStream.GetSize() == 0) { InitEmptyDB(); newDB = true; } else { if (_positionLessStream.Read(_headerData, 0, TotalHeaderSize, 0) != TotalHeaderSize) { throw new BTDBException("Too short header"); } _totalBytesRead += TotalHeaderSize; if (_headerData[0] != (byte)'B' || _headerData[1] != (byte)'T' || _headerData[2] != (byte)'D' || _headerData[3] != (byte)'B' || _headerData[4] != (byte)'1' || _headerData[5] != (byte)'0' || _headerData[6] != (byte)'0' || _headerData[7] != (byte)'2') { throw new BTDBException("Wrong header"); } } _newState.Position = FirstRootOffset; _currentState.Position = SecondRootOffset; if (RetrieveStateFromHeaderBuffer(_newState)) { if (RetrieveStateFromHeaderBuffer(_currentState)) { if (_currentState.TransactionCounter != _newState.TransactionCounter) { if (_currentState.TransactionCounter > _newState.TransactionCounter) { SwapCurrentAndNewState(); } if (!CheckDB(_newState)) { if (CheckDB(_currentState)) { SwapCurrentAndNewState(); } else { ThrowDatabaseCorrupted(); } } } } else { if (!CheckDB(_newState)) { ThrowDatabaseCorrupted(); } } } else { SwapCurrentAndNewState(); if (RetrieveStateFromHeaderBuffer(_newState) == false) { throw new BTDBException("Both root headers corrupted"); } if (!CheckDB(_newState)) { ThrowDatabaseCorrupted(); } } TransferNewStateToCurrentState(); if (_currentState.TransactionAllocSize > 0) { throw new BTDBException("TransactionLog is not supported"); } return newDB; }
public void Setup() { _dbstream = new MemoryPositionLessStream(); OpenDB(); }
public void CreateSource() { using (var os = new System.IO.StreamWriter("source.cs", false, Encoding.UTF8)) { while (!_reader.Eof) { var operation = (KVReplayOperation)_reader.ReadUInt8(); Console.WriteLine(operation); uint tri; switch (operation) { case KVReplayOperation.Open: var initialStreamSize = _reader.ReadVUInt64(); _dbstream = new MemoryPositionLessStream(); var buf = new byte[4096]; ulong pos = 0; while (pos < initialStreamSize) { var r = (int)Math.Min((ulong)buf.Length, initialStreamSize - pos); _reader.ReadBlock(buf, 0, r); _dbstream.Write(buf, 0, r, pos); pos += (ulong)r; } os.WriteLine("var db=new KeyValueDB();"); os.WriteLine("var dbstream = new MemoryPositionLessStream();"); os.WriteLine("db.Open(dbstream,true);"); break; case KVReplayOperation.KeyValueDBDispose: os.WriteLine("db.Dispose();"); break; case KVReplayOperation.StartTransaction: tri = _reader.ReadVUInt32(); os.WriteLine(string.Format("var tr{0}=db.StartTransaction();", tri)); break; case KVReplayOperation.CalculateStats: tri = _reader.ReadVUInt32(); break; case KVReplayOperation.FindKey: tri = _reader.ReadVUInt32(); int keyLen = _reader.ReadVInt32(); int keyOfs = _reader.ReadVInt32(); var keyBuf = new byte[keyLen]; _reader.ReadBlock(keyBuf, 0, keyLen); var strategy = (FindKeyStrategy)_reader.ReadVUInt32(); var content = new StringBuilder(); content.Append("{"); if (keyBuf.Length != 0) { foreach (var b in keyBuf) { content.Append(b); content.Append(","); } content.Length--; } content.Append("}"); if (strategy == FindKeyStrategy.Create) { os.WriteLine(string.Format("tr{0}.CreateKey(new byte[] {1});", tri, content)); } else { throw new NotImplementedException(); } break; case KVReplayOperation.GetKeyIndex: tri = _reader.ReadVUInt32(); break; case KVReplayOperation.SetValue: tri = _reader.ReadVUInt32(); int valOfs = _reader.ReadVInt32(); int valLen = _reader.ReadVInt32(); _reader.SkipBlock(valLen); os.WriteLine(string.Format("tr{0}.SetValue(new byte[{1}]);", tri, valLen)); break; case KVReplayOperation.Commit: tri = _reader.ReadVUInt32(); os.WriteLine(string.Format("tr{0}.Commit();", tri)); break; case KVReplayOperation.TransactionDispose: tri = _reader.ReadVUInt32(); os.WriteLine(string.Format("tr{0}.Dispose();", tri)); break; case KVReplayOperation.EraseCurrent: tri = _reader.ReadVUInt32(); os.WriteLine(string.Format("tr{0}.EraseCurrent();", tri)); break; case KVReplayOperation.FindPreviousKey: tri = _reader.ReadVUInt32(); os.WriteLine(string.Format("tr{0}.FindPreviousKey();", tri)); break; case KVReplayOperation.FindNextKey: tri = _reader.ReadVUInt32(); os.WriteLine(string.Format("tr{0}.FindNextKey();", tri)); break; default: Console.WriteLine(string.Format("Unimplemented operation {0}({1})", operation, (byte)operation)); throw new NotSupportedException(string.Format("Unimplemented operation {0}({1})", operation, (byte)operation)); } } } }
public void Dispose() { InDebuggerCheckDisposeInvariants(); if (_wasAnyCommits) { StoreStateToHeaderBuffer(_newState); _totalBytesWritten += RootSize; _positionLessStream.Write(_headerData, (int)_newState.Position, RootSize, _newState.Position); _positionLessStream.Flush(); } if (_disposeStream) { var disposable = _positionLessStream as IDisposable; if (disposable != null) disposable.Dispose(); } _positionLessStream = null; }