public void Dispose() { if (_disposed) { return; } _disposed = true; // we cannot dispose the journal until we are done with all of the pending writes if (_lazyTransactionBuffer != null) { _lazyTransactionBuffer.WriteBufferToFile(CurrentFile, null); _lazyTransactionBuffer.Dispose(); } _compressionPager.Dispose(); _lz4.Dispose(); _journalApplicator.Dispose(); if (_env.Options.OwnsPagers) { foreach (var logFile in _files) { logFile.Dispose(); } } else { foreach (var logFile in _files) { GC.SuppressFinalize(logFile); } } _files = ImmutableAppendOnlyList <JournalFile> .Empty; }
public void WriteGather(long position, IntPtr[] pages) { _locker.EnterWriteLock(); try { if (position != _lastPos) { throw new InvalidOperationException("Journal writes must be to the next location in the journal"); } var size = pages.Length * AbstractPager.PageSize; _lastPos += size; var handle = Marshal.AllocHGlobal(size); var buffer = new Buffer { Handle = handle, Pointer = (byte *)handle.ToPointer(), SizeInPages = pages.Length }; _buffers = _buffers.Append(buffer); for (int index = 0; index < pages.Length; index++) { Memory.Copy(buffer.Pointer + (index * AbstractPager.PageSize), (byte *)pages[index].ToPointer(), AbstractPager.PageSize); } } finally { _locker.ExitWriteLock(); } }
internal FragmentedPureMemoryPager(StorageEnvironmentOptions options, ImmutableAppendOnlyList <PureMemoryJournalWriter.Buffer> buffers) : base(options) { _buffers = buffers; NumberOfAllocatedPages = buffers.Sum(x => x.SizeInPages); FileName = ":memory:"; }
public void WritePages(long position, byte *p, int numberOfPages) { _locker.EnterWriteLock(); try { if (position != _lastPos) { throw new InvalidOperationException("Journal writes must be to the next location in the journal"); } var size = numberOfPages * _options.PageSize; _lastPos += size; var handle = NativeMemory.AllocateMemory(size); var buffer = new Buffer { Pointer = handle, SizeInPages = numberOfPages }; _buffers = _buffers.Append(buffer); Memory.Copy(buffer.Pointer, p, numberOfPages * _options.PageSize); } finally { _locker.ExitWriteLock(); } }
public unsafe void WriteBuffer(long position, byte *srcPointer, int sizeToWrite) { _locker.EnterWriteLock(); try { if (position != _lastPos) { throw new InvalidOperationException("Journal writes must be to the next location in the journal"); } _lastPos += sizeToWrite; var handle = Marshal.AllocHGlobal(sizeToWrite); var buffer = new Buffer { Handle = handle, Pointer = (byte *)handle.ToPointer(), SizeInPages = sizeToWrite / _options.PageSize }; _buffers = _buffers.Append(buffer); Memory.Copy(buffer.Pointer, (byte *)srcPointer, sizeToWrite); } finally { _locker.ExitWriteLock(); } }
public void Dispose() { if (disposed) { return; } disposed = true; // we cannot dispose the journal until we are done with all of the pending writes _compressionPager.Dispose(); _lz4.Dispose(); if (_env.Options.OwnsPagers) { foreach (var logFile in _files) { logFile.Dispose(); } _journalApplicator.Dispose(); } else { foreach (var logFile in _files) { GC.SuppressFinalize(logFile); } } _files = ImmutableAppendOnlyList <JournalFile> .Empty; }
private JournalFile NextFile(int numberOfPages = 1) { _journalIndex++; var now = DateTime.UtcNow; if ((now - _lastFile).TotalSeconds < 90) { _currentJournalFileSize = Math.Min(_env.Options.MaxLogFileSize, _currentJournalFileSize * 2); } var actualLogSize = _currentJournalFileSize; var minRequiredSize = numberOfPages * AbstractPager.PageSize; if (_currentJournalFileSize < minRequiredSize) { actualLogSize = minRequiredSize; } _lastFile = now; var journalPager = _env.Options.CreateJournalWriter(_journalIndex, actualLogSize); var journal = new JournalFile(journalPager, _journalIndex); journal.AddRef(); // one reference added by a creator - write ahead log _files = _files.Append(journal); _headerAccessor.Modify(_updateLogInfo); return(journal); }
public void Cleanup() { if (_initialized == false) { return; } if (_oldPagers.Count == 0) { return; } var necessaryPages = Interlocked.Read(ref _currentlyUsedBytes) / Constants.Storage.PageSize; var availablePages = _compressionPager.NumberOfAllocatedPages; var pagers = _oldPagers; for (var i = pagers.Count - 1; i >= 0; i--) { var old = pagers[i]; if (availablePages >= necessaryPages) { old.Dispose(); continue; } availablePages += old.NumberOfAllocatedPages; } _oldPagers = _oldPagers.RemoveWhile(x => x.Disposed); }
public void Dispose() { Disposed = true; foreach (var buffer in _buffers) { Marshal.FreeHGlobal(buffer.Handle); } _buffers = ImmutableAppendOnlyList<Buffer>.Empty; }
public void Dispose() { Disposed = true; foreach (var buffer in _buffers) { NativeMemory.Free(buffer.Pointer, buffer.SizeInPages * _options.PageSize); } _buffers = ImmutableAppendOnlyList <Buffer> .Empty; }
public void Dispose() { Disposed = true; foreach (var buffer in _buffers) { Marshal.FreeHGlobal(buffer.Handle); } _buffers = ImmutableAppendOnlyList <Buffer> .Empty; }
public override void Dispose() { base.Dispose(); foreach (var buffer in _buffers) { if (buffer.Handle == IntPtr.Zero) continue; Marshal.FreeHGlobal(new IntPtr(buffer.Base)); buffer.Handle = IntPtr.Zero; } _buffers = ImmutableAppendOnlyList<Buffer>.Empty; }
public void Clear(Transaction tx) { if (tx.Flags != TransactionFlags.ReadWrite) { throw new InvalidOperationException("Clearing of write ahead journal should be called only from a write transaction"); } foreach (var journalFile in _files) { journalFile.Release(); } _files = ImmutableAppendOnlyList <JournalFile> .Empty; CurrentFile = null; }
public override void Dispose() { base.Dispose(); foreach (var buffer in _buffers) { if (buffer.Handle == IntPtr.Zero) { continue; } Marshal.FreeHGlobal(new IntPtr(buffer.Base)); buffer.Handle = IntPtr.Zero; } _buffers = ImmutableAppendOnlyList <Buffer> .Empty; }
public PureMemoryPager(byte[] data) { var ptr = Marshal.AllocHGlobal(data.Length); var buffer = new Buffer { Handle = ptr, Base = (byte*)ptr.ToPointer(), Size = data.Length }; _buffers = _buffers.Append(buffer); NumberOfAllocatedPages = data.Length / PageSize; PagerState.Release(); PagerState = new PagerState(this); PagerState.AddRef(); fixed (byte* origin = data) { NativeMethods.memcpy(buffer.Base, origin, data.Length); } }
private void EnsureInitialized() { if (_initialized) { return; } lock (_decompressionPagerLock) { if (_initialized) { return; } _pool = new[] { new ConcurrentQueue <DecompressionBuffer>() }; _compressionPager = CreateDecompressionPager(DecompressedPagesCache.Size * Constants.Compression.MaxPageSize); _oldPagers = ImmutableAppendOnlyList <AbstractPager> .Empty; _initialized = true; } }
public PureMemoryPager(byte[] data) { var ptr = Marshal.AllocHGlobal(data.Length); var buffer = new Buffer { Handle = ptr, Base = (byte *)ptr.ToPointer(), Size = data.Length }; _buffers = _buffers.Append(buffer); NumberOfAllocatedPages = data.Length / PageSize; PagerState.Release(); PagerState = new PagerState(this); PagerState.AddRef(); fixed(byte *origin = data) { NativeMethods.memcpy(buffer.Base, origin, data.Length); } }
public override void AllocateMorePages(Transaction tx, long newLength) { var oldSize = NumberOfAllocatedPages * PageSize; if (newLength < oldSize) { throw new ArgumentException("Cannot set the legnth to less than the current length"); } if (newLength == oldSize) { return; // nothing to do } var increaseSize = (newLength - oldSize); NumberOfAllocatedPages += increaseSize / PageSize; var newPtr = Marshal.AllocHGlobal(new IntPtr(increaseSize)); var buffer = new Buffer { Handle = newPtr, Base = (byte *)newPtr.ToPointer(), Size = increaseSize }; _buffers = _buffers.Append(buffer); var newPager = new PagerState(this); newPager.AddRef(); // one for the pager if (tx != null) // we only pass null during startup, and we don't need it there { newPager.AddRef(); // one for the current transaction tx.AddPagerState(newPager); } PagerState.Release(); PagerState = newPager; }
public void Cleanup() { if (_initialized == false) { return; } if (_oldPagers.Count == 0) { return; } var necessaryPages = Interlocked.Read(ref _currentlyUsedBytes) / Constants.Storage.PageSize; var availablePages = _compressionPager.NumberOfAllocatedPages; var pagers = _oldPagers; for (var i = pagers.Count - 1; i >= 0; i--) { var old = pagers[i]; if (availablePages >= necessaryPages) { old.Dispose(); _scratchSpaceMonitor.Decrease(old.NumberOfAllocatedPages * Constants.Storage.PageSize); continue; } // PERF: We dont care about the pager data content anymore. So we can discard the whole context to // clean up the modified bit. old.DiscardWholeFile(); availablePages += old.NumberOfAllocatedPages; } _oldPagers = _oldPagers.RemoveWhile(x => x.Disposed); }
public IDisposable GetTemporaryPage(LowLevelTransaction tx, int pageSize, out TemporaryPage tmp) { if (pageSize < Constants.Storage.PageSize) { ThrowInvalidPageSize(pageSize); } if (pageSize > Constants.Compression.MaxPageSize) { ThrowPageSizeTooBig(pageSize); } Debug.Assert(pageSize == Bits.NextPowerOf2(pageSize)); EnsureInitialized(); var index = GetTempPagesPoolIndex(pageSize); if (_pool.Length <= index) { lock (_expandPoolLock) { if (_pool.Length <= index) // someone could get the lock and add it meanwhile { var oldSize = _pool.Length; var newPool = new ConcurrentQueue <DecompressionBuffer> [index + 1]; Array.Copy(_pool, newPool, _pool.Length); for (var i = oldSize; i < newPool.Length; i++) { newPool[i] = new ConcurrentQueue <DecompressionBuffer>(); } _pool = newPool; } } } DecompressionBuffer buffer; var queue = _pool[index]; tmp = null; while (queue.TryDequeue(out buffer)) { if (buffer.CanReuse == false) { continue; } try { buffer.EnsureValidPointer(tx); tmp = buffer.TempPage; break; } catch (ObjectDisposedException) { // we could dispose the pager during the cleanup } } if (tmp == null) { var allocationInPages = pageSize / Constants.Storage.PageSize; lock (_decompressionPagerLock) // once we fill up the pool we won't be allocating additional pages frequently { if (_lastUsedPage + allocationInPages > _maxNumberOfPagesInScratchBufferPool) { CreateNewBuffersPager(_options.MaxScratchBufferSize); } try { _compressionPager.EnsureContinuous(_lastUsedPage, allocationInPages); } catch (InsufficientMemoryException) { // RavenDB-10830: failed to lock memory of temp buffers in encrypted db, let's create new file with initial size CreateNewBuffersPager(DecompressedPagesCache.Size * Constants.Compression.MaxPageSize); throw; } buffer = new DecompressionBuffer(_compressionPager, _lastUsedPage, pageSize, this, index, tx); _lastUsedPage += allocationInPages; void CreateNewBuffersPager(long size) { _oldPagers = _oldPagers.Append(_compressionPager); _compressionPager = CreateDecompressionPager(size); _lastUsedPage = 0; } } tmp = buffer.TempPage; } Interlocked.Add(ref _currentlyUsedBytes, pageSize); return(tmp.ReturnTemporaryPageToPool); }
internal FragmentedPureMemoryPager(ImmutableAppendOnlyList <PureMemoryJournalWriter.Buffer> buffers) { _buffers = buffers; NumberOfAllocatedPages = buffers.Sum(x => x.SizeInPages); }
public bool RecoverDatabase(TransactionHeader *txHeader) { // note, we don't need to do any concurrency here, happens as a single threaded // fashion on db startup var requireHeaderUpdate = false; var logInfo = _headerAccessor.Get(ptr => ptr->Journal); if (logInfo.JournalFilesCount == 0) { _journalIndex = logInfo.LastSyncedJournal; return(false); } var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1; if (_env.Options.IncrementalBackupEnabled == false) { // we want to check that we cleanup old log files if they aren't needed // this is more just to be safe than anything else, they shouldn't be there. var unusedfiles = oldestLogFileStillInUse; while (true) { unusedfiles--; if (_env.Options.TryDeleteJournal(unusedfiles) == false) { break; } } } var lastSyncedTransactionId = logInfo.LastSyncedTransactionId; var journalFiles = new List <JournalFile>(); long lastSyncedTxId = -1; long lastSyncedJournal = logInfo.LastSyncedJournal; for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++) { using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber))) using (var pager = _env.Options.OpenJournalPager(journalNumber)) { RecoverCurrentJournalSize(pager); var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader; var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader); journalReader.RecoverAndValidate(_env.Options); var pagesToWrite = journalReader .TransactionPageTranslation .Select(kvp => recoveryPager.Read(kvp.Value.JournalPos)) .OrderBy(x => x.PageNumber) .ToList(); var lastReadHeaderPtr = journalReader.LastTransactionHeader; if (lastReadHeaderPtr != null) { if (pagesToWrite.Count > 0) { ApplyPagesToDataFileFromJournal(pagesToWrite); } *txHeader = *lastReadHeaderPtr; lastSyncedTxId = txHeader->TransactionId; lastSyncedJournal = journalNumber; } if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal) { var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize); var jrnlFile = new JournalFile(jrnlWriter, journalNumber); jrnlFile.InitFrom(journalReader); jrnlFile.AddRef(); // creator reference - write ahead log journalFiles.Add(jrnlFile); } if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions { requireHeaderUpdate = true; break; } } } _files = _files.AppendRange(journalFiles); Debug.Assert(lastSyncedTxId >= 0); Debug.Assert(lastSyncedJournal >= 0); _journalIndex = lastSyncedJournal; _headerAccessor.Modify( header => { header->Journal.LastSyncedJournal = lastSyncedJournal; header->Journal.LastSyncedTransactionId = lastSyncedTxId; header->Journal.CurrentJournal = lastSyncedJournal; header->Journal.JournalFilesCount = _files.Count; header->IncrementalBackup.LastCreatedJournal = _journalIndex; }); CleanupInvalidJournalFiles(lastSyncedJournal); CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal); if (_files.Count > 0) { var lastFile = _files.Last(); if (lastFile.AvailablePages >= 2) { // it must have at least one page for the next transaction header and one page for data CurrentFile = lastFile; } } return(requireHeaderUpdate); }
private JournalFile NextFile(int numberOfPages = 1) { _journalIndex++; var now = DateTime.UtcNow; if ((now - _lastFile).TotalSeconds < 90) { _currentJournalFileSize = Math.Min(_env.Options.MaxLogFileSize, _currentJournalFileSize * 2); } var actualLogSize = _currentJournalFileSize; var minRequiredSize = numberOfPages * AbstractPager.PageSize; if (_currentJournalFileSize < minRequiredSize) { actualLogSize = minRequiredSize; } _lastFile = now; var journalPager = _env.Options.CreateJournalWriter(_journalIndex, actualLogSize); var journal = new JournalFile(journalPager, _journalIndex); journal.AddRef(); // one reference added by a creator - write ahead log _files = _files.Append(journal); _headerAccessor.Modify(_updateLogInfo); return journal; }
public void Clear(Transaction tx) { if (tx.Flags != TransactionFlags.ReadWrite) throw new InvalidOperationException("Clearing of write ahead journal should be called only from a write transaction"); foreach (var journalFile in _files) { journalFile.Release(); } _files = ImmutableAppendOnlyList<JournalFile>.Empty; CurrentFile = null; }
internal FragmentedPureMemoryPager(ImmutableAppendOnlyList<PureMemoryJournalWriter.Buffer> buffers) { _buffers = buffers; NumberOfAllocatedPages = buffers.Sum(x => x.SizeInPages); }
public void WriteGather(long position, IntPtr[] pages) { _locker.EnterWriteLock(); try { if (position != _lastPos) throw new InvalidOperationException("Journal writes must be to the next location in the journal"); var size = pages.Length * AbstractPager.PageSize; _lastPos += size; var handle = Marshal.AllocHGlobal(size); var buffer = new Buffer { Handle = handle, Pointer = (byte*)handle.ToPointer(), SizeInPages = pages.Length }; _buffers = _buffers.Append(buffer); for (int index = 0; index < pages.Length; index++) { Memory.Copy(buffer.Pointer + (index * AbstractPager.PageSize), (byte*)pages[index].ToPointer(), AbstractPager.PageSize); } } finally { _locker.ExitWriteLock(); } }
public bool RecoverDatabase(TransactionHeader* txHeader) { // note, we don't need to do any concurrency here, happens as a single threaded // fashion on db startup var requireHeaderUpdate = false; var logInfo = _headerAccessor.Get(ptr => ptr->Journal); if (logInfo.JournalFilesCount == 0) { _journalIndex = logInfo.LastSyncedJournal; return false; } var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1; if (_env.Options.IncrementalBackupEnabled == false) { // we want to check that we cleanup old log files if they aren't needed // this is more just to be safe than anything else, they shouldn't be there. var unusedfiles = oldestLogFileStillInUse; while (true) { unusedfiles--; if (_env.Options.TryDeleteJournal(unusedfiles) == false) break; } } var lastSyncedTransactionId = logInfo.LastSyncedTransactionId; var journalFiles = new List<JournalFile>(); long lastSyncedTxId = -1; long lastSyncedJournal = logInfo.LastSyncedJournal; uint lastShippedTxCrc = 0; for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++) { using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber))) using (var pager = _env.Options.OpenJournalPager(journalNumber)) { RecoverCurrentJournalSize(pager); var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader; var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader); journalReader.RecoverAndValidate(_env.Options); var pagesToWrite = journalReader .TransactionPageTranslation .Select(kvp => recoveryPager.Read(kvp.Value.JournalPos)) .OrderBy(x => x.PageNumber) .ToList(); var lastReadHeaderPtr = journalReader.LastTransactionHeader; if (lastReadHeaderPtr != null) { if (pagesToWrite.Count > 0) ApplyPagesToDataFileFromJournal(pagesToWrite); *txHeader = *lastReadHeaderPtr; lastSyncedTxId = txHeader->TransactionId; lastShippedTxCrc = txHeader->Crc; lastSyncedJournal = journalNumber; } if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal) { var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize); var jrnlFile = new JournalFile(jrnlWriter, journalNumber); jrnlFile.InitFrom(journalReader); jrnlFile.AddRef(); // creator reference - write ahead log journalFiles.Add(jrnlFile); } if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions { requireHeaderUpdate = true; break; } } } Shipper.SetPreviousTransaction(lastSyncedTxId, lastShippedTxCrc); _files = _files.AppendRange(journalFiles); Debug.Assert(lastSyncedTxId >= 0); Debug.Assert(lastSyncedJournal >= 0); _journalIndex = lastSyncedJournal; _headerAccessor.Modify( header => { header->Journal.LastSyncedJournal = lastSyncedJournal; header->Journal.LastSyncedTransactionId = lastSyncedTxId; header->Journal.CurrentJournal = lastSyncedJournal; header->Journal.JournalFilesCount = _files.Count; header->IncrementalBackup.LastCreatedJournal = _journalIndex; header->PreviousTransactionCrc = lastShippedTxCrc; }); CleanupInvalidJournalFiles(lastSyncedJournal); CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal); if (_files.Count > 0) { var lastFile = _files.Last(); if (lastFile.AvailablePages >= 2) // it must have at least one page for the next transaction header and one page for data CurrentFile = lastFile; } return requireHeaderUpdate; }
public void Dispose() { if (disposed) return; disposed = true; // we cannot dispose the journal until we are done with all of the pending writes _compressionPager.Dispose(); _lz4.Dispose(); _journalApplicator.Dispose(); _shipppedTransactionsApplicator.Dispose(); if (_env.Options.OwnsPagers) { foreach (var logFile in _files) { logFile.Dispose(); } } else { foreach (var logFile in _files) { GC.SuppressFinalize(logFile); } } _files = ImmutableAppendOnlyList<JournalFile>.Empty; }
public override void AllocateMorePages(Transaction tx, long newLength) { var oldSize = NumberOfAllocatedPages * PageSize; if (newLength < oldSize) throw new ArgumentException("Cannot set the legnth to less than the current length"); if (newLength == oldSize) return; // nothing to do var increaseSize = (newLength - oldSize); NumberOfAllocatedPages += increaseSize / PageSize; var newPtr = Marshal.AllocHGlobal(new IntPtr(increaseSize)); var buffer = new Buffer { Handle = newPtr, Base = (byte*) newPtr.ToPointer(), Size = increaseSize }; _buffers = _buffers.Append(buffer); var newPager = new PagerState(this); newPager.AddRef(); // one for the pager if (tx != null) // we only pass null during startup, and we don't need it there { newPager.AddRef(); // one for the current transaction tx.AddPagerState(newPager); } PagerState.Release(); PagerState = newPager; }