public bool TryGetValue(Transaction tx, long page, out JournalFile.PagePosition value) { ImmutableAppendOnlyList<PageValue> list; if (_values.TryGetValue(page, out list) == false) { value = null; return false; } for (int i = list.Count - 1; i >= 0; i--) { var it = list[i]; if (it.Transaction > tx.Id) continue; if (it.Value.IsFreedPageMarker) break; value = it.Value; Debug.Assert(value != null); return true; } // all the current values are _after_ this transaction started, so it sees nothing value = null; return false; }
private JournalFile NextFile(int numberOfPages = 1) { _journalIndex++; var now = DateTime.UtcNow; if ((now - _lastFile).TotalSeconds < 90) { _currentJournalFileSize = Math.Min(_env.Options.MaxLogFileSize, _currentJournalFileSize * 2); } var actualLogSize = _currentJournalFileSize; var minRequiredSize = numberOfPages * AbstractPager.PageSize; if (_currentJournalFileSize < minRequiredSize) { actualLogSize = minRequiredSize; } _lastFile = now; var journalPager = _env.Options.CreateJournalWriter(_journalIndex, actualLogSize); var journal = new JournalFile(journalPager, _journalIndex); journal.AddRef(); // one reference added by a creator - write ahead log _files = _files.Append(journal); _headerAccessor.Modify(_updateLogInfo); return journal; }
public void WriteBufferToFile(JournalFile journalFile, LowLevelTransaction tx) { if (_firstPositionInJournalFile != null) { journalFile.WriteBuffer(_firstPositionInJournalFile.Value, _lazyTransactionPager.AcquirePagePointer(null, 0), _lastUsedPage * _options.DataPager.PageSize); } if (tx != null) { tx.IsLazyTransaction = false;// so it will notify the flush thread it has work to do } _readTransaction?.Dispose(); _firstPositionInJournalFile = null; _lastUsedPage = 0; _readTransaction = null; NumberOfPages = 0; }
public void ToStream(JournalFile journal, long startPage, long pagesToCopy, Stream output) { var maxNumOfPagesToCopyAtOnce = _buffer.Length/AbstractPager.PageSize; var page = startPage; fixed (byte* ptr = _buffer) { while (pagesToCopy > 0) { var pageCount = Math.Min(maxNumOfPagesToCopyAtOnce, pagesToCopy); var bytesCount = (int) (pageCount*AbstractPager.PageSize); journal.JournalWriter.Read(page, ptr, bytesCount); output.Write(_buffer, 0, bytesCount); page += pageCount; pagesToCopy -= pageCount; } } Debug.Assert(pagesToCopy == 0); }
public void WriteToJournal(Transaction tx, int pageCount) { var pages = CompressPages(tx, pageCount, _compressionPager); if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length) { CurrentFile = NextFile(pages.Length); } var transactionHeader = *(TransactionHeader *)pages[0]; var writePage = CurrentFile.Write(tx, pages); var onTransactionCommit = OnTransactionCommit; if (onTransactionCommit != null) { var bufferSize = pages.Length * AbstractPager.PageSize; var buffer = new byte[bufferSize]; fixed(byte *bp = buffer) CurrentFile.JournalWriter.Read(writePage, bp, bufferSize); var stream = new MemoryStream(buffer, AbstractPager.PageSize, (pages.Length - 1) * AbstractPager.PageSize); var transactionToShip = new TransactionToShip(transactionHeader) { CompressedData = stream, PreviousTransactionCrc = _previousTransactionCrc }; _previousTransactionCrc = transactionHeader.Crc; onTransactionCommit(transactionToShip); } if (CurrentFile.AvailablePages == 0) { CurrentFile = null; } }
public void WriteBufferToFile(JournalFile journalFile, LowLevelTransaction tx) { if (_firstPositionInJournalFile != null) { using (var tempTx = new TempPagerTransaction()) { _lazyTransactionPager.EnsureMapped(tempTx, 0, _lastUsedPage); var src = _lazyTransactionPager.AcquirePagePointer(tempTx, 0); journalFile.JournalWriter.WritePages(_firstPositionInJournalFile.Value, src, _lastUsedPage); } } if (tx != null) { tx.IsLazyTransaction = false;// so it will notify the flush thread it has work to do } _readTransaction?.Dispose(); _firstPositionInJournalFile = null; _lastUsedPage = 0; _readTransaction = null; NumberOfPages = 0; }
public TimeSpan WriteBufferToFile(JournalFile journalFile, LowLevelTransaction tx) { TimeSpan writeToJournalDuration = default; if (_firstPositionInJournalFile != null) { using (var tempTx = new TempPagerTransaction()) { var numberOfPages = _lastUsed4Kbs / (Constants.Storage.PageSize / (4 * Constants.Size.Kilobyte)); if ((_lastUsed4Kbs % (Constants.Storage.PageSize / (4 * Constants.Size.Kilobyte))) != 0) { numberOfPages++; } _lazyTransactionPager.EnsureMapped(tempTx, 0, numberOfPages); var src = _lazyTransactionPager.AcquirePagePointer(tempTx, 0); var sp = Stopwatch.StartNew(); writeToJournalDuration = journalFile.Write(_firstPositionInJournalFile.Value, src, _lastUsed4Kbs); if (_log.IsInfoEnabled) { _log.Info($"Writing lazy transaction buffer with {_lastUsed4Kbs / 4:#,#0} kb took {sp.Elapsed}"); } ZeroLazyTransactionBufferIfNeeded(tempTx); } } if (tx != null) { tx.IsLazyTransaction = false;// so it will notify the flush thread it has work to do } _readTransaction?.Dispose(); _firstPositionInJournalFile = null; _lastUsed4Kbs = 0; _readTransaction = null; NumberOfPages = 0; return(writeToJournalDuration); }
public void WriteToJournal(LowLevelTransaction tx, int pageCount) { var pages = CompressPages(tx, pageCount, _compressionPager); if (tx.IsLazyTransaction && _lazyTransactionBuffer == null) { _lazyTransactionBuffer = new LazyTransactionBuffer(_env.Options); } if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length) { _lazyTransactionBuffer?.WriteBufferToFile(CurrentFile, tx); CurrentFile = NextFile(pages.Length); } CurrentFile.Write(tx, pages, _lazyTransactionBuffer, pageCount); if (CurrentFile.AvailablePages == 0) { _lazyTransactionBuffer?.WriteBufferToFile(CurrentFile, tx); CurrentFile = null; } }
public void WriteToJournal(Transaction tx, int pageCount) { var pages = CompressPages(tx, pageCount, _compressionPager); if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length) { CurrentFile = NextFile(pages.Length); } CurrentFile.Write(tx, pages); if (CurrentFile.AvailablePages == 0) { CurrentFile = null; } }
public void ApplyLogsToDataFile(long oldestActiveTransaction, CancellationToken token, Transaction transaction = null) { bool locked = false; if (_flushingSemaphore.IsWriteLockHeld == false) { if (_flushingSemaphore.TryEnterWriteLock(Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30)) == false) { throw new TimeoutException("Could not acquire the write lock in 30 seconds"); } locked = true; } try { if (token.IsCancellationRequested) { return; } var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite; var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList(); if (jrnls.Count == 0) { return; // nothing to do } Debug.Assert(jrnls.First().Number >= _lastSyncedJournal); var pagesToWrite = new Dictionary <long, JournalFile.PagePosition>(); long lastProcessedJournal = -1; long previousJournalMaxTransactionId = -1; long lastFlushedTransactionId = -1; foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal)) { var currentJournalMaxTransactionId = -1L; foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(journalFile.LastTransaction)) { if (oldestActiveTransaction != 0 && pagePosition.Value.TransactionId >= oldestActiveTransaction) { // we cannot write this yet, there is a read transaction that might be looking at this // however, we _aren't_ going to be writing this to the data file, since that would be a // waste, we would just overwrite that value in the next flush anyway JournalFile.PagePosition existingPagePosition; if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) && pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber) { // remove the page only when it comes from the same journal // otherwise we can damage the journal's page translation table (PTT) // because the existing overwrite in a next journal can be filtered out // so we wouldn't write any page to the data file pagesToWrite.Remove(pagePosition.Key); } continue; } if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId) { continue; } currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId); if (currentJournalMaxTransactionId < previousJournalMaxTransactionId) { throw new InvalidOperationException( "Journal applicator read beyond the oldest active transaction in the next journal file. " + "This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId + ", previous journal max ix id: " + previousJournalMaxTransactionId + ", oldest active transaction: " + oldestActiveTransaction); } lastProcessedJournal = journalFile.Number; pagesToWrite[pagePosition.Key] = pagePosition.Value; lastFlushedTransactionId = currentJournalMaxTransactionId; } if (currentJournalMaxTransactionId == -1L) { continue; } previousJournalMaxTransactionId = currentJournalMaxTransactionId; } if (pagesToWrite.Count == 0) { return; } try { ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx); } catch (DiskFullException diskFullEx) { _waj._env.HandleDataDiskFullException(diskFullEx); return; } var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId); foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number))) { _journalsToDelete.Add(unused.Number, unused); } using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite)) { _lastSyncedJournal = lastProcessedJournal; _lastSyncedTransactionId = lastFlushedTransactionId; _lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal); if (unusedJournals.Count > 0) { var lastUnusedJournalNumber = unusedJournals.Last().Number; _waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber, new List <JournalFile>()); } if (_waj._files.Count == 0) { _waj.CurrentFile = null; } FreeScratchPages(unusedJournals, txw); if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit || DateTime.Now - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit) { _waj._dataPager.Sync(); UpdateFileHeaderAfterDataFileSync(_lastFlushedJournal, oldestActiveTransaction); foreach (var toDelete in _journalsToDelete.Values) { if (_waj._env.Options.IncrementalBackupEnabled == false) { toDelete.DeleteOnClose = true; } toDelete.Release(); } _journalsToDelete.Clear(); _totalWrittenButUnsyncedBytes = 0; _lastDataFileSyncTime = DateTime.Now; } if (txw != null) { txw.Commit(); } } } finally { if (locked) { _flushingSemaphore.ExitWriteLock(); } } }
public bool RecoverDatabase(TransactionHeader *txHeader) { // note, we don't need to do any concurrency here, happens as a single threaded // fashion on db startup var requireHeaderUpdate = false; var logInfo = _headerAccessor.Get(ptr => ptr->Journal); if (logInfo.JournalFilesCount == 0) { _journalIndex = logInfo.LastSyncedJournal; return(false); } var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1; if (_env.Options.IncrementalBackupEnabled == false) { // we want to check that we cleanup old log files if they aren't needed // this is more just to be safe than anything else, they shouldn't be there. var unusedfiles = oldestLogFileStillInUse; while (true) { unusedfiles--; if (_env.Options.TryDeleteJournal(unusedfiles) == false) { break; } } } var lastSyncedTransactionId = logInfo.LastSyncedTransactionId; var journalFiles = new List <JournalFile>(); long lastSyncedTxId = -1; long lastSyncedJournal = logInfo.LastSyncedJournal; for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++) { using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber))) using (var pager = _env.Options.OpenJournalPager(journalNumber)) { RecoverCurrentJournalSize(pager); var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader; var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader); journalReader.RecoverAndValidate(_env.Options); var pagesToWrite = journalReader .TransactionPageTranslation .Select(kvp => recoveryPager.Read(kvp.Value.JournalPos)) .OrderBy(x => x.PageNumber) .ToList(); var lastReadHeaderPtr = journalReader.LastTransactionHeader; if (lastReadHeaderPtr != null) { if (pagesToWrite.Count > 0) { ApplyPagesToDataFileFromJournal(pagesToWrite); } *txHeader = *lastReadHeaderPtr; lastSyncedTxId = txHeader->TransactionId; lastSyncedJournal = journalNumber; } if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal) { var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize); var jrnlFile = new JournalFile(jrnlWriter, journalNumber); jrnlFile.InitFrom(journalReader); jrnlFile.AddRef(); // creator reference - write ahead log journalFiles.Add(jrnlFile); } if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions { requireHeaderUpdate = true; break; } } } _files = _files.AppendRange(journalFiles); Debug.Assert(lastSyncedTxId >= 0); Debug.Assert(lastSyncedJournal >= 0); _journalIndex = lastSyncedJournal; _headerAccessor.Modify( header => { header->Journal.LastSyncedJournal = lastSyncedJournal; header->Journal.LastSyncedTransactionId = lastSyncedTxId; header->Journal.CurrentJournal = lastSyncedJournal; header->Journal.JournalFilesCount = _files.Count; header->IncrementalBackup.LastCreatedJournal = _journalIndex; }); CleanupInvalidJournalFiles(lastSyncedJournal); CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal); if (_files.Count > 0) { var lastFile = _files.Last(); if (lastFile.AvailablePages >= 2) { // it must have at least one page for the next transaction header and one page for data CurrentFile = lastFile; } } return(requireHeaderUpdate); }
public void ApplyLogsToDataFile(CancellationToken token, Transaction transaction = null, bool allowToFlushOverwrittenPages = false) { if (token.IsCancellationRequested) { return; } if (Monitor.IsEntered(_flushingLock) && ignoreLockAlreadyTaken == false) { throw new InvalidJournalFlushRequest("Applying journals to the data file has been already requested on the same thread"); } bool lockTaken = false; try { Monitor.TryEnter(_flushingLock, Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30), ref lockTaken); if (lockTaken == false) { throw new TimeoutException("Could not acquire the write lock in 30 seconds"); } var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite; var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList(); if (jrnls.Count == 0) { return; // nothing to do } Debug.Assert(jrnls.First().Number >= _lastSyncedJournal); var pagesToWrite = new Dictionary <long, PagePosition>(); long lastProcessedJournal = -1; long previousJournalMaxTransactionId = -1; long lastFlushedTransactionId = -1; long oldestActiveTransaction = _waj._env.OldestTransaction; foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal)) { var currentJournalMaxTransactionId = -1L; var iterateLatestAsOf = journalFile.LastTransaction; if (forcedFlushOfOldPages && forcedIterateJournalsAsOf.HasValue) { iterateLatestAsOf = Math.Min(journalFile.LastTransaction, forcedIterateJournalsAsOf.Value); } foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(iterateLatestAsOf)) { if (oldestActiveTransaction != 0 && pagePosition.Value.TransactionId >= oldestActiveTransaction) { if (pagePosition.Value.IsFreedPageMarker) { continue; } // we cannot write this yet, there is a read transaction that might be looking at this // however, we _aren't_ going to be writing this to the data file, since that would be a // waste, we would just overwrite that value in the next flush anyway PagePosition existingPagePosition; if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) && pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber) { // remove the page only when it comes from the same journal // otherwise we can damage the journal's page translation table (PTT) // because the existing overwrite in a next journal can be filtered out // so we wouldn't write any page to the data file pagesToWrite.Remove(pagePosition.Key); } continue; } if (pagePosition.Value.IsFreedPageMarker) { // to ensure that we won't overwrite data by a page from the older journal where it wasn't marked as free // while now it not being considered to be applied to the data file pagesToWrite.Remove(pagePosition.Key); continue; } if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId) { continue; } currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId); if (currentJournalMaxTransactionId < previousJournalMaxTransactionId) { throw new InvalidOperationException( "Journal applicator read beyond the oldest active transaction in the next journal file. " + "This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId + ", previous journal max ix id: " + previousJournalMaxTransactionId + ", oldest active transaction: " + oldestActiveTransaction); } lastProcessedJournal = journalFile.Number; pagesToWrite[pagePosition.Key] = pagePosition.Value; lastFlushedTransactionId = currentJournalMaxTransactionId; } if (currentJournalMaxTransactionId == -1L) { continue; } previousJournalMaxTransactionId = currentJournalMaxTransactionId; } if (pagesToWrite.Count == 0) { if (allowToFlushOverwrittenPages) { // we probably filtered out all pages because they have some overwrites and we applied an optimization // that relays on iterating over pages from end of PTT, however we might want to flush such pages // in order to allow to free them because the scratch buffer might require them // so we can flush all pages that we are sure they aren't being read by any transaction using (ForceFlushingPagesOlderThan(oldestActiveTransaction)) { ApplyLogsToDataFile(token, transaction, false); } } return; } try { ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx); } catch (DiskFullException diskFullEx) { _waj._env.HandleDataDiskFullException(diskFullEx); return; } var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId); foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number))) { _journalsToDelete.Add(unused.Number, unused); } using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite).JournalApplicatorTransaction()) { _lastSyncedJournal = lastProcessedJournal; _lastSyncedTransactionId = lastFlushedTransactionId; _lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal); if (unusedJournals.Count > 0) { var lastUnusedJournalNumber = unusedJournals.Last().Number; _waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber); } if (_waj._files.Count == 0) { _waj.CurrentFile = null; } FreeScratchPages(unusedJournals, txw ?? transaction); if (txw != null) { txw.Commit(); } } if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit || DateTime.UtcNow - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit) { SyncDataFile(); } } finally { if (lockTaken) { Monitor.Exit(_flushingLock); } } }
public void WriteToJournal(Transaction tx, int pageCount) { var pages = CompressPages(tx, pageCount, _compressionPager, Shipper.PreviousTransactionCrc); if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length) { CurrentFile = NextFile(pages.Length); } CurrentFile.Write(tx, pages); var transactionHeader = *(TransactionHeader*)pages[0]; var onTransactionCommit = OnTransactionCommit; if (onTransactionCommit != null) { var transactionToShip = new TransactionToShip(transactionHeader) { CompressedPages = pages }; onTransactionCommit(transactionToShip); } Shipper.SetPreviousTransaction(transactionHeader.TransactionId, transactionHeader.Crc); if (CurrentFile.AvailablePages == 0) CurrentFile = null; }
public void UpdateFileHeaderAfterDataFileSync(JournalFile file, long oldestActiveTransaction) { var txHeaders = stackalloc TransactionHeader[2]; var readTxHeader = &txHeaders[0]; var lastReadTxHeader = txHeaders[1]; var txPos = 0; while (true) { if (file.ReadTransaction(txPos, readTxHeader) == false) break; if (readTxHeader->HeaderMarker != Constants.TransactionHeaderMarker) break; if (readTxHeader->TransactionId + 1 == oldestActiveTransaction) break; lastReadTxHeader = *readTxHeader; var compressedPages = (readTxHeader->CompressedSize / AbstractPager.PageSize) + (readTxHeader->CompressedSize % AbstractPager.PageSize == 0 ? 0 : 1); txPos += compressedPages + 1; } Debug.Assert(_lastSyncedJournal != -1); Debug.Assert(_lastSyncedTransactionId != -1); _waj._headerAccessor.Modify(header => { header->TransactionId = lastReadTxHeader.TransactionId; header->LastPageNumber = lastReadTxHeader.LastPageNumber; header->Journal.LastSyncedJournal = _lastSyncedJournal; header->Journal.LastSyncedTransactionId = _lastSyncedTransactionId; header->Root = lastReadTxHeader.Root; header->FreeSpace = lastReadTxHeader.FreeSpace; header->PreviousTransactionCrc = _waj.Shipper.PreviousTransactionCrc; _waj._updateLogInfo(header); }); }
public void ApplyLogsToDataFile(long oldestActiveTransaction, CancellationToken token, Transaction transaction = null, bool allowToFlushOverwrittenPages = false) { if (token.IsCancellationRequested) return; if (Monitor.IsEntered(_flushingLock) && ignoreLockAlreadyTaken == false) throw new InvalidJournalFlushRequest("Applying journals to the data file has been already requested on the same thread"); bool lockTaken = false; try { Monitor.TryEnter(_flushingLock, Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30), ref lockTaken); if (lockTaken == false) throw new TimeoutException("Could not acquire the write lock in 30 seconds"); var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite; var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList(); if (jrnls.Count == 0) return; // nothing to do Debug.Assert(jrnls.First().Number >= _lastSyncedJournal); var pagesToWrite = new Dictionary<long, PagePosition>(); long lastProcessedJournal = -1; long previousJournalMaxTransactionId = -1; long lastFlushedTransactionId = -1; foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal)) { var currentJournalMaxTransactionId = -1L; var iterateLatestAsOf = journalFile.LastTransaction; if (forcedFlushOfOldPages && forcedIterateJournalsAsOf.HasValue) iterateLatestAsOf = Math.Min(journalFile.LastTransaction, forcedIterateJournalsAsOf.Value); foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(iterateLatestAsOf)) { if (oldestActiveTransaction != 0 && pagePosition.Value.TransactionId >= oldestActiveTransaction) { if(pagePosition.Value.IsFreedPageMarker) continue; // we cannot write this yet, there is a read transaction that might be looking at this // however, we _aren't_ going to be writing this to the data file, since that would be a // waste, we would just overwrite that value in the next flush anyway PagePosition existingPagePosition; if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) && pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber) { // remove the page only when it comes from the same journal // otherwise we can damage the journal's page translation table (PTT) // because the existing overwrite in a next journal can be filtered out // so we wouldn't write any page to the data file pagesToWrite.Remove(pagePosition.Key); } continue; } if (pagePosition.Value.IsFreedPageMarker) { // to ensure that we won't overwrite data by a page from the older journal where it wasn't marked as free // while now it not being considered to be applied to the data file pagesToWrite.Remove(pagePosition.Key); continue; } if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId) continue; currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId); if (currentJournalMaxTransactionId < previousJournalMaxTransactionId) throw new InvalidOperationException( "Journal applicator read beyond the oldest active transaction in the next journal file. " + "This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId + ", previous journal max ix id: " + previousJournalMaxTransactionId + ", oldest active transaction: " + oldestActiveTransaction); lastProcessedJournal = journalFile.Number; pagesToWrite[pagePosition.Key] = pagePosition.Value; lastFlushedTransactionId = currentJournalMaxTransactionId; } if (currentJournalMaxTransactionId == -1L) continue; previousJournalMaxTransactionId = currentJournalMaxTransactionId; } if (pagesToWrite.Count == 0) { if (allowToFlushOverwrittenPages) { // we probably filtered out all pages because they have some overwrites and we applied an optimization // that relays on iterating over pages from end of PTT, however we might want to flush such pages // in order to allow to free them because the scratch buffer might require them // so we can flush all pages that we are sure they aren't being read by any transaction using (ForceFlushingPagesOlderThan(oldestActiveTransaction)) { ApplyLogsToDataFile(oldestActiveTransaction, token, transaction, false); } } return; } try { ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx); } catch (DiskFullException diskFullEx) { _waj._env.HandleDataDiskFullException(diskFullEx); return; } var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId); foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number))) { _journalsToDelete.Add(unused.Number, unused); } using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite).JournalApplicatorTransaction()) { _lastSyncedJournal = lastProcessedJournal; _lastSyncedTransactionId = lastFlushedTransactionId; _lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal); if (unusedJournals.Count > 0) { var lastUnusedJournalNumber = unusedJournals.Last().Number; _waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber); } if (_waj._files.Count == 0) _waj.CurrentFile = null; FreeScratchPages(unusedJournals, txw ?? transaction); if (txw != null) txw.Commit(); } if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit || DateTime.UtcNow - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit) { SyncDataFile(oldestActiveTransaction); } } finally { if(lockTaken) Monitor.Exit(_flushingLock); } }
public void Clear(Transaction tx) { if (tx.Flags != TransactionFlags.ReadWrite) throw new InvalidOperationException("Clearing of write ahead journal should be called only from a write transaction"); foreach (var journalFile in _files) { journalFile.Release(); } _files = ImmutableAppendOnlyList<JournalFile>.Empty; CurrentFile = null; }
public bool RecoverDatabase(TransactionHeader* txHeader) { // note, we don't need to do any concurrency here, happens as a single threaded // fashion on db startup var requireHeaderUpdate = false; var logInfo = _headerAccessor.Get(ptr => ptr->Journal); if (logInfo.JournalFilesCount == 0) { _journalIndex = logInfo.LastSyncedJournal; return false; } var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1; if (_env.Options.IncrementalBackupEnabled == false) { // we want to check that we cleanup old log files if they aren't needed // this is more just to be safe than anything else, they shouldn't be there. var unusedfiles = oldestLogFileStillInUse; while (true) { unusedfiles--; if (_env.Options.TryDeleteJournal(unusedfiles) == false) break; } } var lastSyncedTransactionId = logInfo.LastSyncedTransactionId; var journalFiles = new List<JournalFile>(); long lastSyncedTxId = -1; long lastSyncedJournal = logInfo.LastSyncedJournal; uint lastShippedTxCrc = 0; for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++) { using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber))) using (var pager = _env.Options.OpenJournalPager(journalNumber)) { RecoverCurrentJournalSize(pager); var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader; var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader); journalReader.RecoverAndValidate(_env.Options); var pagesToWrite = journalReader .TransactionPageTranslation .Select(kvp => recoveryPager.Read(kvp.Value.JournalPos)) .OrderBy(x => x.PageNumber) .ToList(); var lastReadHeaderPtr = journalReader.LastTransactionHeader; if (lastReadHeaderPtr != null) { if (pagesToWrite.Count > 0) ApplyPagesToDataFileFromJournal(pagesToWrite); *txHeader = *lastReadHeaderPtr; lastSyncedTxId = txHeader->TransactionId; lastShippedTxCrc = txHeader->Crc; lastSyncedJournal = journalNumber; } if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal) { var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize); var jrnlFile = new JournalFile(jrnlWriter, journalNumber); jrnlFile.InitFrom(journalReader); jrnlFile.AddRef(); // creator reference - write ahead log journalFiles.Add(jrnlFile); } if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions { requireHeaderUpdate = true; break; } } } Shipper.SetPreviousTransaction(lastSyncedTxId, lastShippedTxCrc); _files = _files.AppendRange(journalFiles); Debug.Assert(lastSyncedTxId >= 0); Debug.Assert(lastSyncedJournal >= 0); _journalIndex = lastSyncedJournal; _headerAccessor.Modify( header => { header->Journal.LastSyncedJournal = lastSyncedJournal; header->Journal.LastSyncedTransactionId = lastSyncedTxId; header->Journal.CurrentJournal = lastSyncedJournal; header->Journal.JournalFilesCount = _files.Count; header->IncrementalBackup.LastCreatedJournal = _journalIndex; header->PreviousTransactionCrc = lastShippedTxCrc; }); CleanupInvalidJournalFiles(lastSyncedJournal); CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal); if (_files.Count > 0) { var lastFile = _files.Last(); if (lastFile.AvailablePages >= 2) // it must have at least one page for the next transaction header and one page for data CurrentFile = lastFile; } return requireHeaderUpdate; }
public long ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null) { infoNotify = infoNotify ?? (s => { }); if (env.Options.IncrementalBackupEnabled == false) throw new InvalidOperationException("Incremental backup is disabled for this storage"); long numberOfBackedUpPages = 0; var copier = new DataCopier(AbstractPager.PageSize * 16); var backupSuccess = true; long lastWrittenLogPage = -1; long lastWrittenLogFile = -1; using (var file = new FileStream(backupPath, FileMode.Create)) using (var package = new ZipArchive(file, ZipArchiveMode.Create)) { IncrementalBackupInfo backupInfo; using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) { backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup); if (env.Journal.CurrentFile != null) { lastWrittenLogFile = env.Journal.CurrentFile.Number; lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition; } // txw.Commit(); intentionally not committing } using (env.NewTransaction(TransactionFlags.Read)) { var usedJournals = new List<JournalFile>(); try { long lastBackedUpPage = -1; long lastBackedUpFile = -1; var firstJournalToBackup = backupInfo.LastBackedUpJournal; if (firstJournalToBackup == -1) firstJournalToBackup = 0; // first time that we do incremental backup for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++) { var num = journalNum; var journalFile = env.Journal.Files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use if (journalFile == null) { long journalSize; using (var pager = env.Options.OpenJournalPager(journalNum)) { journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize); } journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum); } journalFile.AddRef(); usedJournals.Add(journalFile); var startBackupAt = 0L; var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages; if (journalFile.Number == backupInfo.LastBackedUpJournal) { startBackupAt = backupInfo.LastBackedUpJournalPage + 1; pagesToCopy -= startBackupAt; } if (startBackupAt >= journalFile.JournalWriter.NumberOfAllocatedPages) // nothing to do here continue; var part = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalNum), compression); Debug.Assert(part != null); if (journalFile.Number == lastWrittenLogFile) pagesToCopy -= (journalFile.JournalWriter.NumberOfAllocatedPages - lastWrittenLogPage); using (var stream = part.Open()) { copier.ToStream(journalFile, startBackupAt, pagesToCopy, stream); infoNotify(string.Format("Voron Incr copy journal number {0}", num)); } lastBackedUpFile = journalFile.Number; if (journalFile.Number == backupInfo.LastCreatedJournal) { lastBackedUpPage = startBackupAt + pagesToCopy - 1; // we used all of this file, so the next backup should start in the next file if (lastBackedUpPage == (journalFile.JournalWriter.NumberOfAllocatedPages - 1)) { lastBackedUpPage = -1; lastBackedUpFile++; } } numberOfBackedUpPages += pagesToCopy; } //Debug.Assert(lastBackedUpPage != -1); env.HeaderAccessor.Modify(header => { header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile; header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage; }); } catch (Exception) { backupSuccess = false; throw; } finally { foreach (var jrnl in usedJournals) { if (backupSuccess) // if backup succeeded we can remove journals { if (jrnl.Number < lastWrittenLogFile) // prevent deletion of the current journal and journals with a greater number { jrnl.DeleteOnClose = true; } } jrnl.Release(); } } infoNotify(string.Format("Voron Incr Backup total {0} pages", numberOfBackedUpPages)); return numberOfBackedUpPages; } } }
public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null, Action backupStarted = null) { infoNotify = infoNotify ?? (s => { }); var dataPager = env.Options.DataPager; var copier = new DataCopier(AbstractPager.PageSize * 16); Transaction txr = null; try { infoNotify("Voron copy headers"); using (var file = new FileStream(backupPath, FileMode.Create)) { using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen:true)) { long allocatedPages; ImmutableAppendOnlyList<JournalFile> files; // thread safety copy var usedJournals = new List<JournalFile>(); long lastWrittenLogPage = -1; long lastWrittenLogFile = -1; using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely { txr = env.NewTransaction(TransactionFlags.Read); // now have snapshot view allocatedPages = dataPager.NumberOfAllocatedPages; Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2); VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options); // journal files snapshot files = env.Journal.Files; JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal); for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1; journalNum <= journalInfo.CurrentJournal; journalNum++) { var journalFile = files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use if (journalFile == null) { long journalSize; using (var pager = env.Options.OpenJournalPager(journalNum)) { journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize); } journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum); } journalFile.AddRef(); usedJournals.Add(journalFile); } if (env.Journal.CurrentFile != null) { lastWrittenLogFile = env.Journal.CurrentFile.Number; lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1; } // txw.Commit(); intentionally not committing } if (backupStarted != null) backupStarted(); // data file backup var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression); Debug.Assert(dataPart != null); if (allocatedPages > 0) //only true if dataPager is still empty at backup start { using (var dataStream = dataPart.Open()) { // now can copy everything else var firstDataPage = dataPager.Read(0); copier.ToStream(firstDataPage.Base, AbstractPager.PageSize * allocatedPages, dataStream); } } try { foreach (var journalFile in usedJournals) { var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression); Debug.Assert(journalPart != null); var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages; if (journalFile.Number == lastWrittenLogFile) pagesToCopy = lastWrittenLogPage + 1; using (var stream = journalPart.Open()) { copier.ToStream(journalFile, 0, pagesToCopy, stream); infoNotify(string.Format("Voron copy journal file {0} ", journalFile)); } } } finally { foreach (var journalFile in usedJournals) { journalFile.Release(); } } } file.Flush(true); // make sure that we fully flushed to disk } } finally { if (txr != null) txr.Dispose(); } infoNotify(string.Format("Voron backup db finished")); }
public void ApplyLogsToDataFile(long oldestActiveTransaction, Transaction transaction = null) { bool locked = false; if (_flushingSemaphore.IsWriteLockHeld == false) { if (_flushingSemaphore.TryEnterWriteLock(Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30)) == false) { throw new TimeoutException("Could not acquire the write lock in 30 seconds"); } locked = true; } try { var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite; var journalSnapshots = _waj.Files.Select(x => x.GetSnapshot()) .OrderBy(x => x.Number) .ToList(); if (journalSnapshots.Count == 0) { return; // nothing to do } Debug.Assert(journalSnapshots.First().Number >= _lastSyncedJournal); var pagesToWrite = GetPagesFromJournals(oldestActiveTransaction, journalSnapshots); if (pagesToWrite.Count == 0) { return; } try { ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx); } catch (DiskFullException diskFullEx) { _waj._env.HandleDataDiskFullException(diskFullEx); return; } var unusedJournals = GetAlreadyHandledJournalFiles(journalSnapshots); using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite)) { _lastFlushedJournal = _waj.Files.First(x => x.Number == _lastSyncedJournal); RemoveUnusedJournalsIfNeeded(unusedJournals); if (_waj._files.Count == 0) { _waj.CurrentFile = null; } FreeScratchPages(unusedJournals, txw); var hasSynced = false; if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit || DateTime.Now - _lastDataFileSyncTime > DelayedDataFileSynchronizationTimeLimit) { SyncDataFileWrites(oldestActiveTransaction); hasSynced = true; } if (txw != null) { txw.Commit(); } if (hasSynced) { OnApplyLogsToDataFileFinished(_lastSyncedTransactionId); } } } finally { if (locked) { _flushingSemaphore.ExitWriteLock(); } } }
public void ApplyLogsToDataFile(long oldestActiveTransaction, CancellationToken token, Transaction transaction = null) { bool locked = false; if (_flushingSemaphore.IsWriteLockHeld == false) { if (_flushingSemaphore.TryEnterWriteLock(Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30)) == false) throw new TimeoutException("Could not acquire the write lock in 30 seconds"); locked = true; } try { if (token.IsCancellationRequested) return; var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite; var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList(); if (jrnls.Count == 0) return; // nothing to do Debug.Assert(jrnls.First().Number >= _lastSyncedJournal); var pagesToWrite = new Dictionary<long, JournalFile.PagePosition>(); long lastProcessedJournal = -1; long previousJournalMaxTransactionId = -1; long lastFlushedTransactionId = -1; foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal)) { var currentJournalMaxTransactionId = -1L; foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(journalFile.LastTransaction)) { if (oldestActiveTransaction != 0 && pagePosition.Value.TransactionId >= oldestActiveTransaction) { // we cannot write this yet, there is a read transaction that might be looking at this // however, we _aren't_ going to be writing this to the data file, since that would be a // waste, we would just overwrite that value in the next flush anyway JournalFile.PagePosition existingPagePosition; if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) && pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber) { // remove the page only when it comes from the same journal // otherwise we can damage the journal's page translation table (PTT) // because the existing overwrite in a next journal can be filtered out // so we wouldn't write any page to the data file pagesToWrite.Remove(pagePosition.Key); } continue; } if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId) continue; currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId); if (currentJournalMaxTransactionId < previousJournalMaxTransactionId) throw new InvalidOperationException( "Journal applicator read beyond the oldest active transaction in the next journal file. " + "This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId + ", previous journal max ix id: " + previousJournalMaxTransactionId + ", oldest active transaction: " + oldestActiveTransaction); lastProcessedJournal = journalFile.Number; pagesToWrite[pagePosition.Key] = pagePosition.Value; lastFlushedTransactionId = currentJournalMaxTransactionId; } if (currentJournalMaxTransactionId == -1L) continue; previousJournalMaxTransactionId = currentJournalMaxTransactionId; } if (pagesToWrite.Count == 0) return; try { ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx); } catch (DiskFullException diskFullEx) { _waj._env.HandleDataDiskFullException(diskFullEx); return; } var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId); foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number))) { _journalsToDelete.Add(unused.Number, unused); } using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite)) { _lastSyncedJournal = lastProcessedJournal; _lastSyncedTransactionId = lastFlushedTransactionId; _lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal); if (unusedJournals.Count > 0) { var lastUnusedJournalNumber = unusedJournals.Last().Number; _waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber, new List<JournalFile>()); } if (_waj._files.Count == 0) _waj.CurrentFile = null; FreeScratchPages(unusedJournals, txw); if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit || DateTime.Now - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit) { _waj._dataPager.Sync(); UpdateFileHeaderAfterDataFileSync(_lastFlushedJournal, oldestActiveTransaction); foreach (var toDelete in _journalsToDelete.Values) { if (_waj._env.Options.IncrementalBackupEnabled == false) toDelete.DeleteOnClose = true; toDelete.Release(); } _journalsToDelete.Clear(); _totalWrittenButUnsyncedBytes = 0; _lastDataFileSyncTime = DateTime.Now; } if (txw != null) txw.Commit(); } } finally { if (locked) _flushingSemaphore.ExitWriteLock(); } }
internal static JournalFile GetJournalFile(StorageEnvironment env, long journalNum, IncrementalBackupInfo backupInfo) { var journalFile = env.Journal.Files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use if (journalFile != null) { journalFile.AddRef(); return journalFile; } try { using (var pager = env.Options.OpenJournalPager(journalNum)) { long journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize); journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum); journalFile.AddRef(); return journalFile; } } catch (Exception e) { if (backupInfo.LastBackedUpJournal == -1 && journalNum == 0 && e.Message.StartsWith("No such journal")) { throw new InvalidOperationException("The first incremental backup creation failed because the first journal file " + StorageEnvironmentOptions.JournalName(journalNum) + " was not found. " + "Did you turn on the incremental backup feature after initializing the storage? " + "In order to create backups incrementally the storage must be created with IncrementalBackupEnabled option set to 'true'.", e); } throw; } }