public void InitFrom(JournalReader journalReader) { _writePage = journalReader.NextWritePage; }
public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null, Action backupStarted = null) { if (env.Options.IncrementalBackupEnabled == false) throw new InvalidOperationException("Incremental backup is disabled for this storage"); var pageNumberToPageInScratch = new Dictionary<long, long>(); if (infoNotify == null) infoNotify = str => { }; var toDispose = new List<IDisposable>(); try { IncrementalBackupInfo backupInfo; long lastWrittenLogPage = -1; long lastWrittenLogFile = -1; using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) { backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup); if (env.Journal.CurrentFile != null) { lastWrittenLogFile = env.Journal.CurrentFile.Number; lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition; } //txw.Commit(); - intentionally not committing } if (backupStarted != null) backupStarted(); infoNotify("Voron - reading storage journals for snapshot pages"); var lastBackedUpFile = backupInfo.LastBackedUpJournal; var lastBackedUpPage = backupInfo.LastBackedUpJournalPage; var firstJournalToBackup = backupInfo.LastBackedUpJournal; if (firstJournalToBackup == -1) firstJournalToBackup = 0; // first time that we do incremental backup var lastTransaction = new TransactionHeader { TransactionId = -1 }; var recoveryPager = env.Options.CreateScratchPager("min-inc-backup.scratch"); toDispose.Add(recoveryPager); int recoveryPage = 0; for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++) { lastBackedUpFile = journalNum; var journalFile = IncrementalBackup.GetJournalFile(env, journalNum, backupInfo); try { using (var filePager = env.Options.OpenJournalPager(journalNum)) { var reader = new JournalReader(filePager, recoveryPager, 0, null, recoveryPage); reader.MaxPageToRead = lastBackedUpPage = journalFile.JournalWriter.NumberOfAllocatedPages; if (journalNum == lastWrittenLogFile) // set the last part of the log file we'll be reading reader.MaxPageToRead = lastBackedUpPage = lastWrittenLogPage; if (lastBackedUpPage == journalFile.JournalWriter.NumberOfAllocatedPages) // past the file size { // move to the next lastBackedUpPage = -1; lastBackedUpFile++; } if (journalNum == backupInfo.LastBackedUpJournal) // continue from last backup reader.SetStartPage(backupInfo.LastBackedUpJournalPage); TransactionHeader* lastJournalTxHeader = null; while (reader.ReadOneTransaction(env.Options)) { // read all transactions here lastJournalTxHeader = reader.LastTransactionHeader; } if (lastJournalTxHeader != null) lastTransaction = *lastJournalTxHeader; recoveryPage = reader.RecoveryPage; foreach (var pagePosition in reader.TransactionPageTranslation) { var pageInJournal = pagePosition.Value.JournalPos; var page = recoveryPager.Read(pageInJournal); pageNumberToPageInScratch[pagePosition.Key] = pageInJournal; if (page.IsOverflow) { var numberOfOverflowPages = recoveryPager.GetNumberOfOverflowPages(page.OverflowSize); for (int i = 1; i < numberOfOverflowPages; i++) pageNumberToPageInScratch.Remove(page.PageNumber + i); } } } } finally { journalFile.Release(); } } if (pageNumberToPageInScratch.Count == 0) { infoNotify("Voron - no changes since last backup, nothing to do"); return; } infoNotify("Voron - started writing snapshot file."); if (lastTransaction.TransactionId == -1) throw new InvalidOperationException("Could not find any transactions in the journals, but found pages to write? That ain't right."); // it is possible that we merged enough transactions so the _merged_ output is too large for us. // Voron limit transactions to about 4GB each. That means that we can't just merge all transactions // blindly, for fear of hitting this limit. So we need to split things. // We are also limited to about 8 TB of data in general before we literally can't fit the number of pages into // pageNumberToPageInScratch even theoretically. // We're fine with saying that you need to run min inc backup before you hit 8 TB in your increment, so that works for now. // We are also going to use env.Options.MaxScratchBufferSize to set the actual transaction limit here, to avoid issues // down the road and to limit how big a single transaction can be before the theoretical 4GB limit. var nextJournalNum = lastBackedUpFile; using (var file = new FileStream(backupPath, FileMode.Create)) { using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true)) { var copier = new DataCopier(AbstractPager.PageSize * 16); var finalPager = env.Options.CreateScratchPager("min-inc-backup-final.scratch"); toDispose.Add(finalPager); finalPager.EnsureContinuous(null, 0, 1);//txHeader foreach (var partition in Partition(pageNumberToPageInScratch.Values, env.Options.MaxNumberOfPagesInMergedTransaction)) { int totalNumberOfPages = 0; int overflowPages = 0; int start = 1; foreach (var pageNum in partition) { var p = recoveryPager.Read(pageNum); var size = 1; if (p.IsOverflow) { size = recoveryPager.GetNumberOfOverflowPages(p.OverflowSize); overflowPages += (size - 1); } totalNumberOfPages += size; finalPager.EnsureContinuous(null, start, size); //maybe increase size Memory.Copy(finalPager.AcquirePagePointer(start), p.Base, size * AbstractPager.PageSize); start += size; } var txPage = finalPager.AcquirePagePointer(0); UnmanagedMemory.Set(txPage, 0, AbstractPager.PageSize); var txHeader = (TransactionHeader*)txPage; txHeader->HeaderMarker = Constants.TransactionHeaderMarker; txHeader->FreeSpace = lastTransaction.FreeSpace; txHeader->Root = lastTransaction.Root; txHeader->OverflowPageCount = overflowPages; txHeader->PageCount = totalNumberOfPages - overflowPages; txHeader->PreviousTransactionCrc = lastTransaction.PreviousTransactionCrc; txHeader->TransactionId = lastTransaction.TransactionId; txHeader->NextPageNumber = lastTransaction.NextPageNumber; txHeader->LastPageNumber = lastTransaction.LastPageNumber; txHeader->TxMarker = TransactionMarker.Commit | TransactionMarker.Merged; txHeader->Compressed = false; txHeader->UncompressedSize = txHeader->CompressedSize = totalNumberOfPages * AbstractPager.PageSize; txHeader->Crc = Crc.Value(finalPager.AcquirePagePointer(1), 0, totalNumberOfPages * AbstractPager.PageSize); var entry = package.CreateEntry(string.Format("{0:D19}.merged-journal", nextJournalNum), compression); nextJournalNum++; using (var stream = entry.Open()) { copier.ToStream(finalPager.AcquirePagePointer(0), (totalNumberOfPages + 1) * AbstractPager.PageSize, stream); } } } file.Flush(true);// make sure we hit the disk and stay there } env.HeaderAccessor.Modify(header => { header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile; header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage; }); } finally { foreach (var disposable in toDispose) disposable.Dispose(); } }
private void Restore(StorageEnvironment env, string singleBackupFile) { using (env.Journal.Applicator.TakeFlushingLock()) { using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) { using (env.Options.AllowManualFlushing()) { env.FlushLogToDataFile(txw); } using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read)) { if (package.Entries.Count == 0) return; var toDispose = new List<IDisposable>(); var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName; try { TransactionHeader* lastTxHeader = null; var pagesToWrite = new Dictionary<long, Func<Page>>(); long journalNumber = -1; foreach (var entry in package.Entries) { switch (Path.GetExtension(entry.Name)) { case ".journal": var jounalFileName = Path.Combine(tempDir, entry.Name); using (var output = new FileStream(jounalFileName, FileMode.Create)) using (var input = entry.Open()) { output.Position = output.Length; input.CopyTo(output); } var pager = new Win32MemoryMapPager(jounalFileName); toDispose.Add(pager); if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false) { throw new InvalidOperationException("Cannot parse journal file number"); } var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber))); toDispose.Add(recoveryPager); var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader); while (reader.ReadOneTransaction(env.Options)) { lastTxHeader = reader.LastTransactionHeader; } foreach (var translation in reader.TransactionPageTranslation) { var pageInJournal = translation.Value.JournalPos; pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal); } break; default: throw new InvalidOperationException("Unknown file, cannot restore: " + entry); } } var sortedPages = pagesToWrite.OrderBy(x => x.Key) .Select(x => x.Value()) .ToList(); var last = sortedPages.Last(); env.Options.DataPager.EnsureContinuous(txw, last.PageNumber, last.IsOverflow ? env.Options.DataPager.GetNumberOfOverflowPages( last.OverflowSize) : 1); foreach (var page in sortedPages) { env.Options.DataPager.Write(page); } env.Options.DataPager.Sync(); txw.State.Root = Tree.Open(txw, env._sliceComparer, &lastTxHeader->Root); txw.State.FreeSpaceRoot = Tree.Open(txw, env._sliceComparer, &lastTxHeader->FreeSpace); txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName; txw.State.Root.Name = Constants.RootTreeName; txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1; env.Journal.Clear(txw); txw.Commit(); env.HeaderAccessor.Modify(header => { header->TransactionId = lastTxHeader->TransactionId; header->LastPageNumber = lastTxHeader->LastPageNumber; header->Journal.LastSyncedJournal = journalNumber; header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId; header->Root = lastTxHeader->Root; header->FreeSpace = lastTxHeader->FreeSpace; header->Journal.CurrentJournal = journalNumber + 1; header->Journal.JournalFilesCount = 0; }); } finally { toDispose.ForEach(x => x.Dispose()); try { Directory.Delete(tempDir, true); } catch (Exception) { // just temp dir - ignore it } } } } } }
public bool RecoverDatabase(TransactionHeader* txHeader) { // note, we don't need to do any concurrency here, happens as a single threaded // fashion on db startup var requireHeaderUpdate = false; var logInfo = _headerAccessor.Get(ptr => ptr->Journal); if (logInfo.JournalFilesCount == 0) { _journalIndex = logInfo.LastSyncedJournal; return false; } var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1; if (_env.Options.IncrementalBackupEnabled == false) { // we want to check that we cleanup old log files if they aren't needed // this is more just to be safe than anything else, they shouldn't be there. var unusedfiles = oldestLogFileStillInUse; while (true) { unusedfiles--; if (_env.Options.TryDeleteJournal(unusedfiles) == false) break; } } var lastSyncedTransactionId = logInfo.LastSyncedTransactionId; var journalFiles = new List<JournalFile>(); long lastSyncedTxId = -1; long lastSyncedJournal = logInfo.LastSyncedJournal; uint lastShippedTxCrc = 0; for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++) { using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber))) using (var pager = _env.Options.OpenJournalPager(journalNumber)) { RecoverCurrentJournalSize(pager); var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader; var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader); journalReader.RecoverAndValidate(_env.Options); var pagesToWrite = journalReader .TransactionPageTranslation .Select(kvp => recoveryPager.Read(kvp.Value.JournalPos)) .OrderBy(x => x.PageNumber) .ToList(); var lastReadHeaderPtr = journalReader.LastTransactionHeader; if (lastReadHeaderPtr != null) { if (pagesToWrite.Count > 0) ApplyPagesToDataFileFromJournal(pagesToWrite); *txHeader = *lastReadHeaderPtr; lastSyncedTxId = txHeader->TransactionId; lastShippedTxCrc = txHeader->Crc; lastSyncedJournal = journalNumber; } if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal) { var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize); var jrnlFile = new JournalFile(jrnlWriter, journalNumber); jrnlFile.InitFrom(journalReader); jrnlFile.AddRef(); // creator reference - write ahead log journalFiles.Add(jrnlFile); } if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions { requireHeaderUpdate = true; break; } } } Shipper.SetPreviousTransaction(lastSyncedTxId, lastShippedTxCrc); _files = _files.AppendRange(journalFiles); Debug.Assert(lastSyncedTxId >= 0); Debug.Assert(lastSyncedJournal >= 0); _journalIndex = lastSyncedJournal; _headerAccessor.Modify( header => { header->Journal.LastSyncedJournal = lastSyncedJournal; header->Journal.LastSyncedTransactionId = lastSyncedTxId; header->Journal.CurrentJournal = lastSyncedJournal; header->Journal.JournalFilesCount = _files.Count; header->IncrementalBackup.LastCreatedJournal = _journalIndex; header->PreviousTransactionCrc = lastShippedTxCrc; }); CleanupInvalidJournalFiles(lastSyncedJournal); CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal); if (_files.Count > 0) { var lastFile = _files.Last(); if (lastFile.AvailablePages >= 2) // it must have at least one page for the next transaction header and one page for data CurrentFile = lastFile; } return requireHeaderUpdate; }
public bool RecoverDatabase(TransactionHeader *txHeader) { // note, we don't need to do any concurrency here, happens as a single threaded // fashion on db startup var requireHeaderUpdate = false; var logInfo = _headerAccessor.Get(ptr => ptr->Journal); if (logInfo.JournalFilesCount == 0) { _journalIndex = logInfo.LastSyncedJournal; return(false); } var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1; if (_env.Options.IncrementalBackupEnabled == false) { // we want to check that we cleanup old log files if they aren't needed // this is more just to be safe than anything else, they shouldn't be there. var unusedfiles = oldestLogFileStillInUse; while (true) { unusedfiles--; if (_env.Options.TryDeleteJournal(unusedfiles) == false) { break; } } } var lastSyncedTransactionId = logInfo.LastSyncedTransactionId; var journalFiles = new List <JournalFile>(); long lastSyncedTxId = -1; long lastSyncedJournal = logInfo.LastSyncedJournal; uint lastShippedTxCrc = 0; for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++) { using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber))) using (var pager = _env.Options.OpenJournalPager(journalNumber)) { RecoverCurrentJournalSize(pager); var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader; var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader); journalReader.RecoverAndValidate(_env.Options); var pagesToWrite = journalReader .TransactionPageTranslation .Select(kvp => recoveryPager.Read(kvp.Value.JournalPos)) .OrderBy(x => x.PageNumber) .ToList(); var lastReadHeaderPtr = journalReader.LastTransactionHeader; if (lastReadHeaderPtr != null) { if (pagesToWrite.Count > 0) { ApplyPagesToDataFileFromJournal(pagesToWrite); } *txHeader = *lastReadHeaderPtr; lastSyncedTxId = txHeader->TransactionId; lastShippedTxCrc = txHeader->Crc; lastSyncedJournal = journalNumber; } if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal) { var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize); var jrnlFile = new JournalFile(jrnlWriter, journalNumber); jrnlFile.InitFrom(journalReader); jrnlFile.AddRef(); // creator reference - write ahead log journalFiles.Add(jrnlFile); } if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions { requireHeaderUpdate = true; break; } } } Shipper.SetPreviousTransaction(lastSyncedTxId, lastShippedTxCrc); _files = _files.AppendRange(journalFiles); Debug.Assert(lastSyncedTxId >= 0); Debug.Assert(lastSyncedJournal >= 0); _journalIndex = lastSyncedJournal; _headerAccessor.Modify( header => { header->Journal.LastSyncedJournal = lastSyncedJournal; header->Journal.LastSyncedTransactionId = lastSyncedTxId; header->Journal.CurrentJournal = lastSyncedJournal; header->Journal.JournalFilesCount = _files.Count; header->IncrementalBackup.LastCreatedJournal = _journalIndex; header->PreviousTransactionCrc = lastShippedTxCrc; }); CleanupInvalidJournalFiles(lastSyncedJournal); CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal); if (_files.Count > 0) { var lastFile = _files.Last(); if (lastFile.AvailablePages >= 2) { // it must have at least one page for the next transaction header and one page for data CurrentFile = lastFile; } } return(requireHeaderUpdate); }
public void InitFrom(JournalReader journalReader, TransactionHeader[] transactionHeaders, int transactionsCount) { _writePosIn4Kb = journalReader.Next4Kb; Array.Copy(transactionHeaders, _transactionHeaders, transactionsCount); }
public void InitFrom(JournalReader journalReader, List <TransactionHeader> transactionHeaders) { _writePosIn4Kb = journalReader.Next4Kb; _transactionHeaders = new List <TransactionHeader>(transactionHeaders); }
public void InitFrom(JournalReader journalReader) { _writePosIn4Kb = journalReader.Next4Kb; }