internal static void CopyHeaders(CompressionLevel compression, ZipArchive package, DataCopier copier, StorageEnvironmentOptions storageEnvironmentOptions) { foreach (var headerFileName in HeaderAccessor.HeaderFileNames) { var header = stackalloc FileHeader[1]; if (!storageEnvironmentOptions.ReadHeader(headerFileName, header)) continue; var headerPart = package.CreateEntry(headerFileName, compression); Debug.Assert(headerPart != null); using (var headerStream = headerPart.Open()) { copier.ToStream((byte*)header, sizeof(FileHeader), headerStream); } } }
public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null, Action backupStarted = null) { if (env.Options.IncrementalBackupEnabled == false) throw new InvalidOperationException("Incremental backup is disabled for this storage"); var pageNumberToPageInScratch = new Dictionary<long, long>(); if (infoNotify == null) infoNotify = str => { }; var toDispose = new List<IDisposable>(); try { IncrementalBackupInfo backupInfo; long lastWrittenLogPage = -1; long lastWrittenLogFile = -1; using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) { backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup); if (env.Journal.CurrentFile != null) { lastWrittenLogFile = env.Journal.CurrentFile.Number; lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition; } //txw.Commit(); - intentionally not committing } if (backupStarted != null) backupStarted(); infoNotify("Voron - reading storage journals for snapshot pages"); var lastBackedUpFile = backupInfo.LastBackedUpJournal; var lastBackedUpPage = backupInfo.LastBackedUpJournalPage; var firstJournalToBackup = backupInfo.LastBackedUpJournal; if (firstJournalToBackup == -1) firstJournalToBackup = 0; // first time that we do incremental backup var lastTransaction = new TransactionHeader { TransactionId = -1 }; var recoveryPager = env.Options.CreateScratchPager("min-inc-backup.scratch"); toDispose.Add(recoveryPager); int recoveryPage = 0; for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++) { lastBackedUpFile = journalNum; var journalFile = IncrementalBackup.GetJournalFile(env, journalNum, backupInfo); try { using (var filePager = env.Options.OpenJournalPager(journalNum)) { var reader = new JournalReader(filePager, recoveryPager, 0, null, recoveryPage); reader.MaxPageToRead = lastBackedUpPage = journalFile.JournalWriter.NumberOfAllocatedPages; if (journalNum == lastWrittenLogFile) // set the last part of the log file we'll be reading reader.MaxPageToRead = lastBackedUpPage = lastWrittenLogPage; if (lastBackedUpPage == journalFile.JournalWriter.NumberOfAllocatedPages) // past the file size { // move to the next lastBackedUpPage = -1; lastBackedUpFile++; } if (journalNum == backupInfo.LastBackedUpJournal) // continue from last backup reader.SetStartPage(backupInfo.LastBackedUpJournalPage); TransactionHeader* lastJournalTxHeader = null; while (reader.ReadOneTransaction(env.Options)) { // read all transactions here lastJournalTxHeader = reader.LastTransactionHeader; } if (lastJournalTxHeader != null) lastTransaction = *lastJournalTxHeader; recoveryPage = reader.RecoveryPage; foreach (var pagePosition in reader.TransactionPageTranslation) { var pageInJournal = pagePosition.Value.JournalPos; var page = recoveryPager.Read(pageInJournal); pageNumberToPageInScratch[pagePosition.Key] = pageInJournal; if (page.IsOverflow) { var numberOfOverflowPages = recoveryPager.GetNumberOfOverflowPages(page.OverflowSize); for (int i = 1; i < numberOfOverflowPages; i++) pageNumberToPageInScratch.Remove(page.PageNumber + i); } } } } finally { journalFile.Release(); } } if (pageNumberToPageInScratch.Count == 0) { infoNotify("Voron - no changes since last backup, nothing to do"); return; } infoNotify("Voron - started writing snapshot file."); if (lastTransaction.TransactionId == -1) throw new InvalidOperationException("Could not find any transactions in the journals, but found pages to write? That ain't right."); // it is possible that we merged enough transactions so the _merged_ output is too large for us. // Voron limit transactions to about 4GB each. That means that we can't just merge all transactions // blindly, for fear of hitting this limit. So we need to split things. // We are also limited to about 8 TB of data in general before we literally can't fit the number of pages into // pageNumberToPageInScratch even theoretically. // We're fine with saying that you need to run min inc backup before you hit 8 TB in your increment, so that works for now. // We are also going to use env.Options.MaxScratchBufferSize to set the actual transaction limit here, to avoid issues // down the road and to limit how big a single transaction can be before the theoretical 4GB limit. var nextJournalNum = lastBackedUpFile; using (var file = new FileStream(backupPath, FileMode.Create)) { using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true)) { var copier = new DataCopier(AbstractPager.PageSize * 16); var finalPager = env.Options.CreateScratchPager("min-inc-backup-final.scratch"); toDispose.Add(finalPager); finalPager.EnsureContinuous(null, 0, 1);//txHeader foreach (var partition in Partition(pageNumberToPageInScratch.Values, env.Options.MaxNumberOfPagesInMergedTransaction)) { int totalNumberOfPages = 0; int overflowPages = 0; int start = 1; foreach (var pageNum in partition) { var p = recoveryPager.Read(pageNum); var size = 1; if (p.IsOverflow) { size = recoveryPager.GetNumberOfOverflowPages(p.OverflowSize); overflowPages += (size - 1); } totalNumberOfPages += size; finalPager.EnsureContinuous(null, start, size); //maybe increase size Memory.Copy(finalPager.AcquirePagePointer(start), p.Base, size * AbstractPager.PageSize); start += size; } var txPage = finalPager.AcquirePagePointer(0); UnmanagedMemory.Set(txPage, 0, AbstractPager.PageSize); var txHeader = (TransactionHeader*)txPage; txHeader->HeaderMarker = Constants.TransactionHeaderMarker; txHeader->FreeSpace = lastTransaction.FreeSpace; txHeader->Root = lastTransaction.Root; txHeader->OverflowPageCount = overflowPages; txHeader->PageCount = totalNumberOfPages - overflowPages; txHeader->PreviousTransactionCrc = lastTransaction.PreviousTransactionCrc; txHeader->TransactionId = lastTransaction.TransactionId; txHeader->NextPageNumber = lastTransaction.NextPageNumber; txHeader->LastPageNumber = lastTransaction.LastPageNumber; txHeader->TxMarker = TransactionMarker.Commit | TransactionMarker.Merged; txHeader->Compressed = false; txHeader->UncompressedSize = txHeader->CompressedSize = totalNumberOfPages * AbstractPager.PageSize; txHeader->Crc = Crc.Value(finalPager.AcquirePagePointer(1), 0, totalNumberOfPages * AbstractPager.PageSize); var entry = package.CreateEntry(string.Format("{0:D19}.merged-journal", nextJournalNum), compression); nextJournalNum++; using (var stream = entry.Open()) { copier.ToStream(finalPager.AcquirePagePointer(0), (totalNumberOfPages + 1) * AbstractPager.PageSize, stream); } } } file.Flush(true);// make sure we hit the disk and stay there } env.HeaderAccessor.Modify(header => { header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile; header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage; }); } finally { foreach (var disposable in toDispose) disposable.Dispose(); } }
public long ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null) { infoNotify = infoNotify ?? (s => { }); if (env.Options.IncrementalBackupEnabled == false) throw new InvalidOperationException("Incremental backup is disabled for this storage"); long numberOfBackedUpPages = 0; var copier = new DataCopier(AbstractPager.PageSize * 16); var backupSuccess = true; long lastWrittenLogPage = -1; long lastWrittenLogFile = -1; using (var file = new FileStream(backupPath, FileMode.Create)) using (var package = new ZipArchive(file, ZipArchiveMode.Create)) { IncrementalBackupInfo backupInfo; using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) { backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup); if (env.Journal.CurrentFile != null) { lastWrittenLogFile = env.Journal.CurrentFile.Number; lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition; } // txw.Commit(); intentionally not committing } using (env.NewTransaction(TransactionFlags.Read)) { var usedJournals = new List<JournalFile>(); try { long lastBackedUpPage = -1; long lastBackedUpFile = -1; var firstJournalToBackup = backupInfo.LastBackedUpJournal; if (firstJournalToBackup == -1) firstJournalToBackup = 0; // first time that we do incremental backup for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++) { var num = journalNum; var journalFile = env.Journal.Files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use if (journalFile == null) { long journalSize; using (var pager = env.Options.OpenJournalPager(journalNum)) { journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize); } journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum); } journalFile.AddRef(); usedJournals.Add(journalFile); var startBackupAt = 0L; var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages; if (journalFile.Number == backupInfo.LastBackedUpJournal) { startBackupAt = backupInfo.LastBackedUpJournalPage + 1; pagesToCopy -= startBackupAt; } if (startBackupAt >= journalFile.JournalWriter.NumberOfAllocatedPages) // nothing to do here continue; var part = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalNum), compression); Debug.Assert(part != null); if (journalFile.Number == lastWrittenLogFile) pagesToCopy -= (journalFile.JournalWriter.NumberOfAllocatedPages - lastWrittenLogPage); using (var stream = part.Open()) { copier.ToStream(journalFile, startBackupAt, pagesToCopy, stream); infoNotify(string.Format("Voron Incr copy journal number {0}", num)); } lastBackedUpFile = journalFile.Number; if (journalFile.Number == backupInfo.LastCreatedJournal) { lastBackedUpPage = startBackupAt + pagesToCopy - 1; // we used all of this file, so the next backup should start in the next file if (lastBackedUpPage == (journalFile.JournalWriter.NumberOfAllocatedPages - 1)) { lastBackedUpPage = -1; lastBackedUpFile++; } } numberOfBackedUpPages += pagesToCopy; } //Debug.Assert(lastBackedUpPage != -1); env.HeaderAccessor.Modify(header => { header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile; header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage; }); } catch (Exception) { backupSuccess = false; throw; } finally { foreach (var jrnl in usedJournals) { if (backupSuccess) // if backup succeeded we can remove journals { if (jrnl.Number < lastWrittenLogFile) // prevent deletion of the current journal and journals with a greater number { jrnl.DeleteOnClose = true; } } jrnl.Release(); } } infoNotify(string.Format("Voron Incr Backup total {0} pages", numberOfBackedUpPages)); return numberOfBackedUpPages; } } }
public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null) { infoNotify = infoNotify ?? (s => { }); var dataPager = env.Options.DataPager; var copier = new DataCopier(AbstractPager.PageSize * 16); Transaction txr = null; try { infoNotify("Voron copy headers"); using (var file = new FileStream(backupPath, FileMode.Create)) using (var package = new ZipArchive(file, ZipArchiveMode.Create)) { long allocatedPages; ImmutableAppendOnlyList<JournalFile> files; // thread safety copy long lastWrittenLogPage = -1; long lastWrittenLogFile = -1; using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely { txr = env.NewTransaction(TransactionFlags.Read); // now have snapshot view allocatedPages = dataPager.NumberOfAllocatedPages; Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2); VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options); // journal files snapshot files = env.Journal.Files; foreach (var journalFile in files) { journalFile.AddRef(); } if (env.Journal.CurrentFile != null) { lastWrittenLogFile = env.Journal.CurrentFile.Number; lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1; } // txw.Commit(); intentionally not committing } // data file backup var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression); Debug.Assert(dataPart != null); if (allocatedPages > 0) //only true if dataPager is still empty at backup start { using (var dataStream = dataPart.Open()) { // now can copy everything else var firstDataPage = dataPager.Read(0); copier.ToStream(firstDataPage.Base, AbstractPager.PageSize*allocatedPages, dataStream); } } try { foreach (var journalFile in files) { var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression); Debug.Assert(journalPart != null); var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages; if (journalFile.Number == lastWrittenLogFile) pagesToCopy = lastWrittenLogPage + 1; using (var stream = journalPart.Open()) { copier.ToStream(journalFile, 0, pagesToCopy, stream); infoNotify(string.Format("Voron copy journal file {0} ", journalFile)); } } } finally { foreach (var journalFile in files) { journalFile.Release(); } } } } finally { if (txr != null) txr.Dispose(); } infoNotify(string.Format("Voron backup db finished")); }
public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null, Action backupStarted = null) { infoNotify = infoNotify ?? (s => { }); var dataPager = env.Options.DataPager; var copier = new DataCopier(AbstractPager.PageSize * 16); Transaction txr = null; try { infoNotify("Voron copy headers"); using (var file = new FileStream(backupPath, FileMode.Create)) { using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen:true)) { long allocatedPages; ImmutableAppendOnlyList<JournalFile> files; // thread safety copy var usedJournals = new List<JournalFile>(); long lastWrittenLogPage = -1; long lastWrittenLogFile = -1; using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely { txr = env.NewTransaction(TransactionFlags.Read); // now have snapshot view allocatedPages = dataPager.NumberOfAllocatedPages; Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2); VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options); // journal files snapshot files = env.Journal.Files; JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal); for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1; journalNum <= journalInfo.CurrentJournal; journalNum++) { var journalFile = files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use if (journalFile == null) { long journalSize; using (var pager = env.Options.OpenJournalPager(journalNum)) { journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize); } journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum); } journalFile.AddRef(); usedJournals.Add(journalFile); } if (env.Journal.CurrentFile != null) { lastWrittenLogFile = env.Journal.CurrentFile.Number; lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1; } // txw.Commit(); intentionally not committing } if (backupStarted != null) backupStarted(); // data file backup var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression); Debug.Assert(dataPart != null); if (allocatedPages > 0) //only true if dataPager is still empty at backup start { using (var dataStream = dataPart.Open()) { // now can copy everything else var firstDataPage = dataPager.Read(0); copier.ToStream(firstDataPage.Base, AbstractPager.PageSize * allocatedPages, dataStream); } } try { foreach (var journalFile in usedJournals) { var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression); Debug.Assert(journalPart != null); var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages; if (journalFile.Number == lastWrittenLogFile) pagesToCopy = lastWrittenLogPage + 1; using (var stream = journalPart.Open()) { copier.ToStream(journalFile, 0, pagesToCopy, stream); infoNotify(string.Format("Voron copy journal file {0} ", journalFile)); } } } finally { foreach (var journalFile in usedJournals) { journalFile.Release(); } } } file.Flush(true); // make sure that we fully flushed to disk } } finally { if (txr != null) txr.Dispose(); } infoNotify(string.Format("Voron backup db finished")); }