Пример #1
0
        internal static void CopyHeaders(CompressionLevel compression, ZipArchive package, DataCopier copier, StorageEnvironmentOptions storageEnvironmentOptions)
        {
            foreach (var headerFileName in HeaderAccessor.HeaderFileNames)
            {
                var header = stackalloc FileHeader[1];

                if (!storageEnvironmentOptions.ReadHeader(headerFileName, header))
                    continue;

                var headerPart = package.CreateEntry(headerFileName, compression);
                Debug.Assert(headerPart != null);

                using (var headerStream = headerPart.Open())
                {
                    copier.ToStream((byte*)header, sizeof(FileHeader), headerStream);
                }
            }
        }
Пример #2
0
        public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null,
            Action backupStarted = null)
        {
            if (env.Options.IncrementalBackupEnabled == false)
                throw new InvalidOperationException("Incremental backup is disabled for this storage");

            var pageNumberToPageInScratch = new Dictionary<long, long>();
            if (infoNotify == null)
                infoNotify = str => { };
            var toDispose = new List<IDisposable>();
            try
            {
                IncrementalBackupInfo backupInfo;
                long lastWrittenLogPage = -1;
                long lastWrittenLogFile = -1;

                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);

                    if (env.Journal.CurrentFile != null)
                    {
                        lastWrittenLogFile = env.Journal.CurrentFile.Number;
                        lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition;
                    }

                    //txw.Commit(); - intentionally not committing
                }

                if (backupStarted != null)
                    backupStarted();

                infoNotify("Voron - reading storage journals for snapshot pages");

                var lastBackedUpFile = backupInfo.LastBackedUpJournal;
                var lastBackedUpPage = backupInfo.LastBackedUpJournalPage;
                var firstJournalToBackup = backupInfo.LastBackedUpJournal;

                if (firstJournalToBackup == -1)
                    firstJournalToBackup = 0; // first time that we do incremental backup

                var lastTransaction = new TransactionHeader { TransactionId = -1 };

                var recoveryPager = env.Options.CreateScratchPager("min-inc-backup.scratch");
                toDispose.Add(recoveryPager);
                int recoveryPage = 0;
                for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++)
                {
                    lastBackedUpFile = journalNum;
                    var journalFile = IncrementalBackup.GetJournalFile(env, journalNum, backupInfo);
                    try
                    {
                        using (var filePager = env.Options.OpenJournalPager(journalNum))
                        {
                            var reader = new JournalReader(filePager, recoveryPager, 0, null, recoveryPage);
                            reader.MaxPageToRead = lastBackedUpPage = journalFile.JournalWriter.NumberOfAllocatedPages;
                            if (journalNum == lastWrittenLogFile) // set the last part of the log file we'll be reading
                                reader.MaxPageToRead = lastBackedUpPage = lastWrittenLogPage;

                            if (lastBackedUpPage == journalFile.JournalWriter.NumberOfAllocatedPages) // past the file size
                            {
                                // move to the next
                                lastBackedUpPage = -1;
                                lastBackedUpFile++;
                            }

                            if (journalNum == backupInfo.LastBackedUpJournal) // continue from last backup
                                reader.SetStartPage(backupInfo.LastBackedUpJournalPage);
                            TransactionHeader* lastJournalTxHeader = null;
                            while (reader.ReadOneTransaction(env.Options))
                            {
                                // read all transactions here
                                lastJournalTxHeader = reader.LastTransactionHeader;
                            }

                            if (lastJournalTxHeader != null)
                                lastTransaction = *lastJournalTxHeader;

                            recoveryPage = reader.RecoveryPage;

                            foreach (var pagePosition in reader.TransactionPageTranslation)
                            {
                                var pageInJournal = pagePosition.Value.JournalPos;
                                var page = recoveryPager.Read(pageInJournal);
                                pageNumberToPageInScratch[pagePosition.Key] = pageInJournal;
                                if (page.IsOverflow)
                                {
                                    var numberOfOverflowPages = recoveryPager.GetNumberOfOverflowPages(page.OverflowSize);
                                    for (int i = 1; i < numberOfOverflowPages; i++)
                                        pageNumberToPageInScratch.Remove(page.PageNumber + i);
                                }
                            }
                        }
                    }
                    finally
                    {
                        journalFile.Release();
                    }
                }

                if (pageNumberToPageInScratch.Count == 0)
                {
                    infoNotify("Voron - no changes since last backup, nothing to do");
                    return;
                }

                infoNotify("Voron - started writing snapshot file.");

                if (lastTransaction.TransactionId == -1)
                    throw new InvalidOperationException("Could not find any transactions in the journals, but found pages to write? That ain't right.");

                // it is possible that we merged enough transactions so the _merged_ output is too large for us.
                // Voron limit transactions to about 4GB each. That means that we can't just merge all transactions
                // blindly, for fear of hitting this limit. So we need to split things.
                // We are also limited to about 8 TB of data in general before we literally can't fit the number of pages into
                // pageNumberToPageInScratch even theoretically.
                // We're fine with saying that you need to run min inc backup before you hit 8 TB in your increment, so that works for now.
                // We are also going to use env.Options.MaxScratchBufferSize to set the actual transaction limit here, to avoid issues
                // down the road and to limit how big a single transaction can be before the theoretical 4GB limit.

                var nextJournalNum = lastBackedUpFile;
                using (var file = new FileStream(backupPath, FileMode.Create))
                {
                    using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
                    {
                        var copier = new DataCopier(AbstractPager.PageSize * 16);

                        var finalPager = env.Options.CreateScratchPager("min-inc-backup-final.scratch");
                        toDispose.Add(finalPager);
                        finalPager.EnsureContinuous(null, 0, 1);//txHeader

                        foreach (var partition in Partition(pageNumberToPageInScratch.Values, env.Options.MaxNumberOfPagesInMergedTransaction))
                        {
                            int totalNumberOfPages = 0;
                            int overflowPages = 0;
                            int start = 1;
                            foreach (var pageNum in partition)
                            {
                                var p = recoveryPager.Read(pageNum);
                                var size = 1;
                                if (p.IsOverflow)
                                {
                                    size = recoveryPager.GetNumberOfOverflowPages(p.OverflowSize);
                                    overflowPages += (size - 1);
                                }
                                totalNumberOfPages += size;
                                finalPager.EnsureContinuous(null, start, size); //maybe increase size

                                Memory.Copy(finalPager.AcquirePagePointer(start), p.Base, size * AbstractPager.PageSize);

                                start += size;
                            }

                            var txPage = finalPager.AcquirePagePointer(0);
                            UnmanagedMemory.Set(txPage, 0, AbstractPager.PageSize);
                            var txHeader = (TransactionHeader*)txPage;
                            txHeader->HeaderMarker = Constants.TransactionHeaderMarker;
                            txHeader->FreeSpace = lastTransaction.FreeSpace;
                            txHeader->Root = lastTransaction.Root;
                            txHeader->OverflowPageCount = overflowPages;
                            txHeader->PageCount = totalNumberOfPages - overflowPages;
                            txHeader->PreviousTransactionCrc = lastTransaction.PreviousTransactionCrc;
                            txHeader->TransactionId = lastTransaction.TransactionId;
                            txHeader->NextPageNumber = lastTransaction.NextPageNumber;
                            txHeader->LastPageNumber = lastTransaction.LastPageNumber;
                            txHeader->TxMarker = TransactionMarker.Commit | TransactionMarker.Merged;
                            txHeader->Compressed = false;
                            txHeader->UncompressedSize = txHeader->CompressedSize = totalNumberOfPages * AbstractPager.PageSize;

                            txHeader->Crc = Crc.Value(finalPager.AcquirePagePointer(1), 0, totalNumberOfPages * AbstractPager.PageSize);

                            var entry = package.CreateEntry(string.Format("{0:D19}.merged-journal", nextJournalNum), compression);
                            nextJournalNum++;
                            using (var stream = entry.Open())
                            {
                                copier.ToStream(finalPager.AcquirePagePointer(0), (totalNumberOfPages + 1) * AbstractPager.PageSize, stream);
                            }
                        }

                    }
                    file.Flush(true);// make sure we hit the disk and stay there
                }

                env.HeaderAccessor.Modify(header =>
                {
                    header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile;
                    header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage;
                });
            }
            finally
            {
                foreach (var disposable in toDispose)
                    disposable.Dispose();
            }
        }
Пример #3
0
        public long ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
			Action<string> infoNotify = null)
        {
			infoNotify = infoNotify ?? (s => { });

            if (env.Options.IncrementalBackupEnabled == false)
                throw new InvalidOperationException("Incremental backup is disabled for this storage");

            long numberOfBackedUpPages = 0;

            var copier = new DataCopier(AbstractPager.PageSize * 16);
            var backupSuccess = true;

            long lastWrittenLogPage = -1;
            long lastWrittenLogFile = -1;

            using (var file = new FileStream(backupPath, FileMode.Create))
            using (var package = new ZipArchive(file, ZipArchiveMode.Create))
            {
                IncrementalBackupInfo backupInfo;
                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);

                    if (env.Journal.CurrentFile != null)
                    {
                        lastWrittenLogFile = env.Journal.CurrentFile.Number;
                        lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition;
                    }

                    // txw.Commit(); intentionally not committing
                }

                using (env.NewTransaction(TransactionFlags.Read))
                {
                    var usedJournals = new List<JournalFile>();

                    try
                    {
                        long lastBackedUpPage = -1;
                        long lastBackedUpFile = -1;

                        var firstJournalToBackup = backupInfo.LastBackedUpJournal;

                        if (firstJournalToBackup == -1)
                            firstJournalToBackup = 0; // first time that we do incremental backup

                        for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++)
                        {
                            var num = journalNum;

                            var journalFile = env.Journal.Files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
                            if (journalFile == null)
                            {
                                long journalSize;
                                using (var pager = env.Options.OpenJournalPager(journalNum))
                                {
                                    journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
                                }

                                journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
                            }

                            journalFile.AddRef();

                            usedJournals.Add(journalFile);

                            var startBackupAt = 0L;
                            var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
                            if (journalFile.Number == backupInfo.LastBackedUpJournal)
                            {
                                startBackupAt = backupInfo.LastBackedUpJournalPage + 1;
                                pagesToCopy -= startBackupAt;
                            }

                            if (startBackupAt >= journalFile.JournalWriter.NumberOfAllocatedPages) // nothing to do here
                                continue;

                            var part = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalNum), compression);
                            Debug.Assert(part != null);

                            if (journalFile.Number == lastWrittenLogFile)
                                pagesToCopy -= (journalFile.JournalWriter.NumberOfAllocatedPages - lastWrittenLogPage);

                            using (var stream = part.Open())
                            {
                                copier.ToStream(journalFile, startBackupAt, pagesToCopy, stream);
                                infoNotify(string.Format("Voron Incr copy journal number {0}", num));

                            }

                            lastBackedUpFile = journalFile.Number;
                            if (journalFile.Number == backupInfo.LastCreatedJournal)
                            {
                                lastBackedUpPage = startBackupAt + pagesToCopy - 1;
                                // we used all of this file, so the next backup should start in the next file
                                if (lastBackedUpPage == (journalFile.JournalWriter.NumberOfAllocatedPages - 1))
                                {
                                    lastBackedUpPage = -1;
                                    lastBackedUpFile++;
                                }
                            }

                            numberOfBackedUpPages += pagesToCopy;
                        }

                        //Debug.Assert(lastBackedUpPage != -1);

                        env.HeaderAccessor.Modify(header =>
                        {
                            header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile;
                            header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage;
                        });
                    }
                    catch (Exception)
                    {
                        backupSuccess = false;
                        throw;
                    }
                    finally
                    {
                        foreach (var jrnl in usedJournals)
                        {
                            if (backupSuccess) // if backup succeeded we can remove journals
                            {
                                if (jrnl.Number < lastWrittenLogFile) // prevent deletion of the current journal and journals with a greater number
                                {
                                    jrnl.DeleteOnClose = true;
                                }
                            }

                            jrnl.Release();
                        }
                    }
                    infoNotify(string.Format("Voron Incr Backup total {0} pages", numberOfBackedUpPages));
                    return numberOfBackedUpPages;
                }
            }
        }
Пример #4
0
		public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
			Action<string> infoNotify = null)
		{
			infoNotify = infoNotify ?? (s => { });

			var dataPager = env.Options.DataPager;
			var copier = new DataCopier(AbstractPager.PageSize * 16);
			Transaction txr = null;
			try
			{

				infoNotify("Voron copy headers");
                                           
				using (var file = new FileStream(backupPath, FileMode.Create))
				using (var package = new ZipArchive(file, ZipArchiveMode.Create))
				{
					long allocatedPages;

					ImmutableAppendOnlyList<JournalFile> files; // thread safety copy
					long lastWrittenLogPage = -1;
					long lastWrittenLogFile = -1;
					using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
					{
						txr = env.NewTransaction(TransactionFlags.Read); // now have snapshot view
						allocatedPages = dataPager.NumberOfAllocatedPages;

						Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);

                        VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options);

						// journal files snapshot
						files = env.Journal.Files;
 
						foreach (var journalFile in files)
						{
							journalFile.AddRef();
						}

						if (env.Journal.CurrentFile != null)
						{
							lastWrittenLogFile = env.Journal.CurrentFile.Number;
							lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
						}

						// txw.Commit(); intentionally not committing
					}

					// data file backup
					var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression);
					Debug.Assert(dataPart != null);

					if (allocatedPages > 0) //only true if dataPager is still empty at backup start
					{
						using (var dataStream = dataPart.Open())
						{
							// now can copy everything else
							var firstDataPage = dataPager.Read(0);

							copier.ToStream(firstDataPage.Base, AbstractPager.PageSize*allocatedPages, dataStream);
						}
					}

					try
					{
						foreach (var journalFile in files)
						{
							var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression);

							Debug.Assert(journalPart != null);

							var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
							if (journalFile.Number == lastWrittenLogFile)
								pagesToCopy = lastWrittenLogPage + 1;

							using (var stream = journalPart.Open())
							{
								copier.ToStream(journalFile, 0, pagesToCopy, stream);
								infoNotify(string.Format("Voron copy journal file {0} ", journalFile));
							}
                        
 						}
					}
					finally
					{
						foreach (var journalFile in files)
						{
							journalFile.Release();
						}
					}
				}
			}
			finally
			{
				if (txr != null)
					txr.Dispose();
			}
			infoNotify(string.Format("Voron backup db finished"));
		}
Пример #5
0
        public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
            Action<string> infoNotify = null,
            Action backupStarted = null)
        {
            infoNotify = infoNotify ?? (s => { });

            var dataPager = env.Options.DataPager;
            var copier = new DataCopier(AbstractPager.PageSize * 16);
            Transaction txr = null;
            try
            {

                infoNotify("Voron copy headers");

                using (var file = new FileStream(backupPath, FileMode.Create))
                {
                    using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen:true))
                    {
                        long allocatedPages;

                        ImmutableAppendOnlyList<JournalFile> files; // thread safety copy
                        var usedJournals = new List<JournalFile>();
                        long lastWrittenLogPage = -1;
                        long lastWrittenLogFile = -1;
                        using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
                        {
                            txr = env.NewTransaction(TransactionFlags.Read); // now have snapshot view
                            allocatedPages = dataPager.NumberOfAllocatedPages;

                            Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);

                            VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options);

                            // journal files snapshot
                            files = env.Journal.Files;

                            JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
                            for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1; journalNum <= journalInfo.CurrentJournal; journalNum++)
                            {
                                var journalFile = files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
                                if (journalFile == null)
                                {
                                    long journalSize;
                                    using (var pager = env.Options.OpenJournalPager(journalNum))
                                    {
                                        journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
                                    }

                                    journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
                                }

                                journalFile.AddRef();
                                usedJournals.Add(journalFile);
                            }

                            if (env.Journal.CurrentFile != null)
                            {
                                lastWrittenLogFile = env.Journal.CurrentFile.Number;
                                lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
                            }

                            // txw.Commit(); intentionally not committing
                        }

                        if (backupStarted != null)
                            backupStarted();

                        // data file backup
                        var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression);
                        Debug.Assert(dataPart != null);

                        if (allocatedPages > 0) //only true if dataPager is still empty at backup start
                        {
                            using (var dataStream = dataPart.Open())
                            {
                                // now can copy everything else
                                var firstDataPage = dataPager.Read(0);

                                copier.ToStream(firstDataPage.Base, AbstractPager.PageSize * allocatedPages, dataStream);
                            }
                        }

                        try
                        {
                            foreach (var journalFile in usedJournals)
                            {
                                var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression);

                                Debug.Assert(journalPart != null);

                                var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
                                if (journalFile.Number == lastWrittenLogFile)
                                    pagesToCopy = lastWrittenLogPage + 1;

                                using (var stream = journalPart.Open())
                                {
                                    copier.ToStream(journalFile, 0, pagesToCopy, stream);
                                    infoNotify(string.Format("Voron copy journal file {0} ", journalFile));
                                }

                            }
                        }
                        finally
                        {
                            foreach (var journalFile in usedJournals)
                            {
                                journalFile.Release();
                            }
                        }
                    }
                    file.Flush(true); // make sure that we fully flushed to disk
                }
            }
            finally
            {
                if (txr != null)
                    txr.Dispose();
            }
            infoNotify(string.Format("Voron backup db finished"));
        }