Пример #1
0
        private static void Backup(
            StorageEnvironment env, CompressionLevel compression, Action <string> infoNotify,
            Action backupStarted, AbstractPager dataPager, ZipArchive package, string basePath, DataCopier copier)
        {
            var  usedJournals       = new List <JournalFile>();
            long lastWrittenLogPage = -1;
            long lastWrittenLogFile = -1;
            LowLevelTransaction txr = null;

            try
            {
                long allocatedPages;
                var  writePesistentContext = new TransactionPersistentContext(true);
                var  readPesistentContext  = new TransactionPersistentContext(true);
                using (var txw = env.NewLowLevelTransaction(writePesistentContext, TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
                {
                    txr            = env.NewLowLevelTransaction(readPesistentContext, TransactionFlags.Read);   // now have snapshot view
                    allocatedPages = dataPager.NumberOfAllocatedPages;

                    Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
                    infoNotify("Voron copy headers for " + basePath);
                    VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options, basePath);

                    // journal files snapshot
                    var files = env.Journal.Files; // thread safety copy

                    JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
                    for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1;
                         journalNum <= journalInfo.CurrentJournal;
                         journalNum++)
                    {
                        var journalFile = files.FirstOrDefault(x => x.Number == journalNum);
                        // first check journal files currently being in use
                        if (journalFile == null)
                        {
                            long journalSize;
                            using (var pager = env.Options.OpenJournalPager(journalNum))
                            {
                                journalSize = Bits.NextPowerOf2(pager.NumberOfAllocatedPages * env.Options.PageSize);
                            }

                            journalFile = new JournalFile(env, env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
                        }

                        journalFile.AddRef();
                        usedJournals.Add(journalFile);
                    }

                    if (env.Journal.CurrentFile != null)
                    {
                        lastWrittenLogFile = env.Journal.CurrentFile.Number;
                        lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
                    }

                    // txw.Commit(); intentionally not committing
                }

                backupStarted?.Invoke();

                // data file backup
                var dataPart = package.CreateEntry(Path.Combine(basePath, Constants.DatabaseFilename), compression);
                Debug.Assert(dataPart != null);

                if (allocatedPages > 0) //only true if dataPager is still empty at backup start
                {
                    using (var dataStream = dataPart.Open())
                    {
                        // now can copy everything else
                        copier.ToStream(dataPager, 0, allocatedPages, dataStream);
                    }
                }

                try
                {
                    foreach (JournalFile journalFile in usedJournals)
                    {
                        var entryName   = Path.Combine(basePath, StorageEnvironmentOptions.JournalName(journalFile.Number));
                        var journalPart = package.CreateEntry(entryName, compression);

                        Debug.Assert(journalPart != null);

                        long pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
                        if (journalFile.Number == lastWrittenLogFile)
                        {
                            pagesToCopy = lastWrittenLogPage + 1;
                        }

                        using (var stream = journalPart.Open())
                        {
                            copier.ToStream(env, journalFile, 0, pagesToCopy, stream);
                            infoNotify(string.Format("Voron copy journal file {0}", entryName));
                        }
                    }
                }
                finally
                {
                    foreach (var journalFile in usedJournals)
                    {
                        journalFile.Release();
                    }
                }
            }
            finally
            {
                txr?.Dispose();
            }
        }
Пример #2
0
        private static void Backup(
            StorageEnvironment env, CompressionLevel compression, Action <string> infoNotify,
            Action backupStarted, AbstractPager dataPager, ZipArchive package, string basePath, DataCopier copier)
        {
            var  usedJournals       = new List <JournalFile>();
            long lastWrittenLogPage = -1;
            long lastWrittenLogFile = -1;
            LowLevelTransaction txr = null;
            var backupSuccess       = false;

            try
            {
                long allocatedPages;
                var  writePesistentContext = new TransactionPersistentContext(true);
                var  readPesistentContext  = new TransactionPersistentContext(true);
                using (var txw = env.NewLowLevelTransaction(writePesistentContext, TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
                {
                    txr            = env.NewLowLevelTransaction(readPesistentContext, TransactionFlags.Read);   // now have snapshot view
                    allocatedPages = dataPager.NumberOfAllocatedPages;

                    Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
                    infoNotify("Voron copy headers for " + basePath);
                    VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options, basePath);

                    // journal files snapshot
                    var files = env.Journal.Files; // thread safety copy

                    JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
                    for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1;
                         journalNum <= journalInfo.CurrentJournal;
                         journalNum++)
                    {
                        var journalFile = files.FirstOrDefault(x => x.Number == journalNum);
                        // first check journal files currently being in use
                        if (journalFile == null)
                        {
                            long journalSize;
                            using (var pager = env.Options.OpenJournalPager(journalNum))
                            {
                                journalSize = Bits.NextPowerOf2(pager.NumberOfAllocatedPages * Constants.Storage.PageSize);
                            }

                            journalFile = new JournalFile(env, env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
                        }

                        journalFile.AddRef();
                        usedJournals.Add(journalFile);
                    }

                    if (env.Journal.CurrentFile != null)
                    {
                        lastWrittenLogFile = env.Journal.CurrentFile.Number;
                        lastWrittenLogPage = env.Journal.CurrentFile.WritePosIn4KbPosition - 1;
                    }

                    // txw.Commit(); intentionally not committing
                }

                backupStarted?.Invoke();

                // data file backup
                var dataPart = package.CreateEntry(Path.Combine(basePath, Constants.DatabaseFilename), compression);
                Debug.Assert(dataPart != null);

                if (allocatedPages > 0) //only true if dataPager is still empty at backup start
                {
                    using (var dataStream = dataPart.Open())
                    {
                        // now can copy everything else
                        copier.ToStream(dataPager, 0, allocatedPages, dataStream);
                    }
                }

                try
                {
                    long lastBackedupJournal = 0;
                    foreach (var journalFile in usedJournals)
                    {
                        var entryName   = StorageEnvironmentOptions.JournalName(journalFile.Number);
                        var journalPart = package.CreateEntry(Path.Combine(basePath, entryName), compression);

                        Debug.Assert(journalPart != null);

                        long pagesToCopy = journalFile.JournalWriter.NumberOfAllocated4Kb;
                        if (journalFile.Number == lastWrittenLogFile)
                        {
                            pagesToCopy = lastWrittenLogPage + 1;
                        }

                        using (var stream = journalPart.Open())
                        {
                            copier.ToStream(env, journalFile, 0, pagesToCopy, stream);
                            infoNotify(string.Format("Voron copy journal file {0}", entryName));
                        }

                        lastBackedupJournal = journalFile.Number;
                    }

                    if (env.Options.IncrementalBackupEnabled)
                    {
                        env.HeaderAccessor.Modify(header =>
                        {
                            header->IncrementalBackup.LastBackedUpJournal = lastBackedupJournal;

                            //since we backed-up everything, no need to start next incremental backup from the middle
                            header->IncrementalBackup.LastBackedUpJournalPage = -1;
                        });
                    }
                    backupSuccess = true;
                }
                catch (Exception)
                {
                    backupSuccess = false;
                    throw;
                }
                finally
                {
                    var lastSyncedJournal = env.HeaderAccessor.Get(header => header->Journal).LastSyncedJournal;
                    foreach (var journalFile in usedJournals)
                    {
                        if (backupSuccess)                                 // if backup succeeded we can remove journals
                        {
                            if (journalFile.Number < lastWrittenLogFile && // prevent deletion of the current journal and journals with a greater number
                                journalFile.Number < lastSyncedJournal)    // prevent deletion of journals that aren't synced with the data file
                            {
                                journalFile.DeleteOnClose = true;
                            }
                        }

                        journalFile.Release();
                    }
                }
            }
            finally
            {
                txr?.Dispose();
            }
        }
Пример #3
0
        public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
                           Action <string> infoNotify = null)
        {
            infoNotify = infoNotify ?? (s => { });

            var         dataPager = env.Options.DataPager;
            var         copier    = new DataCopier(AbstractPager.PageSize * 16);
            Transaction txr       = null;

            try
            {
                infoNotify("Voron copy headers");

                using (var file = new FileStream(backupPath, FileMode.Create))
                    using (var package = new ZipArchive(file, ZipArchiveMode.Create))
                    {
                        long allocatedPages;

                        ImmutableAppendOnlyList <JournalFile> files;                // thread safety copy
                        long lastWrittenLogPage = -1;
                        long lastWrittenLogFile = -1;
                        using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))         // so we can snapshot the headers safely
                        {
                            txr            = env.NewTransaction(TransactionFlags.Read);          // now have snapshot view
                            allocatedPages = dataPager.NumberOfAllocatedPages;

                            Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);

                            VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options);

                            // journal files snapshot
                            files = env.Journal.Files;

                            foreach (var journalFile in files)
                            {
                                journalFile.AddRef();
                            }

                            if (env.Journal.CurrentFile != null)
                            {
                                lastWrittenLogFile = env.Journal.CurrentFile.Number;
                                lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
                            }

                            // txw.Commit(); intentionally not committing
                        }

                        // data file backup
                        var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression);
                        Debug.Assert(dataPart != null);

                        if (allocatedPages > 0)                 //only true if dataPager is still empty at backup start
                        {
                            using (var dataStream = dataPart.Open())
                            {
                                // now can copy everything else
                                var firstDataPage = dataPager.Read(0);

                                copier.ToStream(firstDataPage.Base, AbstractPager.PageSize * allocatedPages, dataStream);
                            }
                        }

                        try
                        {
                            foreach (var journalFile in files)
                            {
                                var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression);

                                Debug.Assert(journalPart != null);

                                var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
                                if (journalFile.Number == lastWrittenLogFile)
                                {
                                    pagesToCopy = lastWrittenLogPage + 1;
                                }

                                using (var stream = journalPart.Open())
                                {
                                    copier.ToStream(journalFile, 0, pagesToCopy, stream);
                                    infoNotify(string.Format("Voron copy journal file {0} ", journalFile));
                                }
                            }
                        }
                        finally
                        {
                            foreach (var journalFile in files)
                            {
                                journalFile.Release();
                            }
                        }
                    }
            }
            finally
            {
                if (txr != null)
                {
                    txr.Dispose();
                }
            }
            infoNotify(string.Format("Voron backup db finished"));
        }
Пример #4
0
        public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
                           Action <string> infoNotify = null,
                           Action backupStarted       = null)
        {
            infoNotify = infoNotify ?? (s => { });

            var         dataPager = env.Options.DataPager;
            var         copier    = new DataCopier(AbstractPager.PageSize * 16);
            Transaction txr       = null;

            try
            {
                infoNotify("Voron copy headers");

                using (var file = new FileStream(backupPath, FileMode.Create))
                {
                    using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
                    {
                        long allocatedPages;

                        ImmutableAppendOnlyList <JournalFile> files; // thread safety copy
                        var  usedJournals       = new List <JournalFile>();
                        long lastWrittenLogPage = -1;
                        long lastWrittenLogFile = -1;
                        using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
                        {
                            txr            = env.NewTransaction(TransactionFlags.Read);  // now have snapshot view
                            allocatedPages = dataPager.NumberOfAllocatedPages;

                            Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);

                            VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options);

                            // journal files snapshot
                            files = env.Journal.Files;

                            JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
                            for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1; journalNum <= journalInfo.CurrentJournal; journalNum++)
                            {
                                var journalFile = files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
                                if (journalFile == null)
                                {
                                    long journalSize;
                                    using (var pager = env.Options.OpenJournalPager(journalNum))
                                    {
                                        journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
                                    }

                                    journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
                                }

                                journalFile.AddRef();
                                usedJournals.Add(journalFile);
                            }

                            if (env.Journal.CurrentFile != null)
                            {
                                lastWrittenLogFile = env.Journal.CurrentFile.Number;
                                lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
                            }

                            // txw.Commit(); intentionally not committing
                        }

                        if (backupStarted != null)
                        {
                            backupStarted();
                        }

                        // data file backup
                        var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression);
                        Debug.Assert(dataPart != null);

                        if (allocatedPages > 0) //only true if dataPager is still empty at backup start
                        {
                            using (var dataStream = dataPart.Open())
                            {
                                // now can copy everything else
                                var firstDataPage = dataPager.Read(null, 0);

                                copier.ToStream(firstDataPage.Base, AbstractPager.PageSize * allocatedPages, dataStream);
                            }
                        }

                        try
                        {
                            foreach (var journalFile in usedJournals)
                            {
                                var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression);

                                Debug.Assert(journalPart != null);

                                var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
                                if (journalFile.Number == lastWrittenLogFile)
                                {
                                    pagesToCopy = lastWrittenLogPage + 1;
                                }

                                using (var stream = journalPart.Open())
                                {
                                    copier.ToStream(journalFile, 0, pagesToCopy, stream);
                                    infoNotify(string.Format("Voron copy journal file {0} ", journalFile));
                                }
                            }
                        }
                        finally
                        {
                            foreach (var journalFile in usedJournals)
                            {
                                journalFile.Release();
                            }
                        }
                    }
                    file.Flush(true); // make sure that we fully flushed to disk
                }
            }
            finally
            {
                if (txr != null)
                {
                    txr.Dispose();
                }
            }
            infoNotify(string.Format("Voron backup db finished"));
        }