Example #1
0
        private void Restore(StorageEnvironment env, string singleBackupFile)
        {
            using (env.Journal.Applicator.TakeFlushingLock())
            {
                using (var txw = env.NewLowLevelTransaction(TransactionFlags.ReadWrite))
                {
                    using (env.Options.AllowManualFlushing())
                    {
                        env.FlushLogToDataFile(txw);
                    }

                    using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
                    {
                        if (package.Entries.Count == 0)
                        {
                            return;
                        }

                        var toDispose = new List <IDisposable>();

                        var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;

                        try
                        {
                            TransactionHeader *lastTxHeader = null;
                            var pagesToWrite = new Dictionary <long, TreePage>();

                            long journalNumber = -1;
                            foreach (var entry in package.Entries)
                            {
                                switch (Path.GetExtension(entry.Name))
                                {
                                case ".merged-journal":
                                case ".journal":

                                    var jounalFileName = Path.Combine(tempDir, entry.Name);
                                    using (var output = new FileStream(jounalFileName, FileMode.Create))
                                        using (var input = entry.Open())
                                        {
                                            output.Position = output.Length;
                                            input.CopyTo(output);
                                        }

                                    var pager = env.Options.OpenPager(jounalFileName);
                                    toDispose.Add(pager);

                                    if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false)
                                    {
                                        throw new InvalidOperationException("Cannot parse journal file number");
                                    }

                                    var recoveryPager = env.Options.CreateScratchPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
                                    toDispose.Add(recoveryPager);

                                    var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);

                                    while (reader.ReadOneTransaction(env.Options))
                                    {
                                        lastTxHeader = reader.LastTransactionHeader;
                                    }

                                    foreach (var translation in reader.TransactionPageTranslation)
                                    {
                                        var pageInJournal = translation.Value.JournalPos;
                                        var page          = recoveryPager.Read(null, pageInJournal);
                                        pagesToWrite[translation.Key] = page;

                                        if (page.IsOverflow)
                                        {
                                            var numberOfOverflowPages = recoveryPager.GetNumberOfOverflowPages(page.OverflowSize);

                                            for (int i = 1; i < numberOfOverflowPages; i++)
                                            {
                                                pagesToWrite.Remove(translation.Key + i);
                                            }
                                        }
                                    }

                                    break;

                                default:
                                    throw new InvalidOperationException("Unknown file, cannot restore: " + entry);
                                }
                            }

                            var sortedPages = pagesToWrite.OrderBy(x => x.Key)
                                              .Select(x => x.Value)
                                              .ToList();

                            if (sortedPages.Count == 0)
                            {
                                return;
                            }
                            var last = sortedPages.Last();

                            var numberOfPages = last.IsOverflow
                                ? env.Options.DataPager.GetNumberOfOverflowPages(
                                last.OverflowSize)
                                : 1;
                            var pagerState = env.Options.DataPager.EnsureContinuous(last.PageNumber, numberOfPages);
                            txw.EnsurePagerStateReference(pagerState);

                            foreach (var page in sortedPages)
                            {
                                env.Options.DataPager.Write(page);
                            }

                            env.Options.DataPager.Sync();

                            var root = Tree.Open(txw, null, &lastTxHeader->Root);
                            root.Name = Constants.RootTreeName;

                            txw.UpdateRootsIfNeeded(root);

                            txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;

                            env.Journal.Clear(txw);

                            txw.Commit();

                            env.HeaderAccessor.Modify(header =>
                            {
                                header->TransactionId  = lastTxHeader->TransactionId;
                                header->LastPageNumber = lastTxHeader->LastPageNumber;

                                header->Journal.LastSyncedJournal       = journalNumber;
                                header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;

                                header->Root = lastTxHeader->Root;

                                header->Journal.CurrentJournal    = journalNumber + 1;
                                header->Journal.JournalFilesCount = 0;
                            });
                        }
                        finally
                        {
                            toDispose.ForEach(x => x.Dispose());

                            try
                            {
                                Directory.Delete(tempDir, true);
                            }
                            catch
                            {
                                // this is just a temporary directory, the worst case scenario is that we dont reclaim the space from the OS temp directory
                                // if for some reason we cannot delete it we are safe to ignore it.
                            }
                        }
                    }
                }
            }
        }
        public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action <string> infoNotify = null,
                           Action backupStarted = null)
        {
            if (env.Options.IncrementalBackupEnabled == false)
            {
                throw new InvalidOperationException("Incremental backup is disabled for this storage");
            }

            var pageNumberToPageInScratch = new Dictionary <long, long>();

            if (infoNotify == null)
            {
                infoNotify = str => { }
            }
            ;
            var toDispose = new List <IDisposable>();

            try
            {
                IncrementalBackupInfo backupInfo;
                long lastWrittenLogPage = -1;
                long lastWrittenLogFile = -1;

                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);

                    if (env.Journal.CurrentFile != null)
                    {
                        lastWrittenLogFile = env.Journal.CurrentFile.Number;
                        lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition;
                    }

                    //txw.Commit(); - intentionally not committing
                }

                if (backupStarted != null)
                {
                    backupStarted();
                }

                infoNotify("Voron - reading storage journals for snapshot pages");

                var lastBackedUpFile     = backupInfo.LastBackedUpJournal;
                var lastBackedUpPage     = backupInfo.LastBackedUpJournalPage;
                var firstJournalToBackup = backupInfo.LastBackedUpJournal;

                if (firstJournalToBackup == -1)
                {
                    firstJournalToBackup = 0;                     // first time that we do incremental backup
                }
                var lastTransaction = new TransactionHeader {
                    TransactionId = -1
                };

                var recoveryPager = env.Options.CreateScratchPager("min-inc-backup.scratch");
                toDispose.Add(recoveryPager);
                int recoveryPage = 0;
                for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++)
                {
                    lastBackedUpFile = journalNum;
                    var journalFile = IncrementalBackup.GetJournalFile(env, journalNum, backupInfo);
                    try
                    {
                        using (var filePager = env.Options.OpenJournalPager(journalNum))
                        {
                            var reader = new JournalReader(filePager, recoveryPager, 0, null, recoveryPage);
                            reader.MaxPageToRead = lastBackedUpPage = journalFile.JournalWriter.NumberOfAllocatedPages;
                            if (journalNum == lastWrittenLogFile)                             // set the last part of the log file we'll be reading
                            {
                                reader.MaxPageToRead = lastBackedUpPage = lastWrittenLogPage;
                            }

                            if (lastBackedUpPage == journalFile.JournalWriter.NumberOfAllocatedPages)                             // past the file size
                            {
                                // move to the next
                                lastBackedUpPage = -1;
                                lastBackedUpFile++;
                            }

                            if (journalNum == backupInfo.LastBackedUpJournal)                             // continue from last backup
                            {
                                reader.SetStartPage(backupInfo.LastBackedUpJournalPage);
                            }
                            TransactionHeader *lastJournalTxHeader = null;
                            while (reader.ReadOneTransaction(env.Options))
                            {
                                // read all transactions here
                                lastJournalTxHeader = reader.LastTransactionHeader;
                            }

                            if (lastJournalTxHeader != null)
                            {
                                lastTransaction = *lastJournalTxHeader;
                            }

                            recoveryPage = reader.RecoveryPage;

                            foreach (var pagePosition in reader.TransactionPageTranslation)
                            {
                                var pageInJournal = pagePosition.Value.JournalPos;
                                var page          = recoveryPager.Read(pageInJournal);
                                pageNumberToPageInScratch[pagePosition.Key] = pageInJournal;
                                if (page.IsOverflow)
                                {
                                    var numberOfOverflowPages = recoveryPager.GetNumberOfOverflowPages(page.OverflowSize);
                                    for (int i = 1; i < numberOfOverflowPages; i++)
                                    {
                                        pageNumberToPageInScratch.Remove(page.PageNumber + i);
                                    }
                                }
                            }
                        }
                    }
                    finally
                    {
                        journalFile.Release();
                    }
                }

                if (pageNumberToPageInScratch.Count == 0)
                {
                    infoNotify("Voron - no changes since last backup, nothing to do");
                    return;
                }

                infoNotify("Voron - started writing snapshot file.");

                if (lastTransaction.TransactionId == -1)
                {
                    throw new InvalidOperationException("Could not find any transactions in the journals, but found pages to write? That ain't right.");
                }


                // it is possible that we merged enough transactions so the _merged_ output is too large for us.
                // Voron limit transactions to about 4GB each. That means that we can't just merge all transactions
                // blindly, for fear of hitting this limit. So we need to split things.
                // We are also limited to about 8 TB of data in general before we literally can't fit the number of pages into
                // pageNumberToPageInScratch even theoretically.
                // We're fine with saying that you need to run min inc backup before you hit 8 TB in your increment, so that works for now.
                // We are also going to use env.Options.MaxScratchBufferSize to set the actual transaction limit here, to avoid issues
                // down the road and to limit how big a single transaction can be before the theoretical 4GB limit.

                var nextJournalNum = lastBackedUpFile;
                using (var file = new FileStream(backupPath, FileMode.Create))
                {
                    using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
                    {
                        var copier = new DataCopier(AbstractPager.PageSize * 16);

                        var finalPager = env.Options.CreateScratchPager("min-inc-backup-final.scratch");
                        toDispose.Add(finalPager);
                        finalPager.EnsureContinuous(null, 0, 1);                        //txHeader

                        foreach (var partition in Partition(pageNumberToPageInScratch.Values, env.Options.MaxNumberOfPagesInMergedTransaction))
                        {
                            int totalNumberOfPages = 0;
                            int overflowPages      = 0;
                            int start = 1;
                            foreach (var pageNum in partition)
                            {
                                var p    = recoveryPager.Read(pageNum);
                                var size = 1;
                                if (p.IsOverflow)
                                {
                                    size           = recoveryPager.GetNumberOfOverflowPages(p.OverflowSize);
                                    overflowPages += (size - 1);
                                }
                                totalNumberOfPages += size;
                                finalPager.EnsureContinuous(null, start, size);                                 //maybe increase size

                                MemoryUtils.Copy(finalPager.AcquirePagePointer(start), p.Base, size * AbstractPager.PageSize);

                                start += size;
                            }


                            var txPage = finalPager.AcquirePagePointer(0);
                            StdLib.memset(txPage, 0, AbstractPager.PageSize);
                            var txHeader = (TransactionHeader *)txPage;
                            txHeader->HeaderMarker           = Constants.TransactionHeaderMarker;
                            txHeader->FreeSpace              = lastTransaction.FreeSpace;
                            txHeader->Root                   = lastTransaction.Root;
                            txHeader->OverflowPageCount      = overflowPages;
                            txHeader->PageCount              = totalNumberOfPages - overflowPages;
                            txHeader->PreviousTransactionCrc = lastTransaction.PreviousTransactionCrc;
                            txHeader->TransactionId          = lastTransaction.TransactionId;
                            txHeader->NextPageNumber         = lastTransaction.NextPageNumber;
                            txHeader->LastPageNumber         = lastTransaction.LastPageNumber;
                            txHeader->TxMarker               = TransactionMarker.Commit | TransactionMarker.Merged;
                            txHeader->Compressed             = false;
                            txHeader->UncompressedSize       = txHeader->CompressedSize = totalNumberOfPages * AbstractPager.PageSize;

                            txHeader->Crc = Crc.Value(finalPager.AcquirePagePointer(1), 0, totalNumberOfPages * AbstractPager.PageSize);


                            var entry = package.CreateEntry(string.Format("{0:D19}.merged-journal", nextJournalNum), compression);
                            nextJournalNum++;
                            using (var stream = entry.Open())
                            {
                                copier.ToStream(finalPager.AcquirePagePointer(0), (totalNumberOfPages + 1) * AbstractPager.PageSize, stream);
                            }
                        }
                    }
                    file.Flush(true);                    // make sure we hit the disk and stay there
                }

                env.HeaderAccessor.Modify(header =>
                {
                    header->IncrementalBackup.LastBackedUpJournal     = lastBackedUpFile;
                    header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage;
                });
            }
            finally
            {
                foreach (var disposable in toDispose)
                {
                    disposable.Dispose();
                }
            }
        }
        private void Restore(StorageEnvironment env, string singleBackupFile)
        {
            using (env.Journal.Applicator.TakeFlushingLock())
            {
                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    using (env.Options.AllowManualFlushing())
                    {
                        env.FlushLogToDataFile(txw);
                    }

                    using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read))
                    {
                        if (package.Entries.Count == 0)
                        {
                            return;
                        }

                        var toDispose = new List <IDisposable>();

                        var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;

                        try
                        {
                            TransactionHeader *lastTxHeader = null;
                            var pagesToWrite = new Dictionary <long, Func <Page> >();

                            long journalNumber = -1;
                            foreach (var entry in package.Entries)
                            {
                                switch (Path.GetExtension(entry.Name))
                                {
                                case ".journal":

                                    var jounalFileName = Path.Combine(tempDir, entry.Name);
                                    using (var output = new FileStream(jounalFileName, FileMode.Create))
                                        using (var input = entry.Open())
                                        {
                                            output.Position = output.Length;
                                            input.CopyTo(output);
                                        }

                                    var pager = new Win32MemoryMapPager(jounalFileName);
                                    toDispose.Add(pager);

                                    if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false)
                                    {
                                        throw new InvalidOperationException("Cannot parse journal file number");
                                    }

                                    var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
                                    toDispose.Add(recoveryPager);

                                    var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);

                                    while (reader.ReadOneTransaction(env.Options))
                                    {
                                        lastTxHeader = reader.LastTransactionHeader;
                                    }

                                    foreach (var translation in reader.TransactionPageTranslation)
                                    {
                                        var pageInJournal = translation.Value.JournalPos;
                                        pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal);
                                    }

                                    break;

                                default:
                                    throw new InvalidOperationException("Unknown file, cannot restore: " + entry);
                                }
                            }

                            var sortedPages = pagesToWrite.OrderBy(x => x.Key)
                                              .Select(x => x.Value())
                                              .ToList();

                            if (sortedPages.Count == 0)
                            {
                                return;
                            }
                            var last = sortedPages.Last();

                            env.Options.DataPager.EnsureContinuous(txw, last.PageNumber,
                                                                   last.IsOverflow
                                    ? env.Options.DataPager.GetNumberOfOverflowPages(
                                                                       last.OverflowSize)
                                    : 1);

                            foreach (var page in sortedPages)
                            {
                                env.Options.DataPager.Write(page);
                            }

                            env.Options.DataPager.Sync();

                            txw.State.Root          = Tree.Open(txw, &lastTxHeader->Root);
                            txw.State.FreeSpaceRoot = Tree.Open(txw, &lastTxHeader->FreeSpace);

                            txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName;
                            txw.State.Root.Name          = Constants.RootTreeName;

                            txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;

                            env.Journal.Clear(txw);

                            txw.Commit();

                            env.HeaderAccessor.Modify(header =>
                            {
                                header->TransactionId  = lastTxHeader->TransactionId;
                                header->LastPageNumber = lastTxHeader->LastPageNumber;

                                header->Journal.LastSyncedJournal       = journalNumber;
                                header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;

                                header->Root      = lastTxHeader->Root;
                                header->FreeSpace = lastTxHeader->FreeSpace;

                                header->Journal.CurrentJournal    = journalNumber + 1;
                                header->Journal.JournalFilesCount = 0;
                            });
                        }
                        finally
                        {
                            toDispose.ForEach(x => x.Dispose());

                            try
                            {
                                Directory.Delete(tempDir, true);
                            }
                            catch (Exception)
                            {
                                // just temp dir - ignore it
                            }
                        }
                    }
                }
            }
        }
        private void Restore(StorageEnvironment env, string backupPath)
        {
            using (env.Journal.Applicator.TakeFlushingLock())
            {
                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    using (env.Options.AllowManualFlushing())
                    {
                        env.FlushLogToDataFile(txw);
                    }

                    List <string> journalNames;

                    using (var package = ZipFile.Open(backupPath, ZipArchiveMode.Read))
                    {
                        journalNames = package.Entries.Select(x => x.Name).ToList();
                    }

                    if (journalNames.Count == 0)
                    {
                        return;
                    }

                    var tempDir   = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
                    var toDispose = new List <IDisposable>();

                    try
                    {
                        ZipFile.ExtractToDirectory(backupPath, tempDir);

                        TransactionHeader *lastTxHeader = null;

                        var pagesToWrite = new Dictionary <long, Func <Page> >();

                        long journalNumber = -1;
                        foreach (var journalName in journalNames)
                        {
                            var pager = new Win32MemoryMapPager(Path.Combine(tempDir, journalName));
                            toDispose.Add(pager);


                            if (long.TryParse(journalName.Replace(".journal", string.Empty), out journalNumber) == false)
                            {
                                throw new InvalidOperationException("Cannot parse journal file number");
                            }

                            var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
                            toDispose.Add(recoveryPager);

                            var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);

                            while (reader.ReadOneTransaction(env.Options))
                            {
                                lastTxHeader = reader.LastTransactionHeader;
                            }

                            foreach (var translation in reader.TransactionPageTranslation)
                            {
                                var pageInJournal = translation.Value.JournalPos;
                                pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal);
                            }
                        }

                        var sortedPages = pagesToWrite.OrderBy(x => x.Key)
                                          .Select(x => x.Value())
                                          .ToList();

                        var last = sortedPages.Last();

                        env.Options.DataPager.EnsureContinuous(txw, last.PageNumber,
                                                               last.IsOverflow
                                                            ? env.Options.DataPager.GetNumberOfOverflowPages(
                                                                   last.OverflowSize)
                                                            : 1);

                        foreach (var page in sortedPages)
                        {
                            env.Options.DataPager.Write(page);
                        }

                        env.Options.DataPager.Sync();

                        txw.State.Root          = Tree.Open(txw, env._sliceComparer, &lastTxHeader->Root);
                        txw.State.FreeSpaceRoot = Tree.Open(txw, env._sliceComparer, &lastTxHeader->FreeSpace);

                        txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName;
                        txw.State.Root.Name          = Constants.RootTreeName;

                        txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;

                        env.Journal.Clear(txw);

                        txw.Commit();

                        env.HeaderAccessor.Modify(header =>
                        {
                            header->TransactionId  = lastTxHeader->TransactionId;
                            header->LastPageNumber = lastTxHeader->LastPageNumber;

                            header->Journal.LastSyncedJournal       = journalNumber;
                            header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;

                            header->Root      = lastTxHeader->Root;
                            header->FreeSpace = lastTxHeader->FreeSpace;

                            header->Journal.CurrentJournal    = journalNumber + 1;
                            header->Journal.JournalFilesCount = 0;
                        });
                    }
                    finally
                    {
                        toDispose.ForEach(x => x.Dispose());

                        Directory.Delete(tempDir, true);
                    }
                }
            }
        }