Example #1
0
 public void InitFrom(JournalReader journalReader)
 {
     _writePage = journalReader.NextWritePage;
 }
        public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action<string> infoNotify = null,
            Action backupStarted = null)
        {
            if (env.Options.IncrementalBackupEnabled == false)
                throw new InvalidOperationException("Incremental backup is disabled for this storage");

            var pageNumberToPageInScratch = new Dictionary<long, long>();
            if (infoNotify == null)
                infoNotify = str => { };
            var toDispose = new List<IDisposable>();
            try
            {
                IncrementalBackupInfo backupInfo;
                long lastWrittenLogPage = -1;
                long lastWrittenLogFile = -1;

                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);

                    if (env.Journal.CurrentFile != null)
                    {
                        lastWrittenLogFile = env.Journal.CurrentFile.Number;
                        lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition;
                    }

                    //txw.Commit(); - intentionally not committing
                }

                if (backupStarted != null)
                    backupStarted();

                infoNotify("Voron - reading storage journals for snapshot pages");

                var lastBackedUpFile = backupInfo.LastBackedUpJournal;
                var lastBackedUpPage = backupInfo.LastBackedUpJournalPage;
                var firstJournalToBackup = backupInfo.LastBackedUpJournal;

                if (firstJournalToBackup == -1)
                    firstJournalToBackup = 0; // first time that we do incremental backup

                var lastTransaction = new TransactionHeader { TransactionId = -1 };

                var recoveryPager = env.Options.CreateScratchPager("min-inc-backup.scratch");
                toDispose.Add(recoveryPager);
                int recoveryPage = 0;
                for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++)
                {
                    lastBackedUpFile = journalNum;
                    var journalFile = IncrementalBackup.GetJournalFile(env, journalNum, backupInfo);
                    try
                    {
                        using (var filePager = env.Options.OpenJournalPager(journalNum))
                        {
                            var reader = new JournalReader(filePager, recoveryPager, 0, null, recoveryPage);
                            reader.MaxPageToRead = lastBackedUpPage = journalFile.JournalWriter.NumberOfAllocatedPages;
                            if (journalNum == lastWrittenLogFile) // set the last part of the log file we'll be reading
                                reader.MaxPageToRead = lastBackedUpPage = lastWrittenLogPage;

                            if (lastBackedUpPage == journalFile.JournalWriter.NumberOfAllocatedPages) // past the file size
                            {
                                // move to the next
                                lastBackedUpPage = -1;
                                lastBackedUpFile++;
                            }

                            if (journalNum == backupInfo.LastBackedUpJournal) // continue from last backup
                                reader.SetStartPage(backupInfo.LastBackedUpJournalPage);
                            TransactionHeader* lastJournalTxHeader = null;
                            while (reader.ReadOneTransaction(env.Options))
                            {
                                // read all transactions here
                                lastJournalTxHeader = reader.LastTransactionHeader;
                            }

                            if (lastJournalTxHeader != null)
                                lastTransaction = *lastJournalTxHeader;

                            recoveryPage = reader.RecoveryPage;

                            foreach (var pagePosition in reader.TransactionPageTranslation)
                            {
                                var pageInJournal = pagePosition.Value.JournalPos;
                                var page = recoveryPager.Read(pageInJournal);
                                pageNumberToPageInScratch[pagePosition.Key] = pageInJournal;
                                if (page.IsOverflow)
                                {
                                    var numberOfOverflowPages = recoveryPager.GetNumberOfOverflowPages(page.OverflowSize);
                                    for (int i = 1; i < numberOfOverflowPages; i++)
                                        pageNumberToPageInScratch.Remove(page.PageNumber + i);
                                }
                            }
                        }
                    }
                    finally
                    {
                        journalFile.Release();
                    }
                }

                if (pageNumberToPageInScratch.Count == 0)
                {
                    infoNotify("Voron - no changes since last backup, nothing to do");
                    return;
                }

                infoNotify("Voron - started writing snapshot file.");

                if (lastTransaction.TransactionId == -1)
                    throw new InvalidOperationException("Could not find any transactions in the journals, but found pages to write? That ain't right.");

                // it is possible that we merged enough transactions so the _merged_ output is too large for us.
                // Voron limit transactions to about 4GB each. That means that we can't just merge all transactions
                // blindly, for fear of hitting this limit. So we need to split things.
                // We are also limited to about 8 TB of data in general before we literally can't fit the number of pages into
                // pageNumberToPageInScratch even theoretically.
                // We're fine with saying that you need to run min inc backup before you hit 8 TB in your increment, so that works for now.
                // We are also going to use env.Options.MaxScratchBufferSize to set the actual transaction limit here, to avoid issues
                // down the road and to limit how big a single transaction can be before the theoretical 4GB limit.

                var nextJournalNum = lastBackedUpFile;
                using (var file = new FileStream(backupPath, FileMode.Create))
                {
                    using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
                    {
                        var copier = new DataCopier(AbstractPager.PageSize * 16);

                        var finalPager = env.Options.CreateScratchPager("min-inc-backup-final.scratch");
                        toDispose.Add(finalPager);
                        finalPager.EnsureContinuous(null, 0, 1);//txHeader

                        foreach (var partition in Partition(pageNumberToPageInScratch.Values, env.Options.MaxNumberOfPagesInMergedTransaction))
                        {
                            int totalNumberOfPages = 0;
                            int overflowPages = 0;
                            int start = 1;
                            foreach (var pageNum in partition)
                            {
                                var p = recoveryPager.Read(pageNum);
                                var size = 1;
                                if (p.IsOverflow)
                                {
                                    size = recoveryPager.GetNumberOfOverflowPages(p.OverflowSize);
                                    overflowPages += (size - 1);
                                }
                                totalNumberOfPages += size;
                                finalPager.EnsureContinuous(null, start, size); //maybe increase size

                                Memory.Copy(finalPager.AcquirePagePointer(start), p.Base, size * AbstractPager.PageSize);

                                start += size;
                            }

                            var txPage = finalPager.AcquirePagePointer(0);
                            UnmanagedMemory.Set(txPage, 0, AbstractPager.PageSize);
                            var txHeader = (TransactionHeader*)txPage;
                            txHeader->HeaderMarker = Constants.TransactionHeaderMarker;
                            txHeader->FreeSpace = lastTransaction.FreeSpace;
                            txHeader->Root = lastTransaction.Root;
                            txHeader->OverflowPageCount = overflowPages;
                            txHeader->PageCount = totalNumberOfPages - overflowPages;
                            txHeader->PreviousTransactionCrc = lastTransaction.PreviousTransactionCrc;
                            txHeader->TransactionId = lastTransaction.TransactionId;
                            txHeader->NextPageNumber = lastTransaction.NextPageNumber;
                            txHeader->LastPageNumber = lastTransaction.LastPageNumber;
                            txHeader->TxMarker = TransactionMarker.Commit | TransactionMarker.Merged;
                            txHeader->Compressed = false;
                            txHeader->UncompressedSize = txHeader->CompressedSize = totalNumberOfPages * AbstractPager.PageSize;

                            txHeader->Crc = Crc.Value(finalPager.AcquirePagePointer(1), 0, totalNumberOfPages * AbstractPager.PageSize);

                            var entry = package.CreateEntry(string.Format("{0:D19}.merged-journal", nextJournalNum), compression);
                            nextJournalNum++;
                            using (var stream = entry.Open())
                            {
                                copier.ToStream(finalPager.AcquirePagePointer(0), (totalNumberOfPages + 1) * AbstractPager.PageSize, stream);
                            }
                        }

                    }
                    file.Flush(true);// make sure we hit the disk and stay there
                }

                env.HeaderAccessor.Modify(header =>
                {
                    header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile;
                    header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage;
                });
            }
            finally
            {
                foreach (var disposable in toDispose)
                    disposable.Dispose();
            }
        }
Example #3
0
        private void Restore(StorageEnvironment env, string singleBackupFile)
        {
            using (env.Journal.Applicator.TakeFlushingLock())
            {
                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    using (env.Options.AllowManualFlushing())
                    {
                        env.FlushLogToDataFile(txw);
                    }

                    using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read))
                    {
                        if (package.Entries.Count == 0)
                            return;

                        var toDispose = new List<IDisposable>();

						var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;

                        try
                        {
                            TransactionHeader* lastTxHeader = null;
                            var pagesToWrite = new Dictionary<long, Func<Page>>();

                            long journalNumber = -1;
                            foreach (var entry in package.Entries)
                            {
                                switch (Path.GetExtension(entry.Name))
                                {
                                    case ".journal":

										var jounalFileName = Path.Combine(tempDir, entry.Name);
                                        using (var output = new FileStream(jounalFileName, FileMode.Create))
                                        using (var input = entry.Open())
                                        {
                                            output.Position = output.Length;
                                            input.CopyTo(output);
                                        }

                                        var pager = new Win32MemoryMapPager(jounalFileName);
                                        toDispose.Add(pager);

                                        if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false)
                                        {
                                            throw new InvalidOperationException("Cannot parse journal file number");
                                        }

										var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
                                        toDispose.Add(recoveryPager);

                                        var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);

                                        while (reader.ReadOneTransaction(env.Options))
                                        {
                                            lastTxHeader = reader.LastTransactionHeader;
                                        }

                                        foreach (var translation in reader.TransactionPageTranslation)
                                        {
                                            var pageInJournal = translation.Value.JournalPos;
                                            pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal);
                                        }

                                        break;
                                    default:
                                        throw new InvalidOperationException("Unknown file, cannot restore: " + entry);
                                }
                            }

                            var sortedPages = pagesToWrite.OrderBy(x => x.Key)
                                .Select(x => x.Value())
                                .ToList();

                            var last = sortedPages.Last();

                            env.Options.DataPager.EnsureContinuous(txw, last.PageNumber,
                                last.IsOverflow
                                    ? env.Options.DataPager.GetNumberOfOverflowPages(
                                        last.OverflowSize)
                                    : 1);

                            foreach (var page in sortedPages)
                            {
                                env.Options.DataPager.Write(page);
                            }

                            env.Options.DataPager.Sync();

                            txw.State.Root = Tree.Open(txw, env._sliceComparer, &lastTxHeader->Root);
                            txw.State.FreeSpaceRoot = Tree.Open(txw, env._sliceComparer, &lastTxHeader->FreeSpace);

                            txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName;
                            txw.State.Root.Name = Constants.RootTreeName;

                            txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;

                            env.Journal.Clear(txw);

                            txw.Commit();

                            env.HeaderAccessor.Modify(header =>
                            {
                                header->TransactionId = lastTxHeader->TransactionId;
                                header->LastPageNumber = lastTxHeader->LastPageNumber;

                                header->Journal.LastSyncedJournal = journalNumber;
                                header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;

                                header->Root = lastTxHeader->Root;
                                header->FreeSpace = lastTxHeader->FreeSpace;

                                header->Journal.CurrentJournal = journalNumber + 1;
                                header->Journal.JournalFilesCount = 0;
                            });
                        }
                        finally
                        {
                            toDispose.ForEach(x => x.Dispose());

	                        try
	                        {
								Directory.Delete(tempDir, true);
	                        }
	                        catch (Exception)
	                        {
								// just temp dir - ignore it
	                        }
                        }
                    }
                }
            }
        }
Example #4
0
		public bool RecoverDatabase(TransactionHeader* txHeader)
		{
			// note, we don't need to do any concurrency here, happens as a single threaded
			// fashion on db startup
			var requireHeaderUpdate = false;

			var logInfo = _headerAccessor.Get(ptr => ptr->Journal);

			if (logInfo.JournalFilesCount == 0)
			{
				_journalIndex = logInfo.LastSyncedJournal;
				return false;
			}

			var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1;
			if (_env.Options.IncrementalBackupEnabled == false)
			{
				// we want to check that we cleanup old log files if they aren't needed
				// this is more just to be safe than anything else, they shouldn't be there.
				var unusedfiles = oldestLogFileStillInUse;
				while (true)
				{
					unusedfiles--;
					if (_env.Options.TryDeleteJournal(unusedfiles) == false)
						break;
				}

			}

			var lastSyncedTransactionId = logInfo.LastSyncedTransactionId;

			var journalFiles = new List<JournalFile>();
			long lastSyncedTxId = -1;
			long lastSyncedJournal = logInfo.LastSyncedJournal;
			uint lastShippedTxCrc = 0;
			for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++)
			{
				using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber)))
				using (var pager = _env.Options.OpenJournalPager(journalNumber))
				{
					RecoverCurrentJournalSize(pager);

					var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader;
					var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader);
					journalReader.RecoverAndValidate(_env.Options);

					var pagesToWrite = journalReader
						.TransactionPageTranslation
						.Select(kvp => recoveryPager.Read(kvp.Value.JournalPos))
						.OrderBy(x => x.PageNumber)
						.ToList();

					var lastReadHeaderPtr = journalReader.LastTransactionHeader;

					if (lastReadHeaderPtr != null)
					{
						if (pagesToWrite.Count > 0)
							ApplyPagesToDataFileFromJournal(pagesToWrite);

						*txHeader = *lastReadHeaderPtr;
						lastSyncedTxId = txHeader->TransactionId;
						lastShippedTxCrc = txHeader->Crc;
						lastSyncedJournal = journalNumber;
					}

					if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal)
					{
						var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize);
						var jrnlFile = new JournalFile(jrnlWriter, journalNumber);
						jrnlFile.InitFrom(journalReader);
						jrnlFile.AddRef(); // creator reference - write ahead log

						journalFiles.Add(jrnlFile);
					}

					if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions
					{
						requireHeaderUpdate = true;
						break;
					}
				}
			}

			Shipper.SetPreviousTransaction(lastSyncedTxId, lastShippedTxCrc);
			

			_files = _files.AppendRange(journalFiles);
			
			Debug.Assert(lastSyncedTxId >= 0);
			Debug.Assert(lastSyncedJournal >= 0);

			_journalIndex = lastSyncedJournal;

			_headerAccessor.Modify(
				header =>
				{
					header->Journal.LastSyncedJournal = lastSyncedJournal;
					header->Journal.LastSyncedTransactionId = lastSyncedTxId;
					header->Journal.CurrentJournal = lastSyncedJournal;
					header->Journal.JournalFilesCount = _files.Count;
					header->IncrementalBackup.LastCreatedJournal = _journalIndex;
					header->PreviousTransactionCrc = lastShippedTxCrc;
				});

			CleanupInvalidJournalFiles(lastSyncedJournal);
			CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal);

			if (_files.Count > 0)
			{
				var lastFile = _files.Last();
				if (lastFile.AvailablePages >= 2)
					// it must have at least one page for the next transaction header and one page for data
					CurrentFile = lastFile;
			}

			return requireHeaderUpdate;
		}
Example #5
0
        public bool RecoverDatabase(TransactionHeader *txHeader)
        {
            // note, we don't need to do any concurrency here, happens as a single threaded
            // fashion on db startup
            var requireHeaderUpdate = false;

            var logInfo = _headerAccessor.Get(ptr => ptr->Journal);

            if (logInfo.JournalFilesCount == 0)
            {
                _journalIndex = logInfo.LastSyncedJournal;
                return(false);
            }

            var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1;

            if (_env.Options.IncrementalBackupEnabled == false)
            {
                // we want to check that we cleanup old log files if they aren't needed
                // this is more just to be safe than anything else, they shouldn't be there.
                var unusedfiles = oldestLogFileStillInUse;
                while (true)
                {
                    unusedfiles--;
                    if (_env.Options.TryDeleteJournal(unusedfiles) == false)
                    {
                        break;
                    }
                }
            }

            var lastSyncedTransactionId = logInfo.LastSyncedTransactionId;

            var  journalFiles      = new List <JournalFile>();
            long lastSyncedTxId    = -1;
            long lastSyncedJournal = logInfo.LastSyncedJournal;
            uint lastShippedTxCrc  = 0;

            for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++)
            {
                using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber)))
                    using (var pager = _env.Options.OpenJournalPager(journalNumber))
                    {
                        RecoverCurrentJournalSize(pager);

                        var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader;
                        var journalReader     = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader);
                        journalReader.RecoverAndValidate(_env.Options);

                        var pagesToWrite = journalReader
                                           .TransactionPageTranslation
                                           .Select(kvp => recoveryPager.Read(kvp.Value.JournalPos))
                                           .OrderBy(x => x.PageNumber)
                                           .ToList();

                        var lastReadHeaderPtr = journalReader.LastTransactionHeader;

                        if (lastReadHeaderPtr != null)
                        {
                            if (pagesToWrite.Count > 0)
                            {
                                ApplyPagesToDataFileFromJournal(pagesToWrite);
                            }

                            *txHeader         = *lastReadHeaderPtr;
                            lastSyncedTxId    = txHeader->TransactionId;
                            lastShippedTxCrc  = txHeader->Crc;
                            lastSyncedJournal = journalNumber;
                        }

                        if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal)
                        {
                            var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize);
                            var jrnlFile   = new JournalFile(jrnlWriter, journalNumber);
                            jrnlFile.InitFrom(journalReader);
                            jrnlFile.AddRef();                     // creator reference - write ahead log

                            journalFiles.Add(jrnlFile);
                        }

                        if (journalReader.RequireHeaderUpdate)                 //this should prevent further loading of transactions
                        {
                            requireHeaderUpdate = true;
                            break;
                        }
                    }
            }

            Shipper.SetPreviousTransaction(lastSyncedTxId, lastShippedTxCrc);


            _files = _files.AppendRange(journalFiles);

            Debug.Assert(lastSyncedTxId >= 0);
            Debug.Assert(lastSyncedJournal >= 0);

            _journalIndex = lastSyncedJournal;

            _headerAccessor.Modify(
                header =>
            {
                header->Journal.LastSyncedJournal            = lastSyncedJournal;
                header->Journal.LastSyncedTransactionId      = lastSyncedTxId;
                header->Journal.CurrentJournal               = lastSyncedJournal;
                header->Journal.JournalFilesCount            = _files.Count;
                header->IncrementalBackup.LastCreatedJournal = _journalIndex;
                header->PreviousTransactionCrc               = lastShippedTxCrc;
            });

            CleanupInvalidJournalFiles(lastSyncedJournal);
            CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal);

            if (_files.Count > 0)
            {
                var lastFile = _files.Last();
                if (lastFile.AvailablePages >= 2)
                {
                    // it must have at least one page for the next transaction header and one page for data
                    CurrentFile = lastFile;
                }
            }

            return(requireHeaderUpdate);
        }
Example #6
0
 public void InitFrom(JournalReader journalReader, TransactionHeader[] transactionHeaders, int transactionsCount)
 {
     _writePosIn4Kb = journalReader.Next4Kb;
     Array.Copy(transactionHeaders, _transactionHeaders, transactionsCount);
 }
Example #7
0
 public void InitFrom(JournalReader journalReader, List <TransactionHeader> transactionHeaders)
 {
     _writePosIn4Kb      = journalReader.Next4Kb;
     _transactionHeaders = new List <TransactionHeader>(transactionHeaders);
 }
Example #8
0
 public void InitFrom(JournalReader journalReader)
 {
     _writePage = journalReader.NextWritePage;
 }
Example #9
0
 public void InitFrom(JournalReader journalReader)
 {
     _writePosIn4Kb = journalReader.Next4Kb;
 }