public void Run(string[] paths, Library.Utility.IFilter filter = null) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin); // If we have both target paths and a filter, combine into a single filter filter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(paths), filter); if (!m_options.NoLocalDb && m_systemIO.FileExists(m_options.Dbpath)) { using(var db = new LocalRestoreDatabase(m_options.Dbpath, m_options.Blocksize)) { db.SetResult(m_result); DoRun(db, filter, m_result); db.WriteResults(); } return; } m_result.AddMessage("No local database, building a temporary database"); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_RecreateDatabase); using(var tmpdb = new Library.Utility.TempFile()) { RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); // Simultaneously with downloading blocklists, we patch as much as we can from the blockvolumes // This prevents repeated downloads, except for cases where the blocklists refer blocks // that have been previously handled. A local blockvolume cache can reduce this issue using(var database = new LocalRestoreDatabase(tmpdb, m_options.Blocksize)) { var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm); if (blockhasher == null) throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm)); if (!blockhasher.CanReuseTransform) throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm)); if (filehasher == null) throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm)); if (!filehasher.CanReuseTransform) throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm)); bool first = true; RecreateDatabaseHandler.BlockVolumePostProcessor localpatcher = (key, rd) => { if (first) { //Figure out what files are to be patched, and what blocks are needed PrepareBlockAndFileList(database, m_options, filter, m_result); // Don't run this again first = false; } else { // Patch the missing blocks list to include the newly discovered blocklists //UpdateMissingBlocksTable(key); } if (m_result.TaskControlRendevouz() == TaskControlState.Stop) return; CreateDirectoryStructure(database, m_options, m_result); //If we are patching an existing target folder, do not touch stuff that is already updated ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result); if (m_result.TaskControlRendevouz() == TaskControlState.Stop) return; #if DEBUG if (!m_options.NoLocalBlocks) #endif // If other local files already have the blocks we want, we use them instead of downloading ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result); if (m_result.TaskControlRendevouz() == TaskControlState.Stop) return; //Update files with data PatchWithBlocklist(database, rd, m_options, m_result, m_blockbuffer); }; // TODO: When UpdateMissingBlocksTable is implemented, the localpatcher can be activated // and this will reduce the need for multiple downloads of the same volume // TODO: This will need some work to preserve the missing block list for use with --fh-dryrun m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using(new Logging.Timer("Recreate temporary database for restore")) new RecreateDatabaseHandler(m_backendurl, m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) .DoRun(database, filter, filelistfilter, /*localpatcher*/null); //If we have --version set, we need to adjust, as the db has only the required versions //TODO: Bit of a hack to set options that way if (m_options.Version != null && m_options.Version.Length > 0) m_options.RawOptions["version"] = string.Join(",", Enumerable.Range(0, m_options.Version.Length).Select(x => x.ToString())); DoRun(database, filter, m_result); } } }
private void DoRun(LocalDatabase dbparent, Library.Utility.IFilter filter, RestoreResults result) { //In this case, we check that the remote storage fits with the database. //We can then query the database and find the blocks that we need to do the restore using(var database = new LocalRestoreDatabase(dbparent, m_options.Blocksize)) using(var backend = new BackendManager(m_backendurl, m_options, result.BackendWriter, database)) { database.SetResult(m_result); Utility.VerifyParameters(database, m_options); var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm); if (blockhasher == null) throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm)); if (!blockhasher.CanReuseTransform) throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm)); if (filehasher == null) throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm)); if (!filehasher.CanReuseTransform) throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm)); if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); FilelistProcessor.VerifyRemoteList(backend, m_options, database, result.BackendWriter); } //Figure out what files are to be patched, and what blocks are needed m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using(new Logging.Timer("PrepareBlockList")) PrepareBlockAndFileList(database, m_options, filter, result); //Make the entire output setup m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using(new Logging.Timer("CreateDirectory")) CreateDirectoryStructure(database, m_options, result); //If we are patching an existing target folder, do not touch stuff that is already updated m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles); using(new Logging.Timer("ScanForexistingTargetBlocks")) ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, result); //Look for existing blocks in the original source files only using(new Logging.Timer("ScanForExistingSourceBlocksFast")) #if DEBUG if (!m_options.NoLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath)) #else if (!string.IsNullOrEmpty(m_options.Restorepath)) #endif { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks); ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, result); } if (m_result.TaskControlRendevouz() == TaskControlState.Stop) { backend.WaitForComplete(database, null); return; } // If other local files already have the blocks we want, we use them instead of downloading if (m_options.PatchWithLocalBlocks) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks); using(new Logging.Timer("PatchWithLocalBlocks")) ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, result); } if (m_result.TaskControlRendevouz() == TaskControlState.Stop) { backend.WaitForComplete(database, null); return; } // Fill BLOCKS with remote sources var volumes = database.GetMissingVolumes().ToList(); if (volumes.Count > 0) { m_result.AddMessage(string.Format("{0} remote files are required to restore", volumes.Count)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); } var brokenFiles = new List<string>(); foreach(var blockvolume in new AsyncDownloader(volumes, backend)) try { if (m_result.TaskControlRendevouz() == TaskControlState.Stop) { backend.WaitForComplete(database, null); return; } using(var tmpfile = blockvolume.TempFile) using(var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Name), tmpfile, m_options)) PatchWithBlocklist(database, blocks, m_options, result, m_blockbuffer); } catch (Exception ex) { brokenFiles.Add(blockvolume.Name); result.AddError(string.Format("Failed to patch with remote file: \"{0}\", message: {1}", blockvolume.Name, ex.Message), ex); if (ex is System.Threading.ThreadAbortException) throw; } // Reset the filehasher if it was used to verify existing files filehasher.Initialize(); if (m_result.TaskControlRendevouz() == TaskControlState.Stop) return; m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify); var fileErrors = 0L; // After all blocks in the files are restored, verify the file hash using(new Logging.Timer("RestoreVerification")) foreach (var file in database.GetFilesToRestore()) { try { if (m_result.TaskControlRendevouz() == TaskControlState.Stop) { backend.WaitForComplete(database, null); return; } result.AddVerboseMessage("Testing restored file integrity: {0}", file.Path); string key; long size; using (var fs = m_systemIO.FileOpenRead(file.Path)) { size = fs.Length; key = Convert.ToBase64String(filehasher.ComputeHash(fs)); } if (key != file.Hash) throw new Exception(string.Format("Failed to restore file: \"{0}\". File hash is {1}, expected hash is {2}", file.Path, key, file.Hash)); result.FilesRestored++; result.SizeOfRestoredFiles += size; } catch (Exception ex) { fileErrors++; result.AddWarning(ex.Message, ex); if (ex is System.Threading.ThreadAbortException) throw; } } if (fileErrors > 0 && brokenFiles.Count > 0) m_result.AddMessage(string.Format("Failed to restore {0} files, additionally the following files failed to download, which may be the cause:{1}", fileErrors, Environment.NewLine, string.Join(Environment.NewLine, brokenFiles))); // Drop the temp tables database.DropRestoreTable(); backend.WaitForComplete(database, null); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); result.EndTime = DateTime.UtcNow; }