private void DoRun(LocalDatabase dbparent, Library.Utility.IFilter filter, RestoreResults result)
{
//In this case, we check that the remote storage fits with the database.
//We can then query the database and find the blocks that we need to do the restore
using(var database = new LocalRestoreDatabase(dbparent, m_options.Blocksize))
using(var backend = new BackendManager(m_backendurl, m_options, result.BackendWriter, database))
{
database.SetResult(m_result);
Utility.VerifyParameters(database, m_options);
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (blockhasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
if (!blockhasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
if (filehasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm));
if (!filehasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm));
if (!m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
FilelistProcessor.VerifyRemoteList(backend, m_options, database, result.BackendWriter);
}
//Figure out what files are to be patched, and what blocks are needed
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList);
using(new Logging.Timer("PrepareBlockList"))
PrepareBlockAndFileList(database, m_options, filter, result);
//Make the entire output setup
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders);
using(new Logging.Timer("CreateDirectory"))
CreateDirectoryStructure(database, m_options, result);
//If we are patching an existing target folder, do not touch stuff that is already updated
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles);
using(new Logging.Timer("ScanForexistingTargetBlocks"))
ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, result);
//Look for existing blocks in the original source files only
using(new Logging.Timer("ScanForExistingSourceBlocksFast"))
#if DEBUG
if (!m_options.NoLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath))
#else
if (!string.IsNullOrEmpty(m_options.Restorepath))
#endif
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks);
ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, result);
}
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
// If other local files already have the blocks we want, we use them instead of downloading
if (m_options.PatchWithLocalBlocks)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks);
using(new Logging.Timer("PatchWithLocalBlocks"))
ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, result);
}
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
// Fill BLOCKS with remote sources
var volumes = database.GetMissingVolumes().ToList();
if (volumes.Count > 0)
{
m_result.AddMessage(string.Format("{0} remote files are required to restore", volumes.Count));
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles);
}
var brokenFiles = new List<string>();
foreach(var blockvolume in new AsyncDownloader(volumes, backend))
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
using(var tmpfile = blockvolume.TempFile)
using(var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Name), tmpfile, m_options))
PatchWithBlocklist(database, blocks, m_options, result, m_blockbuffer);
}
catch (Exception ex)
{
brokenFiles.Add(blockvolume.Name);
result.AddError(string.Format("Failed to patch with remote file: \"{0}\", message: {1}", blockvolume.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
throw;
}
// Reset the filehasher if it was used to verify existing files
filehasher.Initialize();
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
return;
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify);
var fileErrors = 0L;
// After all blocks in the files are restored, verify the file hash
using(new Logging.Timer("RestoreVerification"))
foreach (var file in database.GetFilesToRestore())
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
result.AddVerboseMessage("Testing restored file integrity: {0}", file.Path);
string key;
long size;
using (var fs = m_systemIO.FileOpenRead(file.Path))
{
size = fs.Length;
key = Convert.ToBase64String(filehasher.ComputeHash(fs));
}
if (key != file.Hash)
throw new Exception(string.Format("Failed to restore file: \"{0}\". File hash is {1}, expected hash is {2}", file.Path, key, file.Hash));
result.FilesRestored++;
result.SizeOfRestoredFiles += size;
}
catch (Exception ex)
{
fileErrors++;
result.AddWarning(ex.Message, ex);
if (ex is System.Threading.ThreadAbortException)
throw;
}
}
if (fileErrors > 0 && brokenFiles.Count > 0)
m_result.AddMessage(string.Format("Failed to restore {0} files, additionally the following files failed to download, which may be the cause:{1}", fileErrors, Environment.NewLine, string.Join(Environment.NewLine, brokenFiles)));
// Drop the temp tables
database.DropRestoreTable();
backend.WaitForComplete(database, null);
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete);
result.EndTime = DateTime.UtcNow;
}
private void DoRun(LocalDatabase dbparent, Library.Utility.IFilter filter, RestoreResults result)
{
//In this case, we check that the remote storage fits with the database.
//We can then query the database and find the blocks that we need to do the restore
using (var database = new LocalRestoreDatabase(dbparent, m_options.Blocksize))
using (var backend = new BackendManager(m_backendurl, m_options, result.BackendWriter, database))
{
database.SetResult(m_result);
Utility.VerifyParameters(database, m_options);
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (blockhasher == null)
{
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
}
if (!blockhasher.CanReuseTransform)
{
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
}
if (filehasher == null)
{
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm));
}
if (!filehasher.CanReuseTransform)
{
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm));
}
if (!m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
FilelistProcessor.VerifyRemoteList(backend, m_options, database, result.BackendWriter);
}
//Figure out what files are to be patched, and what blocks are needed
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList);
using (new Logging.Timer("PrepareBlockList"))
PrepareBlockAndFileList(database, m_options, filter, result);
//Make the entire output setup
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders);
using (new Logging.Timer("CreateDirectory"))
CreateDirectoryStructure(database, m_options, result);
//If we are patching an existing target folder, do not touch stuff that is already updated
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles);
using (new Logging.Timer("ScanForexistingTargetBlocks"))
ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, result);
//Look for existing blocks in the original source files only
using (new Logging.Timer("ScanForExistingSourceBlocksFast"))
#if DEBUG
if (!m_options.NoLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath))
#else
if (!string.IsNullOrEmpty(m_options.Restorepath))
#endif
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks);
ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, result);
}
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
// If other local files already have the blocks we want, we use them instead of downloading
if (m_options.PatchWithLocalBlocks)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks);
using (new Logging.Timer("PatchWithLocalBlocks"))
ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, result);
}
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
// Fill BLOCKS with remote sources
var volumes = database.GetMissingVolumes().ToList();
if (volumes.Count > 0)
{
m_result.AddMessage(string.Format("{0} remote files are required to restore", volumes.Count));
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles);
}
var brokenFiles = new List <string>();
foreach (var blockvolume in new AsyncDownloader(volumes, backend))
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
using (var tmpfile = blockvolume.TempFile)
using (var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Name), tmpfile, m_options))
PatchWithBlocklist(database, blocks, m_options, result, m_blockbuffer);
}
catch (Exception ex)
{
brokenFiles.Add(blockvolume.Name);
result.AddError(string.Format("Failed to patch with remote file: \"{0}\", message: {1}", blockvolume.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
// Reset the filehasher if it was used to verify existing files
filehasher.Initialize();
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
return;
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify);
var fileErrors = 0L;
// After all blocks in the files are restored, verify the file hash
using (new Logging.Timer("RestoreVerification"))
foreach (var file in database.GetFilesToRestore())
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(database, null);
return;
}
result.AddVerboseMessage("Testing restored file integrity: {0}", file.Path);
string key;
long size;
using (var fs = m_systemIO.FileOpenRead(file.Path))
{
size = fs.Length;
key = Convert.ToBase64String(filehasher.ComputeHash(fs));
}
if (key != file.Hash)
{
throw new Exception(string.Format("Failed to restore file: \"{0}\". File hash is {1}, expected hash is {2}", file.Path, key, file.Hash));
}
result.FilesRestored++;
result.SizeOfRestoredFiles += size;
}
catch (Exception ex)
{
fileErrors++;
result.AddWarning(ex.Message, ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
if (fileErrors > 0 && brokenFiles.Count > 0)
{
m_result.AddMessage(string.Format("Failed to restore {0} files, additionally the following files failed to download, which may be the cause:{1}", fileErrors, Environment.NewLine, string.Join(Environment.NewLine, brokenFiles)));
}
// Drop the temp tables
database.DropRestoreTable();
backend.WaitForComplete(database, null);
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete);
result.EndTime = DateTime.UtcNow;
}
