private static void ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage)
{
// Fill BLOCKS with data from known local source files
using (var blockmarker = database.CreateBlockMarker())
{
var updateCount = 0L;
foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize))
{
var targetpath = restorelist.TargetPath;
var targetfileid = restorelist.TargetFileID;
var patched = false;
try
{
if (result.TaskControlRendevouz() == TaskControlState.Stop)
return;
var folderpath = m_systemIO.PathGetDirectoryName(targetpath);
if (!options.Dryrun && !m_systemIO.DirectoryExists(folderpath))
{
result.AddWarning(string.Format("Creating missing folder {0} for file {1}", folderpath, targetpath), null);
m_systemIO.DirectoryCreate(folderpath);
}
using (var file = options.Dryrun ? null : m_systemIO.FileOpenReadWrite(targetpath))
using (var block = new Blockprocessor(file, blockbuffer))
foreach (var targetblock in restorelist.Blocks)
{
if (!options.Dryrun && !targetblock.IsMetadata)
file.Position = targetblock.Offset;
foreach (var source in targetblock.Blocksources)
{
try
{
if (result.TaskControlRendevouz() == TaskControlState.Stop)
return;
if (m_systemIO.FileExists(source.Path))
{
if (source.IsMetadata)
{
// TODO: Handle this by reconstructing
// metadata from file and checking the hash
continue;
}
else
{
using (var sourcefile = m_systemIO.FileOpenRead(source.Path))
{
sourcefile.Position = source.Offset;
var size = sourcefile.Read(blockbuffer, 0, blockbuffer.Length);
if (size == targetblock.Size)
{
var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size));
if (key == targetblock.Hash)
{
if (!options.Dryrun)
{
if (targetblock.IsMetadata)
metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size));
else
file.Write(blockbuffer, 0, size);
}
blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false);
patched = true;
break;
}
}
}
}
}
}
catch (Exception ex)
{
result.AddWarning(string.Format("Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
throw;
}
}
}
if (updateCount++ % 20 == 0)
blockmarker.UpdateProcessed(result.OperationProgressUpdater);
}
catch (Exception ex)
{
result.AddWarning(string.Format("Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message), ex);
}
if (patched)
result.AddVerboseMessage("Target file is patched with some local data: {0}", targetpath);
else
result.AddVerboseMessage("Target file is not patched any local data: {0}", targetpath);
if (patched && options.Dryrun)
result.AddDryrunMessage(string.Format("Would patch file with local data: {0}", targetpath));
}
blockmarker.UpdateProcessed(result.OperationProgressUpdater);
blockmarker.Commit(result);
}
}
private static void PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage)
{
var blocksize = options.Blocksize;
var updateCounter = 0L;
var fullblockverification = options.FullBlockVerification;
var blockhasher = fullblockverification ? System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm) : null;
using(var blockmarker = database.CreateBlockMarker())
using(var volumekeeper = database.GetMissingBlockData(blocks, options.Blocksize))
{
foreach(var restorelist in volumekeeper.FilesWithMissingBlocks)
{
var targetpath = restorelist.Path;
result.AddVerboseMessage("Patching file with remote data: {0}", targetpath);
if (options.Dryrun)
{
result.AddDryrunMessage(string.Format("Would patch file with remote data: {0}", targetpath));
}
else
{
try
{
var folderpath = m_systemIO.PathGetDirectoryName(targetpath);
if (!options.Dryrun && !m_systemIO.DirectoryExists(folderpath))
{
result.AddWarning(string.Format("Creating missing folder {0} for file {1}", folderpath, targetpath), null);
m_systemIO.DirectoryCreate(folderpath);
}
// TODO: Much faster if we iterate the volume and checks what blocks are used,
// because the compressors usually like sequential reading
using(var file = m_systemIO.FileOpenReadWrite(targetpath))
foreach(var targetblock in restorelist.Blocks)
{
file.Position = targetblock.Offset;
var size = blocks.ReadBlock(targetblock.Key, blockbuffer);
if (targetblock.Size == size)
{
var valid = !fullblockverification;
if (!valid)
{
blockhasher.Initialize();
var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size));
if (targetblock.Key == key)
valid = true;
else
result.AddWarning(string.Format("Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key), null);
}
if (valid)
{
file.Write(blockbuffer, 0, size);
blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false);
}
}
}
if (updateCounter++ % 20 == 0)
blockmarker.UpdateProcessed(result.OperationProgressUpdater);
}
catch (Exception ex)
{
result.AddWarning(string.Format("Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message), ex);
}
}
}
if (!options.SkipMetadata)
{
foreach(var restoremetadata in volumekeeper.MetadataWithMissingBlocks)
{
var targetpath = restoremetadata.Path;
result.AddVerboseMessage("Recording metadata from remote data: {0}", targetpath);
try
{
// TODO: When we support multi-block metadata this needs to deal with it
using(var ms = new System.IO.MemoryStream())
{
foreach(var targetblock in restoremetadata.Blocks)
{
ms.Position = targetblock.Offset;
var size = blocks.ReadBlock(targetblock.Key, blockbuffer);
if (targetblock.Size == size)
{
ms.Write(blockbuffer, 0, size);
blockmarker.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true);
}
}
ms.Position = 0;
metadatastorage.Add(targetpath, ms);
//blockmarker.RecordMetadata(restoremetadata.FileID, ms);
}
}
catch (Exception ex)
{
result.AddWarning(string.Format("Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message), ex);
}
}
}
blockmarker.UpdateProcessed(result.OperationProgressUpdater);
blockmarker.Commit(result);
}
}