public void RunRepairRemote()
{
if (!System.IO.File.Exists(m_options.Dbpath))
{
throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
}
m_result.OperationProgressUpdater.UpdateProgress(0);
using (var db = new LocalRepairDatabase(m_options.Dbpath))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
m_result.SetDatabase(db);
Utility.VerifyParameters(db, m_options);
var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter);
var buffer = new byte[m_options.Blocksize];
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
var hashsize = blockhasher.HashSize / 8;
if (blockhasher == null)
{
throw new Exception(Strings.Foresthash.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
}
if (!blockhasher.CanReuseTransform)
{
throw new Exception(Strings.Foresthash.InvalidCryptoSystem(m_options.BlockHashAlgorithm));
}
var progress = 0;
var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count();
if (m_options.Dryrun)
{
if (tp.ParsedVolumes.Count() == 0 && tp.OtherVolumes.Count() > 0)
{
if (tp.BackupPrefixes.Length == 1)
{
throw new Exception(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup-prefix?", m_options.Prefix, tp.BackupPrefixes[0]));
}
else
{
throw new Exception(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup-prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)));
}
}
else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Count() > 0)
{
throw new Exception(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()));
}
}
if (tp.ExtraVolumes.Count() > 0 || tp.MissingVolumes.Count() > 0 || tp.VerificationRequiredVolumes.Count() > 0)
{
if (tp.VerificationRequiredVolumes.Any())
{
using (var testdb = new LocalTestDatabase(db))
{
foreach (var n in tp.VerificationRequiredVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
long size;
string hash;
KeyValuePair <string, IEnumerable <KeyValuePair <Duplicati.Library.Interface.TestEntryStatus, string> > > res;
using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, m_result, 1);
if (res.Value.Any())
{
throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First()));
}
if (!m_options.Dryrun)
{
m_result.AddMessage(string.Format("Sucessfully captured hash for {0}, updating database", n.Name));
db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash);
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
}
// TODO: It is actually possible to use the extra files if we parse them
foreach (var n in tp.ExtraVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (!m_options.Dryrun)
{
db.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Deleting);
backend.Delete(n.File.Name, n.File.Size);
}
else
{
m_result.AddDryrunMessage(string.Format("would delete file {0}", n.File.Name));
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
foreach (var n in tp.MissingVolumes)
{
IDisposable newEntry = null;
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (n.Type == RemoteVolumeType.Files)
{
var filesetId = db.GetFilesetIdFromRemotename(n.Name);
var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
newEntry = w;
w.SetRemoteFilename(n.Name);
db.WriteFileset(w, null, filesetId);
w.Close();
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Index)
{
var w = new IndexVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
{
w.StartVolume(blockvolume.Name);
var volumeid = db.GetRemoteVolumeID(blockvolume.Name);
foreach (var b in db.GetBlocks(volumeid))
{
w.AddBlock(b.Hash, b.Size);
}
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
{
foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize))
{
w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
}
}
}
w.Close();
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Blocks)
{
var w = new BlockVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
using (var mbl = db.CreateBlockList(n.Name))
{
//First we grab all known blocks from local files
foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize))
{
var hash = block.Hash;
var size = (int)block.Size;
foreach (var source in block.Sources)
{
var file = source.File;
var offset = source.Offset;
try
{
if (System.IO.File.Exists(file))
{
using (var f = System.IO.File.OpenRead(file))
{
f.Position = offset;
if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size))
{
var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size));
if (newhash == hash)
{
if (mbl.SetBlockRestored(hash, size))
{
w.AddBlock(hash, buffer, 0, size, Duplicati.Library.Interface.CompressionHint.Default);
}
break;
}
}
}
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to access file: {0}", file), ex);
}
}
}
//Then we grab all remote volumes that have the missing blocks
foreach (var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
{
try
{
using (var tmpfile = vol.TempFile)
using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options))
foreach (var b in f.Blocks)
{
if (mbl.SetBlockRestored(b.Key, b.Value))
{
if (f.ReadBlock(b.Key, buffer) == b.Value)
{
w.AddBlock(b.Key, buffer, 0, (int)b.Value, Duplicati.Library.Interface.CompressionHint.Default);
}
}
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to access remote file: {0}", vol.Name), ex);
}
}
// If we managed to recover all blocks, NICE!
var missingBlocks = mbl.GetMissingBlocks().Count();
if (missingBlocks > 0)
{
//TODO: How do we handle this situation?
m_result.AddMessage(string.Format("Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name));
foreach (var f in mbl.GetFilesetsUsingMissingBlocks())
{
m_result.AddMessage(f.Name);
}
if (!m_options.Dryrun)
{
m_result.AddMessage("This may be fixed by deleting the filesets and running repair again");
throw new Exception(string.Format("Repair not possible, missing {0} blocks!!!", missingBlocks));
}
}
else
{
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
}
}
}
catch (Exception ex)
{
if (newEntry != null)
{
try { newEntry.Dispose(); }
catch { }
finally { newEntry = null; }
}
m_result.AddError(string.Format("Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
else
{
m_result.AddMessage("Destination and database are synchronized, not making any changes");
}
m_result.OperationProgressUpdater.UpdateProgress(1);
backend.WaitForComplete(db, null);
db.WriteResults();
}
}
public void RunRepairRemote()
{
if (!System.IO.File.Exists(m_options.Dbpath))
{
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "RepairDatabaseFileDoesNotExist");
}
m_result.OperationProgressUpdater.UpdateProgress(0);
using (var db = new LocalRepairDatabase(m_options.Dbpath))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
m_result.SetDatabase(db);
Utility.UpdateOptionsFromDb(db, m_options);
Utility.VerifyParameters(db, m_options);
if (db.PartiallyRecreated)
{
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");
}
if (db.RepairInProgress)
{
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState");
}
var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter, null);
var buffer = new byte[m_options.Blocksize];
var blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
var hashsize = blockhasher.HashSize / 8;
if (blockhasher == null)
{
throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported");
}
if (!blockhasher.CanReuseTransform)
{
throw new UserInformationException(Strings.Common.InvalidCryptoSystem(m_options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported");
}
var progress = 0;
var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count();
if (m_options.Dryrun)
{
if (tp.ParsedVolumes.Count() == 0 && tp.OtherVolumes.Count() > 0)
{
if (tp.BackupPrefixes.Length == 1)
{
throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]), "RemoteFolderEmptyWithPrefix");
}
else
{
throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)), "RemoteFolderEmptyWithPrefix");
}
}
else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Count() > 0)
{
throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()), "NoRemoteFilesMissing");
}
}
if (tp.ExtraVolumes.Count() > 0 || tp.MissingVolumes.Count() > 0 || tp.VerificationRequiredVolumes.Count() > 0)
{
if (tp.VerificationRequiredVolumes.Any())
{
using (var testdb = new LocalTestDatabase(db))
{
foreach (var n in tp.VerificationRequiredVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
long size;
string hash;
KeyValuePair <string, IEnumerable <KeyValuePair <Duplicati.Library.Interface.TestEntryStatus, string> > > res;
using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1);
if (res.Value.Any())
{
throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First()));
}
if (!m_options.Dryrun)
{
Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name);
db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash);
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
}
// TODO: It is actually possible to use the extra files if we parse them
foreach (var n in tp.ExtraVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
// If this is a new index file, we can accept it if it matches our local data
// This makes it possible to augment the remote store with new index data
if (n.FileType == RemoteVolumeType.Index && m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
try
{
string hash;
long size;
using (var tf = backend.GetWithInfo(n.File.Name, out size, out hash))
using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize))
{
foreach (var rv in ifr.Volumes)
{
var entry = db.GetRemoteVolume(rv.Filename);
if (entry.ID < 0)
{
throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename));
}
if (!new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State))
{
throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, entry.State));
}
if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State))
{
throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length));
}
db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks);
}
var blocksize = m_options.Blocksize;
foreach (var ixb in ifr.BlockLists)
{
db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize);
}
var selfid = db.GetRemoteVolumeID(n.File.Name);
foreach (var rv in ifr.Volumes)
{
db.AddIndexBlockLink(selfid, db.GetRemoteVolumeID(rv.Filename), null);
}
}
// All checks fine, we accept the new index file
Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name);
db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, size, RemoteVolumeState.Uploading);
db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash);
continue;
}
catch (Exception rex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "FailedNewIndexFile", rex, "Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message);
}
}
if (!m_options.Dryrun)
{
db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting);
backend.Delete(n.File.Name, n.File.Size);
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name);
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "FailedExtraFileCleanup", ex, "Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
if (!m_options.RebuildMissingDblockFiles)
{
var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks).ToArray();
if (missingDblocks.Length > 0)
{
throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles");
}
}
foreach (var n in tp.MissingVolumes)
{
IDisposable newEntry = null;
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (n.Type == RemoteVolumeType.Files)
{
var filesetId = db.GetFilesetIdFromRemotename(n.Name);
var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
newEntry = w;
w.SetRemoteFilename(n.Name);
db.WriteFileset(w, filesetId, null);
w.Close();
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Index)
{
var w = new IndexVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
var h = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
{
w.StartVolume(blockvolume.Name);
var volumeid = db.GetRemoteVolumeID(blockvolume.Name);
foreach (var b in db.GetBlocks(volumeid))
{
w.AddBlock(b.Hash, b.Size);
}
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
{
foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize))
{
var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
if (bh != b.Item1)
{
throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
}
w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
}
}
}
w.Close();
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Blocks)
{
var w = new BlockVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
using (var mbl = db.CreateBlockList(n.Name))
{
//First we grab all known blocks from local files
foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize))
{
var hash = block.Hash;
var size = (int)block.Size;
foreach (var source in block.Sources)
{
var file = source.File;
var offset = source.Offset;
try
{
if (System.IO.File.Exists(file))
{
using (var f = System.IO.File.OpenRead(file))
{
f.Position = offset;
if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size))
{
var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size));
if (newhash == hash)
{
if (mbl.SetBlockRestored(hash, size))
{
w.AddBlock(hash, buffer, 0, size, Duplicati.Library.Interface.CompressionHint.Default);
}
break;
}
}
}
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", file);
}
}
}
//Then we grab all remote volumes that have the missing blocks
foreach (var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
{
try
{
using (var tmpfile = vol.TempFile)
using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options))
foreach (var b in f.Blocks)
{
if (mbl.SetBlockRestored(b.Key, b.Value))
{
if (f.ReadBlock(b.Key, buffer) == b.Value)
{
w.AddBlock(b.Key, buffer, 0, (int)b.Value, Duplicati.Library.Interface.CompressionHint.Default);
}
}
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", vol.Name);
}
}
// If we managed to recover all blocks, NICE!
var missingBlocks = mbl.GetMissingBlocks().Count();
if (missingBlocks > 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name);
foreach (var f in mbl.GetFilesetsUsingMissingBlocks())
{
Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name);
}
var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files");
if (!m_options.Dryrun)
{
Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", "This may be fixed by deleting the filesets and running repair again");
throw new UserInformationException(string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks), "RepairIsNotPossible");
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", recoverymsg);
}
}
else
{
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadBlockFile", "would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
}
}
}
catch (Exception ex)
{
if (newEntry != null)
{
try { newEntry.Dispose(); }
catch { }
finally { newEntry = null; }
}
Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "DatabaseIsSynchronized", "Destination and database are synchronized, not making any changes");
}
m_result.OperationProgressUpdater.UpdateProgress(1);
backend.WaitForComplete(db, null);
db.WriteResults();
}
}
internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction, BackendManager sharedBackend)
{
var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction);
report.ReportCompactData();
if (report.ShouldReclaim || report.ShouldCompact)
{
// Workaround where we allow a running backendmanager to be used
using (var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
{
var backend = bk ?? sharedBackend;
if (!hasVerifiedBackend && !m_options.NoBackendverification)
{
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
}
BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
IndexVolumeWriter newvolindex = null;
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
newvolindex = new IndexVolumeWriter(m_options);
newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
}
long blocksInVolume = 0;
byte[] buffer = new byte[m_options.Blocksize];
var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
//These are for bookkeeping
var uploadedVolumes = new List <KeyValuePair <string, long> >();
var deletedVolumes = new List <KeyValuePair <string, long> >();
var downloadedVolumes = new List <KeyValuePair <string, long> >();
//We start by deleting unused volumes to save space before uploading new stuff
var fullyDeleteable = (from v in remoteList
where report.DeleteableVolumes.Contains(v.Name)
select(IRemoteVolume) v).ToList();
deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction));
// This list is used to pick up unused volumes,
// so they can be deleted once the upload of the
// required fragments is complete
var deleteableVolumes = new List <IRemoteVolume>();
if (report.ShouldCompact)
{
newvolindex?.StartVolume(newvol.RemoteFilename);
var volumesToDownload = (from v in remoteList
where report.CompactableVolumes.Contains(v.Name)
select(IRemoteVolume) v).ToList();
using (var q = db.CreateBlockQueryHelper(transaction))
{
foreach (var entry in new AsyncDownloader(volumesToDownload, backend))
{
using (var tmpfile = entry.TempFile)
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, transaction);
return(false);
}
downloadedVolumes.Add(new KeyValuePair <string, long>(entry.Name, entry.Size));
var inst = VolumeBase.ParseFilename(entry.Name);
using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
{
foreach (var e in f.Blocks)
{
if (q.UseBlock(e.Key, e.Value, transaction))
{
//TODO: How do we get the compression hint? Reverse query for filename in db?
var s = f.ReadBlock(e.Key, buffer);
if (s != e.Value)
{
throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
}
newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
if (newvolindex != null)
{
newvolindex.AddBlock(e.Key, e.Value);
}
db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction);
blocksInVolume++;
if (newvol.Filesize > m_options.VolumeSize)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvol.RemoteFilename, newvol.Filesize));
if (newvolindex != null)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
}
if (!m_options.Dryrun)
{
backend.Put(newvol, newvolindex);
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadGeneratedBlockset", "Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize));
}
newvol = new BlockVolumeWriter(m_options);
newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
newvolindex = new IndexVolumeWriter(m_options);
newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
newvolindex.StartVolume(newvol.RemoteFilename);
}
blocksInVolume = 0;
//After we upload this volume, we can delete all previous encountered volumes
deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
deleteableVolumes = new List <IRemoteVolume>();
}
}
}
}
deleteableVolumes.Add(entry);
}
}
if (blocksInVolume > 0)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvol.RemoteFilename, newvol.Filesize));
if (newvolindex != null)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
}
if (!m_options.Dryrun)
{
backend.Put(newvol, newvolindex);
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadGeneratedBlockset", "Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize));
}
}
else
{
db.RemoveRemoteVolume(newvol.RemoteFilename, transaction);
if (newvolindex != null)
{
db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction);
newvolindex.FinishVolume(null, 0);
}
}
}
}
else
{
newvolindex?.Dispose();
newvol.Dispose();
}
deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
m_result.DeletedFileCount = deletedVolumes.Count;
m_result.DownloadedFileCount = downloadedVolumes.Count;
m_result.UploadedFileCount = uploadedVolumes.Count;
m_result.DeletedFileSize = deletedSize;
m_result.DownloadedFileSize = downloadSize;
m_result.UploadedFileSize = uploadSize;
m_result.Dryrun = m_options.Dryrun;
if (m_result.Dryrun)
{
if (downloadedVolumes.Count == 0)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}",
m_result.DownloadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize),
m_result.DeletedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
m_result.DeletedFileCount - m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
}
}
else
{
if (m_result.DownloadedFileCount == 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}",
m_result.DownloadedFileCount,
Library.Utility.Utility.FormatSizeString(downloadSize),
m_result.DeletedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize),
m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
m_result.DeletedFileCount - m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
}
}
backend.WaitForComplete(db, transaction);
}
m_result.EndTime = DateTime.UtcNow;
return((m_result.DeletedFileCount + m_result.UploadedFileCount) > 0);
}
else
{
m_result.EndTime = DateTime.UtcNow;
return(false);
}
}
public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(
new
{
Input = Channels.SpillPickup.ForRead,
Output = Channels.BackendRequest.ForWrite,
},
async self =>
{
var lst = new List <VolumeUploadRequest>();
while (!await self.Input.IsRetiredAsync)
{
try
{
lst.Add((VolumeUploadRequest)await self.Input.ReadAsync());
}
catch (Exception ex)
{
if (ex.IsRetiredException())
{
break;
}
throw;
}
}
while (lst.Count > 1)
{
// We ignore the stop signal, but not the pause and terminate
await taskreader.ProgressAsync;
VolumeUploadRequest target = null;
var source = lst[0];
// Finalize the current work
source.BlockVolume.Close();
// Remove it from the list of active operations
lst.RemoveAt(0);
var buffer = new byte[options.Blocksize];
using (var rd = new BlockVolumeReader(options.CompressionModule, source.BlockVolume.LocalFilename, options))
{
foreach (var file in rd.Blocks)
{
// Grab a target
if (target == null)
{
if (lst.Count == 0)
{
// No more targets, make one
target = new VolumeUploadRequest(new BlockVolumeWriter(options), source.IndexVolume == null ? null : new TemporaryIndexVolume(options));
target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
}
else
{
// Grab the next target
target = lst[0];
lst.RemoveAt(0);
}
// We copy all the blocklisthashes, which may create duplicates
// but otherwise we need to query all hashes to see if they are blocklisthashes
if (source.IndexVolume != null)
{
source.IndexVolume.CopyTo(target.IndexVolume, true);
}
}
var len = rd.ReadBlock(file.Key, buffer);
target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default);
await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID);
if (target.IndexVolume != null)
{
target.IndexVolume.AddBlock(file.Key, len);
}
if (target.BlockVolume.Filesize > options.VolumeSize - options.Blocksize)
{
target.BlockVolume.Close();
await self.Output.WriteAsync(target);
target = null;
}
}
}
// Make sure they are out of the database
System.IO.File.Delete(source.BlockVolume.LocalFilename);
await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename);
// Re-inject the target if it has content
if (target != null)
{
lst.Insert(lst.Count == 0 ? 0 : 1, target);
}
}
foreach (var n in lst)
{
// We ignore the stop signal, but not the pause and terminate
await taskreader.ProgressAsync;
n.BlockVolume.Close();
await self.Output.WriteAsync(n);
}
}));
}
private static void PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer)
{
var blocksize = options.Blocksize;
var updateCounter = 0L;
using (var blockmarker = database.CreateBlockMarker())
{
foreach (var restorelist in database.GetFilesWithMissingBlocks(blocks))
{
var targetpath = restorelist.Path;
result.AddVerboseMessage("Patching file with remote data: {0}", targetpath);
if (options.Dryrun)
{
result.AddDryrunMessage(string.Format("Would patch file with remote data: {0}", targetpath));
}
else
{
try
{
var folderpath = m_systemIO.PathGetDirectoryName(targetpath);
if (!options.Dryrun && !m_systemIO.DirectoryExists(folderpath))
{
result.AddWarning(string.Format("Creating missing folder {0} for file {1}", folderpath, targetpath), null);
m_systemIO.DirectoryCreate(folderpath);
}
// TODO: Much faster if we iterate the volume and checks what blocks are used,
// because the compressors usually like sequential reading
using (var file = m_systemIO.FileOpenReadWrite(targetpath))
foreach (var targetblock in restorelist.Blocks)
{
file.Position = targetblock.Offset;
var size = blocks.ReadBlock(targetblock.Key, blockbuffer);
if (targetblock.Size == size)
{
file.Write(blockbuffer, 0, size);
blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size);
}
}
if (updateCounter++ % 20 == 0)
{
blockmarker.UpdateProcessed(result.OperationProgressUpdater);
}
}
catch (Exception ex)
{
result.AddWarning(string.Format("Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message), ex);
}
try
{
ApplyMetadata(targetpath, database);
}
catch (Exception ex)
{
result.AddWarning(string.Format("Failed to apply metadata to file: \"{0}\", message: {1}", targetpath, ex.Message), ex);
}
}
}
blockmarker.UpdateProcessed(result.OperationProgressUpdater);
blockmarker.Commit(result);
}
}
