/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
/// <param name="log">The log instance to use</param>
public static void VerifyLocalList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
{
var locallist = database.GetRemoteVolumes();
foreach(var i in locallist)
{
switch (i.State)
{
case RemoteVolumeState.Uploaded:
case RemoteVolumeState.Verified:
case RemoteVolumeState.Deleted:
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
case RemoteVolumeState.Uploading:
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
try
{
backend.Delete(i.Name, i.Size, true);
}
catch (Exception ex)
{
log.AddWarning(string.Format("Failed to erase file {0}, treating as deleted: {1}", i.Name, ex.Message), ex);
}
break;
default:
log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
break;
}
backend.FlushDbMessages();
}
}
private IEnumerable <KeyValuePair <string, long> > PerformDelete(BackendManager backend, IEnumerable <IRemoteVolume> list)
{
foreach (var f in list)
{
if (!m_options.Dryrun)
{
backend.Delete(f.Name, f.Size);
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size));
}
yield return(new KeyValuePair <string, long>(f.Name, f.Size));
}
}
private IEnumerable <KeyValuePair <string, long> > DoDelete(LocalDeleteDatabase db, BackendManager backend, List <IRemoteVolume> deleteableVolumes, System.Data.IDbTransaction transaction)
{
foreach (var f in db.GetDeletableVolumes(deleteableVolumes, transaction))
{
if (!m_options.Dryrun)
{
backend.Delete(f.Name, f.Size);
}
else
{
m_result.AddDryrunMessage(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)));
}
yield return(new KeyValuePair <string, long>(f.Name, f.Size));
}
deleteableVolumes.Clear();
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
/// <param name="log">The log instance to use</param>
public static void VerifyLocalList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
{
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
switch (i.State)
{
case RemoteVolumeState.Uploaded:
case RemoteVolumeState.Verified:
case RemoteVolumeState.Deleted:
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
case RemoteVolumeState.Uploading:
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
try
{
backend.Delete(i.Name, i.Size, true);
}
catch (Exception ex)
{
log.AddWarning(string.Format("Failed to erase file {0}, treating as deleted: {1}", i.Name, ex.Message), ex);
}
break;
default:
log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
break;
}
backend.FlushDbMessages();
}
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="database">The database to compare with</param>
public static void VerifyLocalList(BackendManager backend, LocalDatabase database)
{
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
switch (i.State)
{
case RemoteVolumeState.Uploaded:
case RemoteVolumeState.Verified:
case RemoteVolumeState.Deleted:
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
case RemoteVolumeState.Uploading:
Logging.Log.WriteInformationMessage(LOGTAG, "RemovingStaleFile", "Removing remote file listed as {0}: {1}", i.State, i.Name);
try
{
backend.Delete(i.Name, i.Size, true);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(LOGTAG, "DeleteFileFailed", ex, "Failed to erase file {0}, treating as deleted: {1}", i.Name, ex.Message);
}
break;
default:
Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "Unknown state for remote file listed as {0}: {1}", i.State, i.Name);
break;
}
backend.FlushDbMessages();
}
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
/// <param name="protectedfile">A filename that should be excempted for deletion</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, string protectedfile)
{
var rawlist = backend.List();
var lookup = new Dictionary <string, Volumes.IParsedVolume>();
protectedfile = protectedfile ?? string.Empty;
var remotelist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix == options.Prefix
select p).ToList();
var otherlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix != options.Prefix
select p).ToList();
var unknownlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p == null
select n).ToList();
var filesets = (from n in remotelist
where n.FileType == RemoteVolumeType.Files orderby n.Time descending
select n).ToList();
log.KnownFileCount = remotelist.Count;
log.KnownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum();
log.UnknownFileCount = unknownlist.Count;
log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum();
log.BackupListCount = filesets.Count;
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
// TODO: We should query through the backendmanager
using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions))
if (bk is Library.Interface.IQuotaEnabledBackend)
{
Library.Interface.IQuotaInfo quota = ((Library.Interface.IQuotaEnabledBackend)bk).Quota;
if (quota != null)
{
log.TotalQuotaSpace = quota.TotalQuotaSpace;
log.FreeQuotaSpace = quota.FreeQuotaSpace;
}
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach (var s in remotelist)
{
lookup[s.File.Name] = s;
}
var missing = new List <RemoteVolumeEntry>();
var missingHash = new List <Tuple <long, RemoteVolumeEntry> >();
var cleanupRemovedRemoteVolumes = new HashSet <string>();
foreach (var e in database.DuplicateRemoteVolumes())
{
if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
{
database.UnlinkRemoteVolume(e.Key, e.Value);
}
else
{
throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()));
}
}
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
Volumes.IParsedVolume r;
var remoteFound = lookup.TryGetValue(i.Name, out r);
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
lookup.Remove(i.Name);
switch (i.State)
{
case RemoteVolumeState.Deleted:
if (remoteFound)
{
log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name));
}
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
if (remoteFound)
{
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
else
{
if (i.deleteGracePeriod > DateTime.UtcNow)
{
log.AddMessage(string.Format("keeping delete request for {0} until {1}", i.Name, i.deleteGracePeriod.ToLocalTime()));
}
else
{
if (string.Equals(i.Name, protectedfile) && i.State == RemoteVolumeState.Temporary)
{
log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
}
else
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
cleanupRemovedRemoteVolumes.Add(i.Name);
}
}
}
break;
case RemoteVolumeState.Uploading:
if (remoteFound && correctSize && r.File.Size >= 0)
{
log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded));
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
}
else if (!remoteFound)
{
if (string.Equals(i.Name, protectedfile))
{
log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null);
}
else
{
log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));
cleanupRemovedRemoteVolumes.Add(i.Name);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);
}
}
else
{
if (string.Equals(i.Name, protectedfile))
{
log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
}
else
{
log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
}
break;
case RemoteVolumeState.Uploaded:
if (!remoteFound)
{
missing.Add(i);
}
else if (correctSize)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
else
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
case RemoteVolumeState.Verified:
if (!remoteFound)
{
missing.Add(i);
}
else if (!correctSize)
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
default:
log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
break;
}
backend.FlushDbMessages();
}
// cleanup deleted volumes in DB en block
database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null);
foreach (var i in missingHash)
{
log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null);
}
return(new RemoteAnalysisResult()
{
ParsedVolumes = remotelist,
OtherVolumes = otherlist,
ExtraVolumes = lookup.Values,
MissingVolumes = missing,
VerificationRequiredVolumes = missingHash.Select(x => x.Item2)
});
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
{
var rawlist = backend.List();
var lookup = new Dictionary <string, Volumes.IParsedVolume>();
var remotelist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix == options.Prefix
select p).ToList();
var otherlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix != options.Prefix
select p).ToList();
var unknownlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p == null
select n).ToList();
var filesets = (from n in remotelist
where n.FileType == RemoteVolumeType.Files orderby n.Time descending
select n).ToList();
log.KnownFileCount = remotelist.Count();
log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum();
log.UnknownFileCount = unknownlist.Count();
log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum();
log.BackupListCount = filesets.Count;
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
if (backend is Library.Interface.IQuotaEnabledBackend)
{
log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace;
log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace;
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach (var s in remotelist)
{
lookup[s.File.Name] = s;
}
var missing = new List <RemoteVolumeEntry>();
var missingHash = new List <Tuple <long, RemoteVolumeEntry> >();
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
Volumes.IParsedVolume r;
var remoteFound = lookup.TryGetValue(i.Name, out r);
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
lookup.Remove(i.Name);
switch (i.State)
{
case RemoteVolumeState.Deleted:
if (remoteFound)
{
log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name));
}
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
if (remoteFound)
{
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
else
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
}
break;
case RemoteVolumeState.Uploading:
if (remoteFound && correctSize && r.File.Size >= 0)
{
log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded));
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
}
else if (!remoteFound)
{
log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
database.RegisterRemoteVolume(i.Name, i.Type, RemoteVolumeState.Deleting, null);
}
else
{
log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
break;
case RemoteVolumeState.Uploaded:
if (!remoteFound)
{
missing.Add(i);
}
else if (correctSize)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
else
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
case RemoteVolumeState.Verified:
if (!remoteFound)
{
missing.Add(i);
}
else if (!correctSize)
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
default:
log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
break;
}
backend.FlushDbMessages();
}
foreach (var i in missingHash)
{
log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null);
}
return(new RemoteAnalysisResult()
{
ParsedVolumes = remotelist,
OtherVolumes = otherlist,
ExtraVolumes = lookup.Values,
MissingVolumes = missing,
VerificationRequiredVolumes = missingHash.Select(x => x.Item2)
});
}
public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBacked, bool forceCompact)
{
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
if (!hasVerifiedBacked && !m_options.NoBackendverification)
{
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
}
var filesetNumbers = db.FilesetTimes.Zip(Enumerable.Range(0, db.FilesetTimes.Count()), (a, b) => new Tuple <long, DateTime>(b, a.Value));
var toDelete = m_options.GetFilesetsToDelete(db.FilesetTimes.Select(x => x.Value).ToArray());
if (toDelete != null && toDelete.Length > 0)
{
m_result.AddMessage(string.Format("Deleting {0} remote fileset(s) ...", toDelete.Length));
}
var lst = db.DropFilesetsFromTable(toDelete, transaction).ToArray();
foreach (var f in lst)
{
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction);
}
if (!m_options.Dryrun)
{
transaction.Commit();
transaction = db.BeginTransaction();
}
foreach (var f in lst)
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, transaction);
return;
}
if (!m_options.Dryrun)
{
backend.Delete(f.Key, f.Value);
}
else
{
m_result.AddDryrunMessage(string.Format("Would delete remote fileset: {0}", f.Key));
}
}
backend.WaitForComplete(db, transaction);
var count = lst.Length;
if (!m_options.Dryrun)
{
if (count == 0)
{
m_result.AddMessage("No remote filesets were deleted");
}
else
{
m_result.AddMessage(string.Format("Deleted {0} remote fileset(s)", count));
}
}
else
{
if (count == 0)
{
m_result.AddDryrunMessage("No remote filesets would be deleted");
}
else
{
m_result.AddDryrunMessage(string.Format("{0} remote fileset(s) would be deleted", count));
}
if (count > 0 && m_options.Dryrun)
{
m_result.AddDryrunMessage("Remove --dry-run to actually delete files");
}
}
if (!m_options.NoAutoCompact && (forceCompact || (toDelete != null && toDelete.Length > 0)))
{
m_result.CompactResults = new CompactResults(m_result);
new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction);
}
m_result.SetResults(
from n in filesetNumbers
where toDelete.Contains(n.Item2)
select n,
m_options.Dryrun);
}
}
public void RunRepairRemote()
{
if (!System.IO.File.Exists(m_options.Dbpath))
throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
m_result.OperationProgressUpdater.UpdateProgress(0);
using(var db = new LocalRepairDatabase(m_options.Dbpath))
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
m_result.SetDatabase(db);
Utility.VerifyParameters(db, m_options);
var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter);
var buffer = new byte[m_options.Blocksize];
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
var hashsize = blockhasher.HashSize / 8;
if (blockhasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
if (!blockhasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
var progress = 0;
var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count();
if (tp.ExtraVolumes.Count() > 0 || tp.MissingVolumes.Count() > 0 || tp.VerificationRequiredVolumes.Count() > 0)
{
if (m_options.Dryrun)
{
if (!tp.BackupPrefixes.Contains(m_options.Prefix) && tp.ParsedVolumes.Count() > 0)
{
if (tp.BackupPrefixes.Length == 1)
throw new Exception(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup-prefix?", m_options.Prefix, tp.BackupPrefixes[0]));
else
throw new Exception(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup-prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)));
}
else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Count() > 0)
{
throw new Exception(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()));
}
}
if (tp.VerificationRequiredVolumes.Any())
{
using(var testdb = new LocalTestDatabase(db))
{
foreach(var n in tp.VerificationRequiredVolumes)
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
long size;
string hash;
KeyValuePair<string, IEnumerable<KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>>> res;
using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, m_result, 1);
if (res.Value.Any())
throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First()));
if (!m_options.Dryrun)
{
m_result.AddMessage(string.Format("Sucessfully captured hash for {0}, updating database", n.Name));
db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash);
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
throw;
}
}
}
// TODO: It is actually possible to use the extra files if we parse them
foreach(var n in tp.ExtraVolumes)
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (!m_options.Dryrun)
{
db.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Deleting);
backend.Delete(n.File.Name, n.File.Size);
}
else
m_result.AddDryrunMessage(string.Format("would delete file {0}", n.File.Name));
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
throw;
}
foreach(var n in tp.MissingVolumes)
{
IDisposable newEntry = null;
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (n.Type == RemoteVolumeType.Files)
{
var filesetId = db.GetFilesetIdFromRemotename(n.Name);
var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
newEntry = w;
w.SetRemoteFilename(n.Name);
db.WriteFileset(w, null, filesetId);
w.Close();
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Index)
{
var w = new IndexVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
{
w.StartVolume(blockvolume.Name);
var volumeid = db.GetRemoteVolumeID(blockvolume.Name);
foreach(var b in db.GetBlocks(volumeid))
w.AddBlock(b.Hash, b.Size);
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
foreach(var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize))
w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
}
w.Close();
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Blocks)
{
var w = new BlockVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
using(var mbl = db.CreateBlockList(n.Name))
{
//First we grab all known blocks from local files
foreach(var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize))
{
var hash = block.Hash;
var size = (int)block.Size;
foreach(var source in block.Sources)
{
var file = source.File;
var offset = source.Offset;
try
{
if (System.IO.File.Exists(file))
using(var f = System.IO.File.OpenRead(file))
{
f.Position = offset;
if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size))
{
var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size));
if (newhash == hash)
{
if (mbl.SetBlockRestored(hash, size))
w.AddBlock(hash, buffer, 0, size, Duplicati.Library.Interface.CompressionHint.Default);
break;
}
}
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to access file: {0}", file), ex);
}
}
}
//Then we grab all remote volumes that have the missing blocks
foreach(var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
{
try
{
using(var tmpfile = vol.TempFile)
using(var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options))
foreach(var b in f.Blocks)
if (mbl.SetBlockRestored(b.Key, b.Value))
if (f.ReadBlock(b.Key, buffer) == b.Value)
w.AddBlock(b.Key, buffer, 0, (int)b.Value, Duplicati.Library.Interface.CompressionHint.Default);
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to access remote file: {0}", vol.Name), ex);
}
}
// If we managed to recover all blocks, NICE!
var missingBlocks = mbl.GetMissingBlocks().Count();
if (missingBlocks > 0)
{
//TODO: How do we handle this situation?
m_result.AddMessage(string.Format("Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name));
foreach(var f in mbl.GetFilesetsUsingMissingBlocks())
m_result.AddMessage(f.Name);
if (!m_options.Dryrun)
{
m_result.AddMessage("This may be fixed by deleting the filesets and running repair again");
throw new Exception(string.Format("Repair not possible, missing {0} blocks!!!", missingBlocks));
}
}
else
{
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
}
}
}
catch (Exception ex)
{
if (newEntry != null)
try { newEntry.Dispose(); }
catch { }
finally { newEntry = null; }
m_result.AddError(string.Format("Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
throw;
}
}
}
else
{
m_result.AddMessage("Destination and database are synchronized, not making any changes");
}
m_result.OperationProgressUpdater.UpdateProgress(1);
backend.WaitForComplete(db, null);
db.WriteResults();
}
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
/// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable <string> protectedFiles)
{
var rawlist = backend.List();
var lookup = new Dictionary <string, Volumes.IParsedVolume>();
protectedFiles = protectedFiles ?? Enumerable.Empty <string>();
var remotelist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix == options.Prefix
select p).ToList();
var otherlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix != options.Prefix
select p).ToList();
var unknownlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p == null
select n).ToList();
var filesets = (from n in remotelist
where n.FileType == RemoteVolumeType.Files orderby n.Time descending
select n).ToList();
log.KnownFileCount = remotelist.Count;
long knownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum();
log.KnownFileSize = knownFileSize;
log.UnknownFileCount = unknownlist.Count;
log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum();
log.BackupListCount = database.FilesetTimes.Count();
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
// TODO: We should query through the backendmanager
using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions))
if (bk is IQuotaEnabledBackend enabledBackend)
{
Library.Interface.IQuotaInfo quota = enabledBackend.Quota;
if (quota != null)
{
log.TotalQuotaSpace = quota.TotalQuotaSpace;
log.FreeQuotaSpace = quota.FreeQuotaSpace;
// Check to see if there should be a warning or error about the quota
// Since this processor may be called multiple times during a backup
// (both at the start and end, for example), the log keeps track of
// whether a quota error or warning has been sent already.
// Note that an error can still be sent later even if a warning was sent earlier.
if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0)
{
log.ReportedQuotaError = true;
Logging.Log.WriteErrorMessage(LOGTAG, "BackendQuotaExceeded", null, "Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
}
else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info
{
// Warnings are sent if the available free space is less than the given percentage of the total backup size.
double warningThreshold = options.QuotaWarningThreshold / (double)100;
if (quota.FreeQuotaSpace < warningThreshold * knownFileSize)
{
log.ReportedQuotaWarning = true;
Logging.Log.WriteWarningMessage(LOGTAG, "BackendQuotaNear", null, "Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
}
}
}
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach (var s in remotelist)
{
lookup[s.File.Name] = s;
}
var missing = new List <RemoteVolumeEntry>();
var missingHash = new List <Tuple <long, RemoteVolumeEntry> >();
var cleanupRemovedRemoteVolumes = new HashSet <string>();
foreach (var e in database.DuplicateRemoteVolumes())
{
if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
{
database.UnlinkRemoteVolume(e.Key, e.Value);
}
else
{
throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()));
}
}
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
Volumes.IParsedVolume r;
var remoteFound = lookup.TryGetValue(i.Name, out r);
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
lookup.Remove(i.Name);
switch (i.State)
{
case RemoteVolumeState.Deleted:
if (remoteFound)
{
Logging.Log.WriteInformationMessage(LOGTAG, "IgnoreRemoteDeletedFile", "ignoring remote file listed as {0}: {1}", i.State, i.Name);
}
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
if (remoteFound)
{
Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "removing remote file listed as {0}: {1}", i.State, i.Name);
backend.Delete(i.Name, i.Size, true);
}
else
{
if (i.DeleteGracePeriod > DateTime.UtcNow)
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepDeleteRequest", "keeping delete request for {0} until {1}", i.Name, i.DeleteGracePeriod.ToLocalTime());
}
else
{
if (i.State == RemoteVolumeState.Temporary && protectedFiles.Any(pf => pf == i.Name))
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "RemoteUnwantedMissingFile", "removing file listed as {0}: {1}", i.State, i.Name);
cleanupRemovedRemoteVolumes.Add(i.Name);
}
}
}
break;
case RemoteVolumeState.Uploading:
if (remoteFound && correctSize && r.File.Size >= 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
}
else if (!remoteFound)
{
if (protectedFiles.Any(pf => pf == i.Name))
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null);
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name);
cleanupRemovedRemoteVolumes.Add(i.Name);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);
}
}
else
{
if (protectedFiles.Any(pf => pf == i.Name))
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "Remove incomplete file", "removing incomplete remote file listed as {0}: {1}", i.State, i.Name);
backend.Delete(i.Name, i.Size, true);
}
}
break;
case RemoteVolumeState.Uploaded:
if (!remoteFound)
{
missing.Add(i);
}
else if (correctSize)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
else
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
case RemoteVolumeState.Verified:
if (!remoteFound)
{
missing.Add(i);
}
else if (!correctSize)
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
default:
Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "unknown state for remote file listed as {0}: {1}", i.State, i.Name);
break;
}
backend.FlushDbMessages();
}
// cleanup deleted volumes in DB en block
database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null);
foreach (var i in missingHash)
{
Logging.Log.WriteWarningMessage(LOGTAG, "MissingRemoteHash", null, "remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash);
}
return(new RemoteAnalysisResult()
{
ParsedVolumes = remotelist,
OtherVolumes = otherlist,
ExtraVolumes = lookup.Values,
MissingVolumes = missing,
VerificationRequiredVolumes = missingHash.Select(x => x.Item2)
});
}
private void DoRun(Database.LocalPurgeDatabase db, Library.Utility.IFilter filter, Action <System.Data.IDbCommand, long, string> filtercommand, float pgoffset, float pgspan)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin);
Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation");
var doCompactStep = !m_options.NoAutoCompact && filtercommand == null;
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
if (db.PartiallyRecreated)
{
throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase");
}
if (db.RepairInProgress && filtercommand == null)
{
throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase");
}
var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray();
if (versions.Length <= 0)
{
throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion");
}
var orphans = db.CountOrphanFiles(null);
if (orphans != 0)
{
throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans");
}
Utility.UpdateOptionsFromDb(db, m_options);
Utility.VerifyParameters(db, m_options);
if (filtercommand == null)
{
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null);
if (m_options.NoBackendverification)
{
FilelistProcessor.VerifyLocalList(backend, db);
}
else
{
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, null);
}
}
var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray();
var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan;
var currentprogress = pgoffset;
var progress = 0;
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Process);
m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
// Reverse makes sure we re-write the old versions first
foreach (var versionid in versions.Reverse())
{
progress++;
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length);
using (var tr = db.BeginTransaction())
{
var ix = -1;
for (var i = 0; i < filesets.Length; i++)
{
if (filesets[i].Key == versionid)
{
ix = i;
break;
}
}
if (ix < 0)
{
throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
}
var secs = 0;
while (secs < 60)
{
secs++;
var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs));
if (db.GetRemoteVolumeID(tfn, tr) < 0)
{
break;
}
}
var tsOriginal = filesets[ix].Value;
var ts = tsOriginal.AddSeconds(secs);
var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr);
if (secs >= 60)
{
throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts));
}
if (ix != 0 && filesets[ix - 1].Value <= ts)
{
throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
}
using (var tempset = db.CreateTemporaryFileset(versionid, tr))
{
if (filtercommand == null)
{
tempset.ApplyFilter(filter);
}
else
{
tempset.ApplyFilter(filtercommand);
}
if (tempset.RemovedFileCount == 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename);
currentprogress += versionprogress;
tr.Rollback();
continue;
}
else
{
using (var tf = new Library.Utility.TempFile())
using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts))
{
var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal);
var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup);
vol.VolumeID = newids.Item1;
vol.CreateFilesetFile(isOriginalFilesetFullBackup);
Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize));
db.WriteFileset(vol, newids.Item2, tr);
m_result.RemovedFileSize += tempset.RemovedFileSize;
m_result.RemovedFileCount += tempset.RemovedFileCount;
m_result.RewrittenFileLists++;
currentprogress += (versionprogress / 2);
m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
if (m_options.Dryrun || m_options.FullResult)
{
foreach (var fe in tempset.ListAllDeletedFiles())
{
var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg);
Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg);
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg);
}
}
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage");
}
Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage");
}
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount);
tr.Rollback();
}
else
{
var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray();
foreach (var f in lst)
{
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr);
}
tr.Commit();
backend.Put(vol, synchronous: true);
backend.Delete(prevfilename, -1, true);
backend.FlushDbMessages();
}
}
}
}
}
currentprogress += (versionprogress / 2);
m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
}
if (doCompactStep)
{
if (m_result.RewrittenFileLists == 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "SkippingCompacting", "Skipping compacting as no new volumes were written");
}
else
{
m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan));
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact);
m_result.CompactResults = new CompactResults(m_result);
using (var cdb = new Database.LocalDeleteDatabase(db))
{
var tr = cdb.BeginTransaction();
try
{
new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, ref tr, backend);
}
catch
{
try { tr.Rollback(); }
catch { }
}
finally
{
try { tr.Commit(); }
catch { }
}
}
}
m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan);
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete);
}
backend.WaitForComplete(db, null);
}
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
{
var rawlist = backend.List();
var lookup = new Dictionary <string, Volumes.IParsedVolume>();
var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null select p).ToList();
var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList();
var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList();
log.KnownFileCount = remotelist.Count();
log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum();
log.UnknownFileCount = unknownlist.Count();
log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum();
log.BackupListCount = filesets.Count;
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
if (backend is Library.Interface.IQuotaEnabledBackend)
{
log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace;
log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace;
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach (var s in remotelist)
{
if (s.Prefix == options.Prefix)
{
lookup[s.File.Name] = s;
}
}
var missing = new List <RemoteVolumeEntry>();
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
//Ignore those that are deleted
if (i.State == RemoteVolumeState.Deleted)
{
continue;
}
if (i.State == RemoteVolumeState.Temporary)
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
}
else if (i.State == RemoteVolumeState.Deleting && lookup.ContainsKey(i.Name))
{
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
lookup.Remove(i.Name);
}
else
{
Volumes.IParsedVolume r;
if (!lookup.TryGetValue(i.Name, out r))
{
if (i.State == RemoteVolumeState.Uploading || i.State == RemoteVolumeState.Deleting || (r != null && r.File.Size != i.Size && r.File.Size >= 0 && i.Size >= 0))
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
}
else
{
missing.Add(i);
}
}
else if (i.State != RemoteVolumeState.Verified)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
lookup.Remove(i.Name);
}
}
return(new RemoteAnalysisResult()
{
ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing
});
}
public void RunRepairRemote()
{
if (!System.IO.File.Exists(m_options.Dbpath))
{
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "RepairDatabaseFileDoesNotExist");
}
m_result.OperationProgressUpdater.UpdateProgress(0);
using (var db = new LocalRepairDatabase(m_options.Dbpath))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
m_result.SetDatabase(db);
Utility.UpdateOptionsFromDb(db, m_options);
Utility.VerifyParameters(db, m_options);
if (db.PartiallyRecreated)
{
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");
}
if (db.RepairInProgress)
{
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState");
}
var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter, null);
var buffer = new byte[m_options.Blocksize];
var blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
var hashsize = blockhasher.HashSize / 8;
if (blockhasher == null)
{
throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported");
}
if (!blockhasher.CanReuseTransform)
{
throw new UserInformationException(Strings.Common.InvalidCryptoSystem(m_options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported");
}
var progress = 0;
var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count();
if (m_options.Dryrun)
{
if (tp.ParsedVolumes.Count() == 0 && tp.OtherVolumes.Count() > 0)
{
if (tp.BackupPrefixes.Length == 1)
{
throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]), "RemoteFolderEmptyWithPrefix");
}
else
{
throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)), "RemoteFolderEmptyWithPrefix");
}
}
else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Count() > 0)
{
throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()), "NoRemoteFilesMissing");
}
}
if (tp.ExtraVolumes.Count() > 0 || tp.MissingVolumes.Count() > 0 || tp.VerificationRequiredVolumes.Count() > 0)
{
if (tp.VerificationRequiredVolumes.Any())
{
using (var testdb = new LocalTestDatabase(db))
{
foreach (var n in tp.VerificationRequiredVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
long size;
string hash;
KeyValuePair <string, IEnumerable <KeyValuePair <Duplicati.Library.Interface.TestEntryStatus, string> > > res;
using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1);
if (res.Value.Any())
{
throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First()));
}
if (!m_options.Dryrun)
{
Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name);
db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash);
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
}
// TODO: It is actually possible to use the extra files if we parse them
foreach (var n in tp.ExtraVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
// If this is a new index file, we can accept it if it matches our local data
// This makes it possible to augment the remote store with new index data
if (n.FileType == RemoteVolumeType.Index && m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
try
{
string hash;
long size;
using (var tf = backend.GetWithInfo(n.File.Name, out size, out hash))
using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize))
{
foreach (var rv in ifr.Volumes)
{
var entry = db.GetRemoteVolume(rv.Filename);
if (entry.ID < 0)
{
throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename));
}
if (!new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State))
{
throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, entry.State));
}
if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State))
{
throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length));
}
db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks);
}
var blocksize = m_options.Blocksize;
foreach (var ixb in ifr.BlockLists)
{
db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize);
}
var selfid = db.GetRemoteVolumeID(n.File.Name);
foreach (var rv in ifr.Volumes)
{
db.AddIndexBlockLink(selfid, db.GetRemoteVolumeID(rv.Filename), null);
}
}
// All checks fine, we accept the new index file
Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name);
db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, size, RemoteVolumeState.Uploading);
db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash);
continue;
}
catch (Exception rex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "FailedNewIndexFile", rex, "Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message);
}
}
if (!m_options.Dryrun)
{
db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting);
backend.Delete(n.File.Name, n.File.Size);
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name);
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "FailedExtraFileCleanup", ex, "Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
if (!m_options.RebuildMissingDblockFiles)
{
var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks).ToArray();
if (missingDblocks.Length > 0)
{
throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles");
}
}
foreach (var n in tp.MissingVolumes)
{
IDisposable newEntry = null;
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (n.Type == RemoteVolumeType.Files)
{
var filesetId = db.GetFilesetIdFromRemotename(n.Name);
var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
newEntry = w;
w.SetRemoteFilename(n.Name);
db.WriteFileset(w, filesetId, null);
w.Close();
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Index)
{
var w = new IndexVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
var h = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
{
w.StartVolume(blockvolume.Name);
var volumeid = db.GetRemoteVolumeID(blockvolume.Name);
foreach (var b in db.GetBlocks(volumeid))
{
w.AddBlock(b.Hash, b.Size);
}
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
{
foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize))
{
var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
if (bh != b.Item1)
{
throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
}
w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
}
}
}
w.Close();
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Blocks)
{
var w = new BlockVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
using (var mbl = db.CreateBlockList(n.Name))
{
//First we grab all known blocks from local files
foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize))
{
var hash = block.Hash;
var size = (int)block.Size;
foreach (var source in block.Sources)
{
var file = source.File;
var offset = source.Offset;
try
{
if (System.IO.File.Exists(file))
{
using (var f = System.IO.File.OpenRead(file))
{
f.Position = offset;
if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size))
{
var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size));
if (newhash == hash)
{
if (mbl.SetBlockRestored(hash, size))
{
w.AddBlock(hash, buffer, 0, size, Duplicati.Library.Interface.CompressionHint.Default);
}
break;
}
}
}
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", file);
}
}
}
//Then we grab all remote volumes that have the missing blocks
foreach (var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
{
try
{
using (var tmpfile = vol.TempFile)
using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options))
foreach (var b in f.Blocks)
{
if (mbl.SetBlockRestored(b.Key, b.Value))
{
if (f.ReadBlock(b.Key, buffer) == b.Value)
{
w.AddBlock(b.Key, buffer, 0, (int)b.Value, Duplicati.Library.Interface.CompressionHint.Default);
}
}
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", vol.Name);
}
}
// If we managed to recover all blocks, NICE!
var missingBlocks = mbl.GetMissingBlocks().Count();
if (missingBlocks > 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name);
foreach (var f in mbl.GetFilesetsUsingMissingBlocks())
{
Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name);
}
var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files");
if (!m_options.Dryrun)
{
Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", "This may be fixed by deleting the filesets and running repair again");
throw new UserInformationException(string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks), "RepairIsNotPossible");
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", recoverymsg);
}
}
else
{
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadBlockFile", "would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
}
}
}
catch (Exception ex)
{
if (newEntry != null)
{
try { newEntry.Dispose(); }
catch { }
finally { newEntry = null; }
}
Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "DatabaseIsSynchronized", "Destination and database are synchronized, not making any changes");
}
m_result.OperationProgressUpdater.UpdateProgress(1);
backend.WaitForComplete(db, null);
db.WriteResults();
}
}
private IEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, BackendManager backend, List<IRemoteVolume> deleteableVolumes, System.Data.IDbTransaction transaction)
{
foreach(var f in db.GetDeletableVolumes(deleteableVolumes, transaction))
{
if (!m_options.Dryrun)
backend.Delete(f.Name, f.Size);
else
m_result.AddDryrunMessage(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)));
yield return new KeyValuePair<string, long>(f.Name, f.Size);
}
deleteableVolumes.Clear();
}
public void Run(Library.Utility.IFilter filter)
{
if (!System.IO.File.Exists(m_options.Dbpath))
{
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
}
if (filter != null && !filter.Empty)
{
throw new UserInformationException("Filters are not supported for this operation");
}
List <Database.RemoteVolumeEntry> missing = null;
using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath))
using (var tr = db.BeginTransaction())
{
if (db.PartiallyRecreated)
{
throw new UserInformationException("The command does not work on partially recreated databases");
}
var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, tr, m_options, out missing);
if (sets == null)
{
return;
}
if (sets.Length == 0)
{
if (missing == null)
{
m_result.AddMessage("Found no broken filesets");
}
else if (missing.Count == 0)
{
m_result.AddMessage("Found no broken filesets and no missing remote files");
}
else
{
throw new UserInformationException(string.Format("Found no broken filesets, but {0} missing remote files", sets.Length));
}
}
m_result.AddMessage(string.Format("Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3)));
var pgoffset = 0.0f;
var pgspan = 0.95f / sets.Length;
var filesets = db.FilesetTimes.ToList();
var compare_list = sets.Select(x => new
{
FilesetID = x.Item2,
Timestamp = x.Item1,
RemoveCount = x.Item3,
Version = filesets.FindIndex(y => y.Key == x.Item2),
SetCount = db.GetFilesetFileCount(x.Item2, tr)
}).ToArray();
var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
if (fully_emptied.Length != 0)
{
if (fully_emptied.Length == 1)
{
m_result.AddMessage(string.Format("Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount));
}
else
{
m_result.AddMessage(string.Format("Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))));
}
m_result.DeleteResults = new DeleteResults(m_result);
using (var rmdb = new Database.LocalDeleteDatabase(db))
{
var deltr = rmdb.BeginTransaction();
try
{
var opts = new Options(new Dictionary <string, string>(m_options.RawOptions));
opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
opts.RawOptions.Remove("time");
opts.RawOptions["no-auto-compact"] = "true";
new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
.DoRun(rmdb, ref deltr, true, false);
if (!m_options.Dryrun)
{
using (new Logging.Timer("CommitDelete"))
deltr.Commit();
rmdb.WriteResults();
}
else
{
deltr.Rollback();
}
}
finally
{
if (deltr != null)
{
try { deltr.Rollback(); }
catch { }
}
}
}
pgoffset += (pgspan * fully_emptied.Length);
m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
}
if (to_purge.Length > 0)
{
m_result.PurgeResults = new PurgeFilesResults(m_result);
foreach (var bs in to_purge)
{
m_result.AddMessage(string.Format("Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()));
var opts = new Options(new Dictionary <string, string>(m_options.RawOptions));
using (var pgdb = new Database.LocalPurgeDatabase(db))
{
// Recompute the version number after we deleted the versions before
filesets = pgdb.FilesetTimes.ToList();
var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
if (thisversion < 0)
{
throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString()))));
}
opts.RawOptions["version"] = thisversion.ToString();
opts.RawOptions.Remove("time");
opts.RawOptions["no-auto-compact"] = "true";
new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
{
if (filesetid != bs.FilesetID)
{
throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
}
db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
});
}
pgoffset += pgspan;
m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
}
}
if (m_options.Dryrun)
{
tr.Rollback();
}
else
{
tr.Commit();
}
m_result.OperationProgressUpdater.UpdateProgress(0.95f);
if (missing != null && missing.Count > 0)
{
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
foreach (var f in missing)
{
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)));
}
else
{
backend.Delete(f.Name, f.Size);
}
}
}
}
if (!m_options.Dryrun && db.RepairInProgress)
{
m_result.AddMessage("Database was previously marked as in-progress, checking if it is valid after purging files");
db.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, true);
m_result.AddMessage("Purge completed, and consistency checks completed, marking database as complete");
db.RepairInProgress = false;
}
m_result.OperationProgressUpdater.UpdateProgress(1.0f);
}
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
{
var rawlist = backend.List();
var lookup = new Dictionary<string, Volumes.IParsedVolume>();
var remotelist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null
select p).ToList();
var unknownlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p == null
select n).ToList();
var filesets = (from n in remotelist
where n.FileType == RemoteVolumeType.Files orderby n.Time descending
select n).ToList();
log.KnownFileCount = remotelist.Count();
log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum();
log.UnknownFileCount = unknownlist.Count();
log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum();
log.BackupListCount = filesets.Count;
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
if (backend is Library.Interface.IQuotaEnabledBackend)
{
log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace;
log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace;
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach(var s in remotelist)
if (s.Prefix == options.Prefix)
lookup[s.File.Name] = s;
var missing = new List<RemoteVolumeEntry>();
var missingHash = new List<Tuple<long, RemoteVolumeEntry>>();
var locallist = database.GetRemoteVolumes();
foreach(var i in locallist)
{
Volumes.IParsedVolume r;
var remoteFound = lookup.TryGetValue(i.Name, out r);
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
lookup.Remove(i.Name);
switch (i.State)
{
case RemoteVolumeState.Deleted:
if (remoteFound)
log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name));
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
if (remoteFound)
{
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
else
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
}
break;
case RemoteVolumeState.Uploading:
if (remoteFound && correctSize && r.File.Size >= 0)
{
log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded));
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
}
else
{
log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
break;
case RemoteVolumeState.Uploaded:
if (!remoteFound)
missing.Add(i);
else if (correctSize)
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
else
missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
break;
case RemoteVolumeState.Verified:
if (!remoteFound)
missing.Add(i);
else if (!correctSize)
missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
break;
default:
log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
break;
}
}
foreach(var i in missingHash)
log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null);
return new RemoteAnalysisResult() { ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing, VerificationRequiredVolumes = missingHash.Select(x => x.Item2) };
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
{
var rawlist = backend.List();
var lookup = new Dictionary<string, Volumes.IParsedVolume>();
var remotelist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix == options.Prefix
select p).ToList();
var otherlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix != options.Prefix
select p).ToList();
var unknownlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p == null
select n).ToList();
var filesets = (from n in remotelist
where n.FileType == RemoteVolumeType.Files orderby n.Time descending
select n).ToList();
log.KnownFileCount = remotelist.Count();
log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum();
log.UnknownFileCount = unknownlist.Count();
log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum();
log.BackupListCount = filesets.Count;
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
if (backend is Library.Interface.IQuotaEnabledBackend)
{
log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace;
log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace;
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach(var s in remotelist)
lookup[s.File.Name] = s;
var missing = new List<RemoteVolumeEntry>();
var missingHash = new List<Tuple<long, RemoteVolumeEntry>>();
var cleanupRemovedRemoteVolumes = new HashSet<string>();
foreach(var e in database.DuplicateRemoteVolumes())
{
if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
database.UnlinkRemoteVolume(e.Key, e.Value);
else
throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()));
}
var locallist = database.GetRemoteVolumes();
foreach(var i in locallist)
{
Volumes.IParsedVolume r;
var remoteFound = lookup.TryGetValue(i.Name, out r);
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
lookup.Remove(i.Name);
switch (i.State)
{
case RemoteVolumeState.Deleted:
if (remoteFound)
log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name));
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
if (remoteFound)
{
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
else
{
if (i.deleteGracePeriod > DateTime.UtcNow)
{
log.AddMessage(string.Format("keeping delete request for {0} until {1}", i.Name, i.deleteGracePeriod.ToLocalTime()));
}
else
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
cleanupRemovedRemoteVolumes.Add(i.Name);
}
}
break;
case RemoteVolumeState.Uploading:
if (remoteFound && correctSize && r.File.Size >= 0)
{
log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded));
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
}
else if (!remoteFound)
{
log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));
cleanupRemovedRemoteVolumes.Add(i.Name);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);
}
else
{
log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
break;
case RemoteVolumeState.Uploaded:
if (!remoteFound)
missing.Add(i);
else if (correctSize)
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
else
missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
break;
case RemoteVolumeState.Verified:
if (!remoteFound)
missing.Add(i);
else if (!correctSize)
missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
break;
default:
log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
break;
}
backend.FlushDbMessages();
}
// cleanup deleted volumes in DB en block
database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null);
foreach(var i in missingHash)
log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null);
return new RemoteAnalysisResult() {
ParsedVolumes = remotelist,
OtherVolumes = otherlist,
ExtraVolumes = lookup.Values,
MissingVolumes = missing,
VerificationRequiredVolumes = missingHash.Select(x => x.Item2)
};
}
private IEnumerable<KeyValuePair<string, long>> PerformDelete(BackendManager backend, IEnumerable<IRemoteVolume> list)
{
foreach(var f in list)
{
if (!m_options.Dryrun)
backend.Delete(f.Name, f.Size);
else
m_result.AddDryrunMessage(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)));
yield return new KeyValuePair<string, long>(f.Name, f.Size);
}
}
public void DoRun(Database.LocalDeleteDatabase db, System.Data.IDbTransaction transaction, bool hasVerifiedBacked, bool forceCompact)
{
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
if (!hasVerifiedBacked && !m_options.NoBackendverification)
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
var filesetNumbers = db.FilesetTimes.Zip(Enumerable.Range(0, db.FilesetTimes.Count()), (a, b) => new Tuple<long, DateTime>(b, a.Value));
var toDelete = m_options.GetFilesetsToDelete(db.FilesetTimes.Select(x => x.Value).ToArray());
if (toDelete != null && toDelete.Length > 0)
m_result.AddMessage(string.Format("Deleting {0} remote fileset(s) ...", toDelete.Length));
var count = 0L;
foreach(var f in db.DropFilesetsFromTable(toDelete, transaction))
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, transaction);
return;
}
count++;
if (!m_options.Dryrun)
backend.Delete(f.Key, f.Value);
else
m_result.AddDryrunMessage(string.Format("Would delete remote fileset: {0}", f.Key));
}
backend.WaitForComplete(db, transaction);
if (!m_options.Dryrun)
{
if (count == 0)
m_result.AddMessage("No remote filesets were deleted");
else
m_result.AddMessage(string.Format("Deleted {0} remote fileset(s)", count));
}
else
{
if (count == 0)
m_result.AddDryrunMessage("No remote filesets would be deleted");
else
m_result.AddDryrunMessage(string.Format("{0} remote fileset(s) would be deleted", count));
if (count > 0 && m_options.Dryrun)
m_result.AddDryrunMessage("Remove --dry-run to actually delete files");
}
if (!m_options.NoAutoCompact && (forceCompact || (toDelete != null && toDelete.Length > 0)))
{
m_result.CompactResults = new CompactResults(m_result);
new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, transaction);
}
m_result.SetResults(
from n in filesetNumbers
where toDelete.Contains(n.Item2)
select n,
m_options.Dryrun);
}
}
public void RunRepairRemote()
{
if (!System.IO.File.Exists(m_options.Dbpath))
{
throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
}
m_result.OperationProgressUpdater.UpdateProgress(0);
using (var db = new LocalRepairDatabase(m_options.Dbpath))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
m_result.SetDatabase(db);
Utility.VerifyParameters(db, m_options);
var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter);
var buffer = new byte[m_options.Blocksize];
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
var hashsize = blockhasher.HashSize / 8;
if (blockhasher == null)
{
throw new Exception(Strings.Foresthash.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
}
if (!blockhasher.CanReuseTransform)
{
throw new Exception(Strings.Foresthash.InvalidCryptoSystem(m_options.BlockHashAlgorithm));
}
var progress = 0;
var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count();
if (m_options.Dryrun)
{
if (tp.ParsedVolumes.Count() == 0 && tp.OtherVolumes.Count() > 0)
{
if (tp.BackupPrefixes.Length == 1)
{
throw new Exception(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup-prefix?", m_options.Prefix, tp.BackupPrefixes[0]));
}
else
{
throw new Exception(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup-prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)));
}
}
else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Count() > 0)
{
throw new Exception(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()));
}
}
if (tp.ExtraVolumes.Count() > 0 || tp.MissingVolumes.Count() > 0 || tp.VerificationRequiredVolumes.Count() > 0)
{
if (tp.VerificationRequiredVolumes.Any())
{
using (var testdb = new LocalTestDatabase(db))
{
foreach (var n in tp.VerificationRequiredVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
long size;
string hash;
KeyValuePair <string, IEnumerable <KeyValuePair <Duplicati.Library.Interface.TestEntryStatus, string> > > res;
using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, m_result, 1);
if (res.Value.Any())
{
throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First()));
}
if (!m_options.Dryrun)
{
m_result.AddMessage(string.Format("Sucessfully captured hash for {0}, updating database", n.Name));
db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash);
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
}
// TODO: It is actually possible to use the extra files if we parse them
foreach (var n in tp.ExtraVolumes)
{
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (!m_options.Dryrun)
{
db.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Deleting);
backend.Delete(n.File.Name, n.File.Size);
}
else
{
m_result.AddDryrunMessage(string.Format("would delete file {0}", n.File.Name));
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
foreach (var n in tp.MissingVolumes)
{
IDisposable newEntry = null;
try
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, null);
return;
}
progress++;
m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
if (n.Type == RemoteVolumeType.Files)
{
var filesetId = db.GetFilesetIdFromRemotename(n.Name);
var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
newEntry = w;
w.SetRemoteFilename(n.Name);
db.WriteFileset(w, null, filesetId);
w.Close();
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Index)
{
var w = new IndexVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
{
w.StartVolume(blockvolume.Name);
var volumeid = db.GetRemoteVolumeID(blockvolume.Name);
foreach (var b in db.GetBlocks(volumeid))
{
w.AddBlock(b.Hash, b.Size);
}
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
{
foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize))
{
w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
}
}
}
w.Close();
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
else if (n.Type == RemoteVolumeType.Blocks)
{
var w = new BlockVolumeWriter(m_options);
newEntry = w;
w.SetRemoteFilename(n.Name);
using (var mbl = db.CreateBlockList(n.Name))
{
//First we grab all known blocks from local files
foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize))
{
var hash = block.Hash;
var size = (int)block.Size;
foreach (var source in block.Sources)
{
var file = source.File;
var offset = source.Offset;
try
{
if (System.IO.File.Exists(file))
{
using (var f = System.IO.File.OpenRead(file))
{
f.Position = offset;
if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size))
{
var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size));
if (newhash == hash)
{
if (mbl.SetBlockRestored(hash, size))
{
w.AddBlock(hash, buffer, 0, size, Duplicati.Library.Interface.CompressionHint.Default);
}
break;
}
}
}
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to access file: {0}", file), ex);
}
}
}
//Then we grab all remote volumes that have the missing blocks
foreach (var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
{
try
{
using (var tmpfile = vol.TempFile)
using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options))
foreach (var b in f.Blocks)
{
if (mbl.SetBlockRestored(b.Key, b.Value))
{
if (f.ReadBlock(b.Key, buffer) == b.Value)
{
w.AddBlock(b.Key, buffer, 0, (int)b.Value, Duplicati.Library.Interface.CompressionHint.Default);
}
}
}
}
catch (Exception ex)
{
m_result.AddError(string.Format("Failed to access remote file: {0}", vol.Name), ex);
}
}
// If we managed to recover all blocks, NICE!
var missingBlocks = mbl.GetMissingBlocks().Count();
if (missingBlocks > 0)
{
//TODO: How do we handle this situation?
m_result.AddMessage(string.Format("Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name));
foreach (var f in mbl.GetFilesetsUsingMissingBlocks())
{
m_result.AddMessage(f.Name);
}
if (!m_options.Dryrun)
{
m_result.AddMessage("This may be fixed by deleting the filesets and running repair again");
throw new Exception(string.Format("Repair not possible, missing {0} blocks!!!", missingBlocks));
}
}
else
{
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
}
else
{
db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
backend.Put(w);
}
}
}
}
}
catch (Exception ex)
{
if (newEntry != null)
{
try { newEntry.Dispose(); }
catch { }
finally { newEntry = null; }
}
m_result.AddError(string.Format("Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message), ex);
if (ex is System.Threading.ThreadAbortException)
{
throw;
}
}
}
}
else
{
m_result.AddMessage("Destination and database are synchronized, not making any changes");
}
m_result.OperationProgressUpdater.UpdateProgress(1);
backend.WaitForComplete(db, null);
db.WriteResults();
}
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
{
var rawlist = backend.List();
var lookup = new Dictionary<string, Volumes.IParsedVolume>();
var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null select p).ToList();
var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList();
var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList();
log.KnownFileCount = remotelist.Count();
log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum();
log.UnknownFileCount = unknownlist.Count();
log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum();
log.BackupListCount = filesets.Count;
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
if (backend is Library.Interface.IQuotaEnabledBackend)
{
log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace;
log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace;
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach (var s in remotelist)
if (s.Prefix == options.Prefix)
lookup[s.File.Name] = s;
var missing = new List<RemoteVolumeEntry>();
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
//Ignore those that are deleted
if (i.State == RemoteVolumeState.Deleted)
continue;
if (i.State == RemoteVolumeState.Temporary)
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
}
else if (i.State == RemoteVolumeState.Deleting && lookup.ContainsKey(i.Name))
{
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
lookup.Remove(i.Name);
}
else
{
Volumes.IParsedVolume r;
if (!lookup.TryGetValue(i.Name, out r))
{
if (i.State == RemoteVolumeState.Uploading || i.State == RemoteVolumeState.Deleting || (r != null && r.File.Size != i.Size && r.File.Size >= 0 && i.Size >= 0))
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
}
else
missing.Add(i);
}
else if (i.State != RemoteVolumeState.Verified)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
lookup.Remove(i.Name);
}
}
return new RemoteAnalysisResult() { ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing };
}