/// <summary>
/// Handler for computing backend statistics, without relying on a remote folder listing
/// </summary>
private void UpdateStorageStatsFromDatabase()
{
if (m_result.BackendWriter != null)
{
m_result.BackendWriter.KnownFileCount = m_database.GetRemoteVolumes().Count();
m_result.BackendWriter.KnownFileSize = m_database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).Sum();
m_result.BackendWriter.UnknownFileCount = 0;
m_result.BackendWriter.UnknownFileSize = 0;
m_result.BackendWriter.BackupListCount = m_database.FilesetTimes.Count();
m_result.BackendWriter.LastBackupDate = m_database.FilesetTimes.FirstOrDefault().Value.ToLocalTime();
// TODO: If we have a BackendManager, we should query through that
using (var backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions))
{
if (backend is Library.Interface.IQuotaEnabledBackend)
{
Library.Interface.IQuotaInfo quota = ((Library.Interface.IQuotaEnabledBackend)backend).Quota;
if (quota != null)
{
m_result.BackendWriter.TotalQuotaSpace = quota.TotalQuotaSpace;
m_result.BackendWriter.FreeQuotaSpace = quota.FreeQuotaSpace;
}
}
}
}
m_result.BackendWriter.AssignedQuotaSpace = m_options.QuotaSize;
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
/// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable <string> protectedFiles)
{
var rawlist = backend.List();
var lookup = new Dictionary <string, Volumes.IParsedVolume>();
protectedFiles = protectedFiles ?? Enumerable.Empty <string>();
var remotelist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix == options.Prefix
select p).ToList();
var otherlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix != options.Prefix
select p).ToList();
var unknownlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p == null
select n).ToList();
var filesets = (from n in remotelist
where n.FileType == RemoteVolumeType.Files orderby n.Time descending
select n).ToList();
log.KnownFileCount = remotelist.Count;
long knownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum();
log.KnownFileSize = knownFileSize;
log.UnknownFileCount = unknownlist.Count;
log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum();
log.BackupListCount = database.FilesetTimes.Count();
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
// TODO: We should query through the backendmanager
using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions))
if (bk is IQuotaEnabledBackend enabledBackend)
{
Library.Interface.IQuotaInfo quota = enabledBackend.Quota;
if (quota != null)
{
log.TotalQuotaSpace = quota.TotalQuotaSpace;
log.FreeQuotaSpace = quota.FreeQuotaSpace;
// Check to see if there should be a warning or error about the quota
// Since this processor may be called multiple times during a backup
// (both at the start and end, for example), the log keeps track of
// whether a quota error or warning has been sent already.
// Note that an error can still be sent later even if a warning was sent earlier.
if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0)
{
log.ReportedQuotaError = true;
Logging.Log.WriteErrorMessage(LOGTAG, "BackendQuotaExceeded", null, "Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
}
else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info
{
// Warnings are sent if the available free space is less than the given percentage of the total backup size.
double warningThreshold = options.QuotaWarningThreshold / (double)100;
if (quota.FreeQuotaSpace < warningThreshold * knownFileSize)
{
log.ReportedQuotaWarning = true;
Logging.Log.WriteWarningMessage(LOGTAG, "BackendQuotaNear", null, "Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
}
}
}
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach (var s in remotelist)
{
lookup[s.File.Name] = s;
}
var missing = new List <RemoteVolumeEntry>();
var missingHash = new List <Tuple <long, RemoteVolumeEntry> >();
var cleanupRemovedRemoteVolumes = new HashSet <string>();
foreach (var e in database.DuplicateRemoteVolumes())
{
if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
{
database.UnlinkRemoteVolume(e.Key, e.Value);
}
else
{
throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()));
}
}
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
Volumes.IParsedVolume r;
var remoteFound = lookup.TryGetValue(i.Name, out r);
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
lookup.Remove(i.Name);
switch (i.State)
{
case RemoteVolumeState.Deleted:
if (remoteFound)
{
Logging.Log.WriteInformationMessage(LOGTAG, "IgnoreRemoteDeletedFile", "ignoring remote file listed as {0}: {1}", i.State, i.Name);
}
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
if (remoteFound)
{
Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "removing remote file listed as {0}: {1}", i.State, i.Name);
backend.Delete(i.Name, i.Size, true);
}
else
{
if (i.DeleteGracePeriod > DateTime.UtcNow)
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepDeleteRequest", "keeping delete request for {0} until {1}", i.Name, i.DeleteGracePeriod.ToLocalTime());
}
else
{
if (i.State == RemoteVolumeState.Temporary && protectedFiles.Any(pf => pf == i.Name))
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "RemoteUnwantedMissingFile", "removing file listed as {0}: {1}", i.State, i.Name);
cleanupRemovedRemoteVolumes.Add(i.Name);
}
}
}
break;
case RemoteVolumeState.Uploading:
if (remoteFound && correctSize && r.File.Size >= 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
}
else if (!remoteFound)
{
if (protectedFiles.Any(pf => pf == i.Name))
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null);
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name);
cleanupRemovedRemoteVolumes.Add(i.Name);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);
}
}
else
{
if (protectedFiles.Any(pf => pf == i.Name))
{
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "Remove incomplete file", "removing incomplete remote file listed as {0}: {1}", i.State, i.Name);
backend.Delete(i.Name, i.Size, true);
}
}
break;
case RemoteVolumeState.Uploaded:
if (!remoteFound)
{
missing.Add(i);
}
else if (correctSize)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
else
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
case RemoteVolumeState.Verified:
if (!remoteFound)
{
missing.Add(i);
}
else if (!correctSize)
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
default:
Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "unknown state for remote file listed as {0}: {1}", i.State, i.Name);
break;
}
backend.FlushDbMessages();
}
// cleanup deleted volumes in DB en block
database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null);
foreach (var i in missingHash)
{
Logging.Log.WriteWarningMessage(LOGTAG, "MissingRemoteHash", null, "remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash);
}
return(new RemoteAnalysisResult()
{
ParsedVolumes = remotelist,
OtherVolumes = otherlist,
ExtraVolumes = lookup.Values,
MissingVolumes = missing,
VerificationRequiredVolumes = missingHash.Select(x => x.Item2)
});
}
/// <summary>
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
/// </summary>
/// <param name="backend">The backend instance to use</param>
/// <param name="options">The options used</param>
/// <param name="database">The database to compare with</param>
/// <param name="protectedfile">A filename that should be excempted for deletion</param>
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, string protectedfile)
{
var rawlist = backend.List();
var lookup = new Dictionary <string, Volumes.IParsedVolume>();
protectedfile = protectedfile ?? string.Empty;
var remotelist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix == options.Prefix
select p).ToList();
var otherlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p != null && p.Prefix != options.Prefix
select p).ToList();
var unknownlist = (from n in rawlist
let p = Volumes.VolumeBase.ParseFilename(n)
where p == null
select n).ToList();
var filesets = (from n in remotelist
where n.FileType == RemoteVolumeType.Files orderby n.Time descending
select n).ToList();
log.KnownFileCount = remotelist.Count;
log.KnownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum();
log.UnknownFileCount = unknownlist.Count;
log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum();
log.BackupListCount = filesets.Count;
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
// TODO: We should query through the backendmanager
using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions))
if (bk is Library.Interface.IQuotaEnabledBackend)
{
Library.Interface.IQuotaInfo quota = ((Library.Interface.IQuotaEnabledBackend)bk).Quota;
if (quota != null)
{
log.TotalQuotaSpace = quota.TotalQuotaSpace;
log.FreeQuotaSpace = quota.FreeQuotaSpace;
}
}
log.AssignedQuotaSpace = options.QuotaSize;
foreach (var s in remotelist)
{
lookup[s.File.Name] = s;
}
var missing = new List <RemoteVolumeEntry>();
var missingHash = new List <Tuple <long, RemoteVolumeEntry> >();
var cleanupRemovedRemoteVolumes = new HashSet <string>();
foreach (var e in database.DuplicateRemoteVolumes())
{
if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
{
database.UnlinkRemoteVolume(e.Key, e.Value);
}
else
{
throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()));
}
}
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
Volumes.IParsedVolume r;
var remoteFound = lookup.TryGetValue(i.Name, out r);
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
lookup.Remove(i.Name);
switch (i.State)
{
case RemoteVolumeState.Deleted:
if (remoteFound)
{
log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name));
}
break;
case RemoteVolumeState.Temporary:
case RemoteVolumeState.Deleting:
if (remoteFound)
{
log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
else
{
if (i.deleteGracePeriod > DateTime.UtcNow)
{
log.AddMessage(string.Format("keeping delete request for {0} until {1}", i.Name, i.deleteGracePeriod.ToLocalTime()));
}
else
{
if (string.Equals(i.Name, protectedfile) && i.State == RemoteVolumeState.Temporary)
{
log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
}
else
{
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
cleanupRemovedRemoteVolumes.Add(i.Name);
}
}
}
break;
case RemoteVolumeState.Uploading:
if (remoteFound && correctSize && r.File.Size >= 0)
{
log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded));
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
}
else if (!remoteFound)
{
if (string.Equals(i.Name, protectedfile))
{
log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null);
}
else
{
log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));
cleanupRemovedRemoteVolumes.Add(i.Name);
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);
}
}
else
{
if (string.Equals(i.Name, protectedfile))
{
log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
}
else
{
log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));
backend.Delete(i.Name, i.Size, true);
}
}
break;
case RemoteVolumeState.Uploaded:
if (!remoteFound)
{
missing.Add(i);
}
else if (correctSize)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
else
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
case RemoteVolumeState.Verified:
if (!remoteFound)
{
missing.Add(i);
}
else if (!correctSize)
{
missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i));
}
break;
default:
log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
break;
}
backend.FlushDbMessages();
}
// cleanup deleted volumes in DB en block
database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null);
foreach (var i in missingHash)
{
log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null);
}
return(new RemoteAnalysisResult()
{
ParsedVolumes = remotelist,
OtherVolumes = otherlist,
ExtraVolumes = lookup.Values,
MissingVolumes = missing,
VerificationRequiredVolumes = missingHash.Select(x => x.Item2)
});
}
