/// <summary>
/// Gets the filesets selected for deletion
/// </summary>
/// <returns>The filesets to delete</returns>
/// <param name="allBackups">The list of backups that can be deleted</param>
private DateTime[] GetFilesetsToDelete(Database.LocalDeleteDatabase db, DateTime[] allBackups)
{
if (allBackups.Length == 0)
{
return(allBackups);
}
DateTime[] sortedAllBackups = allBackups.OrderByDescending(x => x.ToUniversalTime()).ToArray();
if (sortedAllBackups.Select(x => x.ToUniversalTime()).Distinct().Count() != sortedAllBackups.Length)
{
throw new Exception($"List of backup timestamps contains duplicates: {string.Join(", ", sortedAllBackups.Select(x => x.ToString()))}");
}
List <DateTime> toDelete = new List <DateTime>();
// Remove backups explicitly specified via option
var versions = m_options.Version;
if (versions != null && versions.Length > 0)
{
foreach (var ix in versions.Distinct())
{
if (ix >= 0 && ix < sortedAllBackups.Length)
{
toDelete.Add(sortedAllBackups[ix]);
}
}
}
// Remove backups that are older than date specified via option while ensuring
// that we always have at least one full backup.
var keepTime = m_options.KeepTime;
if (keepTime.Ticks > 0)
{
bool haveFullBackup = false;
toDelete.AddRange(sortedAllBackups.SkipWhile(x =>
{
bool keepBackup = (x >= keepTime) || !haveFullBackup;
haveFullBackup = haveFullBackup || db.IsFilesetFullBackup(x);
return(keepBackup);
}));
}
// Remove backups via retention policy option
toDelete.AddRange(ApplyRetentionPolicy(db, sortedAllBackups));
// Check how many full backups will be remaining after the previous steps
// and remove oldest backups while there are still more backups than should be kept as specified via option
var backupsRemaining = sortedAllBackups.Except(toDelete).ToList();
var fullVersionsToKeep = m_options.KeepVersions;
if (fullVersionsToKeep > 0 && fullVersionsToKeep < backupsRemaining.Count)
{
int fullVersionsKept = 0;
ISet <DateTime> intermediatePartials = new HashSet <DateTime>();
// Enumerate the collection starting from the most recent full backup.
foreach (DateTime backup in backupsRemaining.SkipWhile(x => !db.IsFilesetFullBackup(x)))
{
if (fullVersionsKept >= fullVersionsToKeep)
{
// If we have enough full backups, delete all older backups.
toDelete.Add(backup);
}
else if (db.IsFilesetFullBackup(backup))
{
// We can delete partial backups that are surrounded by full backups.
toDelete.AddRange(intermediatePartials);
intermediatePartials.Clear();
fullVersionsKept++;
}
else
{
intermediatePartials.Add(backup);
}
}
}
var toDeleteDistinct = toDelete.Distinct().OrderByDescending(x => x.ToUniversalTime()).ToArray();
var removeCount = toDeleteDistinct.Length;
if (removeCount > sortedAllBackups.Length)
{
throw new Exception($"Too many entries {removeCount} vs {sortedAllBackups.Length}, lists: {string.Join(", ", toDeleteDistinct.Select(x => x.ToString(CultureInfo.InvariantCulture)))} vs {string.Join(", ", sortedAllBackups.Select(x => x.ToString(CultureInfo.InvariantCulture)))}");
}
return(toDeleteDistinct);
}
/// <summary>
/// Deletes backups according to the retention policy configuration.
/// Backups that are not within any of the specified time frames will will NOT be deleted.
/// </summary>
/// <returns>The filesets to delete</returns>
/// <param name="backups">The list of backups that can be deleted</param>
private List <DateTime> ApplyRetentionPolicy(Database.LocalDeleteDatabase db, DateTime[] backups)
{
// Any work to do?
var retentionPolicyOptionValues = m_options.RetentionPolicy;
if (retentionPolicyOptionValues.Count == 0 || backups.Length == 0)
{
return(new List <DateTime>()); // don't delete any backups
}
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "StartCheck", "Start checking if backups can be removed");
// Work with a copy to not modify the enumeration that the caller passed
List <DateTime> clonedBackupList = new List <DateTime>(backups);
// Make sure the backups are in descending order (newest backup in the beginning)
clonedBackupList = clonedBackupList.OrderByDescending(x => x).ToList();
// Most recent backup usually should never get deleted in this process, so exclude it for now,
// but keep a reference to potential delete it when allow-full-removal is set
var mostRecentBackup = clonedBackupList.ElementAt(0);
clonedBackupList.RemoveAt(0);
var deleteMostRecentBackup = m_options.AllowFullRemoval;
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "FramesAndIntervals", "Time frames and intervals pairs: {0}",
string.Join(", ", retentionPolicyOptionValues));
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "BackupList", "Backups to consider: {0}",
string.Join(", ", clonedBackupList));
// Collect all potential backups in each time frame and thin out according to the specified interval,
// starting with the oldest backup in that time frame.
// The order in which the time frames values are checked has to be from the smallest to the largest.
List <DateTime> backupsToDelete = new List <DateTime>();
var now = DateTime.Now;
foreach (var singleRetentionPolicyOptionValue in retentionPolicyOptionValues.OrderBy(x => x.Timeframe))
{
// The timeframe in the retention policy option is only a timespan which has to be applied to the current DateTime to get the actual lower bound
DateTime timeFrame = (singleRetentionPolicyOptionValue.IsUnlimtedTimeframe()) ? DateTime.MinValue : (now - singleRetentionPolicyOptionValue.Timeframe);
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "NextTimeAndFrame", "Next time frame and interval pair: {0}", singleRetentionPolicyOptionValue.ToString());
List <DateTime> backupsInTimeFrame = new List <DateTime>();
while (clonedBackupList.Count > 0 && clonedBackupList[0] >= timeFrame)
{
backupsInTimeFrame.Insert(0, clonedBackupList[0]); // Insert at beginning to reverse order, which is necessary for next step
clonedBackupList.RemoveAt(0); // remove from here to not handle the same backup in two time frames
}
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "BackupsInFrame", "Backups in this time frame: {0}",
string.Join(", ", backupsInTimeFrame));
// Run through backups in this time frame
DateTime?lastKept = null;
foreach (DateTime backup in backupsInTimeFrame)
{
var isFullBackup = db.IsFilesetFullBackup(backup);
// Keep this backup if
// - no backup has yet been added to the time frame (keeps at least the oldest backup in a time frame)
// - difference between last added backup and this backup is bigger than the specified interval
if (lastKept == null || singleRetentionPolicyOptionValue.IsKeepAllVersions() || (backup - lastKept.Value) >= singleRetentionPolicyOptionValue.Interval)
{
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "KeepBackups", $"Keeping {(isFullBackup ? "" : "partial")} backup: {backup}", Logging.LogMessageType.Profiling);
if (isFullBackup)
{
lastKept = backup;
}
}
else
{
if (isFullBackup)
{
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "DeletingBackups",
"Deleting backup: {0}", backup);
backupsToDelete.Add(backup);
}
else
{
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "KeepBackups", $"Keeping partial backup: {backup}", Logging.LogMessageType.Profiling);
}
}
}
// Check if most recent backup is outside of this time frame (meaning older/smaller)
deleteMostRecentBackup &= (mostRecentBackup < timeFrame);
}
// Delete all remaining backups
backupsToDelete.AddRange(clonedBackupList);
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "BackupsToDelete", "Backups outside of all time frames and thus getting deleted: {0}",
string.Join(", ", clonedBackupList));
// Delete most recent backup if allow-full-removal is set and the most current backup is outside of any time frame
if (deleteMostRecentBackup)
{
backupsToDelete.Add(mostRecentBackup);
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "DeleteMostRecent", "Deleting most recent backup: {0}",
mostRecentBackup);
}
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "AllBackupsToDelete", "All backups to delete: {0}",
string.Join(", ", backupsToDelete.OrderByDescending(x => x)));
return(backupsToDelete);
}
/// <summary>
/// Gets the filesets selected for deletion
/// </summary>
/// <returns>The filesets to delete</returns>
/// <param name="allBackups">The list of backups that can be deleted</param>
private DateTime[] GetFilesetsToDelete(Database.LocalDeleteDatabase db, DateTime[] allBackups)
{
if (allBackups.Length == 0)
{
return(allBackups);
}
DateTime[] sortedAllBackups = allBackups.OrderByDescending(x => x.ToUniversalTime()).ToArray();
if (sortedAllBackups.Select(x => x.ToUniversalTime()).Distinct().Count() != sortedAllBackups.Length)
{
throw new Exception($"List of backup timestamps contains duplicates: {string.Join(", ", sortedAllBackups.Select(x => x.ToString()))}");
}
List <DateTime> toDelete = new List <DateTime>();
// Remove backups explicitly specified via option
var versions = m_options.Version;
if (versions != null && versions.Length > 0)
{
foreach (var ix in versions.Distinct())
{
if (ix >= 0 && ix < sortedAllBackups.Length)
{
toDelete.Add(sortedAllBackups[ix]);
}
}
}
// Remove backups that are older than date specified via option
var keepTime = m_options.KeepTime;
if (keepTime.Ticks > 0)
{
toDelete.AddRange(sortedAllBackups.SkipWhile(x => x >= keepTime));
}
// Remove backups via retention policy option
toDelete.AddRange(ApplyRetentionPolicy(db, sortedAllBackups));
// Check how many full backups will be remaining after the previous steps
// and remove oldest backups while there are still more backups than should be kept as specified via option
var backupsRemaining = sortedAllBackups.Except(toDelete).ToList();
var fullVersionsToKeep = m_options.KeepVersions;
var fullVersionsKeptCount = 0;
if (fullVersionsToKeep > 0 && fullVersionsToKeep < backupsRemaining.Count)
{
// keep the number of full backups specified in fullVersionsToKeep.
// once the last full backup t okeep is found, also keep the partials immediately after it the full backup.
// add the remainder of full and partial backups to toDelete
bool foundLastFullBackupToKeep = false;
foreach (var backup in backupsRemaining)
{
bool isFullBackup;
if (fullVersionsKeptCount < fullVersionsToKeep)
{
isFullBackup = db.IsFilesetFullBackup(backup);
// count only a full backup
if (fullVersionsKeptCount < fullVersionsToKeep && isFullBackup)
{
fullVersionsKeptCount++;
}
continue;
}
// do not include any partial backup that precedes the last full backup
if (!foundLastFullBackupToKeep)
{
isFullBackup = db.IsFilesetFullBackup(backup);
if (!isFullBackup)
{
continue;
}
foundLastFullBackupToKeep = true;
}
toDelete.Add(backup);
}
}
var toDeleteDistinct = toDelete.Distinct().OrderByDescending(x => x.ToUniversalTime()).ToArray();
var removeCount = toDeleteDistinct.Length;
if (removeCount > sortedAllBackups.Length)
{
throw new Exception($"Too many entries {removeCount} vs {sortedAllBackups.Length}, lists: {string.Join(", ", toDeleteDistinct.Select(x => x.ToString(CultureInfo.InvariantCulture)))} vs {string.Join(", ", sortedAllBackups.Select(x => x.ToString(CultureInfo.InvariantCulture)))}");
}
return(toDeleteDistinct);
}