/// <summary>
/// Gets Invoked When a policy is created.
/// </summary>
/// <returns></returns>
internal async Task CreatePolicyAsync(BackupPolicy backupPolicy, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction)
{
// create a metadata and initialise the RetentionMetadata.
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Scheduling retention for backupPolicy : {0}", backupPolicy.Name);
RetentionPolicy retentionPolicy = backupPolicy.RetentionPolicy;
if (null == retentionPolicy)
{
// there is nothing to be done.
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "RetentionPolicy for backupPolicy : {0} is not defined.", backupPolicy.Name);
}
else
{
RetentionMetadata retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy);
await this.RetentionStore.AddAsync(backupPolicy.Name, retentionMetadata, timeout, cancellationToken, transaction);
RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback);
if (!RetentionSchedulerDict.TryAdd(backupPolicy.Name, retentionScheduler))
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "CreatePolicyAsync: Not able to add retention scheduler for backupPolicy : {0}", backupPolicy.Name);
throw new InvalidOperationException(string.Format("{0}: Key already exists ", backupPolicy.Name));
}
retentionScheduler.ArmTimer(retentionMetadata);
}
}
internal TimeSpan GetDueTimeForNextRetentionSchedule(RetentionMetadata retentionMetadata)
{
if (this.retentionPolicy.RetentionPolicyType == RetentionPolicyType.Basic)
{
var derivedRetentionPolicy = (BasicRetentionPolicy)this.retentionPolicy;
var retentionDuration = derivedRetentionPolicy.RetentionDuration;
if (retentionMetadata.LastRetentionStartTime == DateTime.MinValue)
{
return(TimeSpan.Zero);
}
if ((int)retentionDuration.TotalDays != 0)
{
return(TimeSpan.FromDays(1));
}
else if ((int)retentionDuration.TotalHours != 0)
{
return(TimeSpan.FromHours(1));
}
else
{
throw new ArgumentException(StringResources.InvalidRetentionInterval);
}
}
else
{
// Implement the timer code for advanced.
throw new NotImplementedException();
}
}
internal void ArmTimer(RetentionMetadata retentionMetadata, bool updatePolicyToAddRetention = false)
{
if (!this.isStopped)
{
this.retentionTimer.armTimer(false, retentionMetadata, updatePolicyToAddRetention);
this.retentionRescheduleTimer.Reset();
}
}
internal async Task DeletePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction)
{
/**
* cases:
* 1. If cleanup going on, then stall deleting the policy.
*
* need to change cleanup store.
* Cleanup store change ---> {"backupPolicyName": "PartitionList"}--> this should have been there at the first place.
*
* Calling function should call it again and again and timeout if it true is not returned.
* * */
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DeletePolicyAsync : {0} is getting deleted", backupPolicyName);
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
List <string> partitionList = await this.CleanupStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
if (partitionList == null || partitionList.Count == 0)
{
// Finally,remove retentionMetadata from retentionStore for the backupPolicyName.
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
if (retentionMetadata != null)
{
await this.RetentionStore.DeleteValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Stopping RetentionScheduler for backupPolicy: {0}", backupPolicyName);
RetentionScheduler retentionScheduler;
if (!this.RetentionSchedulerDict.TryRemove(backupPolicyName, out retentionScheduler))
{
// We wanted to delete the value, we did not find it, so moving on.
return;
}
retentionScheduler.Stop();
}
return;
}
await Task.Delay(waitTimeInMilliSecondForDeleteOp);
}
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "{0} Policy cannot be deleted because the operation is cancelled because of timeout ", backupPolicyName);
}
else
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "{0} Policy cannot be deleted because of the exception: {1} ", backupPolicyName, ex);
}
throw ex;
}
}
internal async Task CleanUpTaskUtil(string partitionId, BackupPolicy backupPolicy)
{
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Running CleanUp task for partitionId : {0} and backupPolicy: {1}", partitionId, backupPolicy.Name);
await IntroduceRandomDelay();
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicy.Name, this.timeout, this.tokenSource.Token);
Func <bool> condition = () =>
{
if (retentionMetadata == null || retentionMetadata.LastRetentionStartTime == DateTime.MinValue || retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime ||
!retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId))
{
return(true);
}
return(false);
};
bool cleanUpComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
(partition, token) =>
{
return(CleanUpTask(partition, backupPolicy, token));
}, condition, partitionId,
this.tokenSource.Token,
minimumWaitTimeForRetryOperationInMs,
maxRetryCountForCleanup,
maximumWaitTimeForCleanUpRetryOperationInMs
);
if (!cleanUpComplete)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} was not completed." +
"It could be because retention of the partition was going on from a long time which did not complete 40 seconds.", partitionId, backupPolicy.Name);
}
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} is canceled", partitionId, backupPolicy.Name);
}
else
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} has thrown an exception : {2}", partitionId, backupPolicy.Name, ex);
}
}
}
internal void RearmTimer(bool postFailedScheduled, RetentionMetadata retentionMetadata)
{
if (postFailedScheduled && !this.retentionRescheduleTimer.IsReschduleCountExhausted())
{
this.retentionRescheduleTimer.Reschedule();
}
else if (postFailedScheduled && this.retentionRescheduleTimer.IsReschduleCountExhausted())
{
this.retentionRescheduleTimer.Reset();
this.retentionTimer.armTimer(true, retentionMetadata);
}
else
{
this.retentionTimer.armTimer(false, retentionMetadata);
}
}
private async Task DisposeRetentionScheduler(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DisposeRetentionScheduler for backupPolicy {0}", backupPolicyName);
cancellationToken.ThrowIfCancellationRequested();
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
await this.RetentionStore.DeleteValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
// It means timer callback is not running.
RetentionScheduler retentionScheduler;
if (!this.RetentionSchedulerDict.TryRemove(backupPolicyName, out retentionScheduler))
{
// This means key does not exists and we are just trying to remove the key.
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Key: {0} not found in retentionSchedulerDict.", backupPolicyName);
return;
}
retentionScheduler.Stop();
}
internal void armTimer(bool postFailedReschedule, RetentionMetadata retentionMetadata, bool updatePolicyToAddRetention = false)
{
TimeSpan dueTime = TimeSpan.Zero;
Random rand = new Random();
dueTime = dueTime + TimeSpan.FromMilliseconds((double)rand.Next(jitterLow, jitterHigh));
if (updatePolicyToAddRetention)
{
this.timer.ArmTimer(dueTime.TotalMilliseconds);
return;
}
TimeSpan timeSinceLastRetention = postFailedReschedule == true ? TimeSpan.Zero : GetTimeSinceLastRetentionCompletion(DateTime.Now, retentionMetadata);
TimeSpan dueTimeForNextRetention = GetDueTimeForNextRetentionSchedule(retentionMetadata);
if (dueTimeForNextRetention >= timeSinceLastRetention)
{
dueTime = dueTime + (dueTimeForNextRetention - timeSinceLastRetention);
}
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Timer armed to run after {0} milliseconds ", dueTime.TotalMilliseconds);
this.timer.ArmTimer(dueTime.TotalMilliseconds);
}
private async Task DeleteFilesForPartition(string partitionId, DateTime endDateFilter, int minimumNumberOfBackups, BackupStorage storage, string backupPolicyName, bool isCleanupTask = false)
{
// Design:
/**
* Need out Two API's from Azure share and file share
* 1. List of all backups before a certain date for the partition
* 2. Start deleting the files one by one.(one file at a time with some random wait time.)
*
**/
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DeleteFilesForPartition for partitionId : {0} with endDateFilter {1}", partitionId, endDateFilter);
string applicationName = null;
string serviceName = null;
string partitionID = null;
string fabricUri = await UtilityHelper.GetFabricUriFromPartitionId(partitionId, this.timeout, this.tokenSource.Token);
var FabricBackupResourceType = UtilityHelper.GetApplicationAndServicePartitionName(fabricUri, out applicationName,
out serviceName, out partitionID);
Debug.Assert(partitionId == partitionID);
var storeManager = RecoveryPointManagerFactory.GetRecoveryPointManager(storage);
this.tokenSource.Token.ThrowIfCancellationRequested();
BRSContinuationToken bRSContinuationToken = new BRSContinuationToken();
List <string> allBackupFiles = storeManager.EnumerateRecoveryPointMetadataFiles(storage.GetRelativeStorePath(applicationName, serviceName, partitionId),
DateTime.MinValue, DateTime.MaxValue, false, bRSContinuationToken, 0);
List <RestorePoint> restorePoints = new List <RestorePoint>();
if ((allBackupFiles != null) && (allBackupFiles.Count > 0))
{
restorePoints = await storeManager.GetRecoveryPointDetailsAsync(allBackupFiles, this.tokenSource.Token);
}
this.tokenSource.Token.ThrowIfCancellationRequested();
int backupCount = allBackupFiles.Count;
if (backupCount <= minimumNumberOfBackups)
{
return;
}
restorePoints = restorePoints.OrderBy(restorePoint => restorePoint.CreationTimeUtc).ToList();
restorePoints = restorePoints.Where(restorePoint => restorePoint.CreationTimeUtc < endDateFilter).ToList();
restorePoints.Reverse();
bool lastFullBackupFound = false;
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token);
if (isCleanupTask)
{
lastFullBackupFound = true;
}
if (!isCleanupTask && retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId) &&
retentionMetadata.OnGoingRetentionFile[partitionId] != null)
{
storeManager.DeleteBackupFiles(retentionMetadata.OnGoingRetentionFile[partitionId]);
}
foreach (var restorePoint in restorePoints)
{
this.tokenSource.Token.ThrowIfCancellationRequested();
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Trying to Delete Restore point with backupLocation: {0}", restorePoint.BackupLocation);
if (!isCleanupTask)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0} with partitionId : {1} and BackupLocation {2}"
, backupPolicyName, partitionId, restorePoint.BackupLocation);
retentionMetadata.UpdateOnGoingRetentionFile(partitionId, restorePoint.BackupLocation);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null);
}
if (!lastFullBackupFound)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Skipping RestorePoint with location : {0} as full backup is not found", restorePoint.BackupLocation);
if (restorePoint.BackupType == BackupOptionType.Full)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Full Backup found so, after restore points after :{0} will be deleted. ", restorePoint.BackupLocation);
lastFullBackupFound = true;
}
}
else
{
if (backupCount <= minimumNumberOfBackups)
{
break;
}
if (storeManager.DeleteBackupFiles(restorePoint.BackupLocation))
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Deleted Restore point with backupLocation: {0}", restorePoint.BackupLocation);
backupCount--;
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " Not able to delete Restore point with backupLocation: {0}", restorePoint.BackupLocation);
}
}
await IntroduceRandomDelay();
}
// Deletion for a partition is completed.
if (!isCleanupTask)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0}", backupPolicyName);
retentionMetadata.UpdateOnGoingRetentionFile(partitionId);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null);
}
}
/// <summary>
/// Gets called when a policy is updated
/// </summary>
internal async Task UpdatePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction)
{
/**
* Cases:
* 1. UpdateRetention ----> Re-arm the timer.
* 0. Add Retention.
* a. Remove retention --> dispose the timer.
* b. update the retention.
* 2. UpdateStorage ---->
* 3. Update Backup Schedule. --> no need to do anything/
*
*/
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In UpdatePolicyAsync of Retention Manager for Backup policy: {0}", backupPolicyName);
var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService);
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
BackupPolicy backupPolicy = await backupPolicyStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
if (retentionMetadata == null)
{
if (backupPolicy.RetentionPolicy != null)
{
retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy);
await this.RetentionStore.AddAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);
RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback);
// Here we have updated the policy to introduce retention.
retentionScheduler.ArmTimer(retentionMetadata, true);
if (!RetentionSchedulerDict.TryAdd(backupPolicyName, retentionScheduler))
{
// If we are not able to add retention in RetentionSchedulerDict then, stop timer in retention scheduler
retentionScheduler.Stop();
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "UpdatePolicyAsync, Not able to add retentionScheduler to dict for policy : {0} ", backupPolicyName);
throw new InvalidOperationException(string.Format("{0}: Key already exists.", backupPolicyName));
}
}
}
else if (backupPolicy.RetentionPolicy == null)
{
// need to remove the retentionScheduler from RetentionStore.
Func <bool> condition = () =>
{
if (retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || retentionMetadata.LastRetentionStartTime == DateTime.MinValue)
{
return(true);
}
return(false);
};
bool disposeComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
(policyName, token) =>
{
return(DisposeRetentionScheduler(policyName, timeout, token, transaction));
},
condition,
backupPolicyName,
cancellationToken,
minimumWaitTimeForRetryOperationInMs,
maxRetryCountForDisposingRetention,
minimumWaitTimeForRetryOperationInMs);
if (!disposeComplete)
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "Dispose retention did not complete for {0}", backupPolicyName);
throw new FabricBackupRestoreLocalRetryException();
}
}
else
{
if (retentionMetadata.UpdateRetentionPolicyMetadata(backupPolicy.RetentionPolicy))
{
// Update the retention Scheduler store and arm the timer.
RetentionScheduler retentionScheduler;
if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler))
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "UpdateBackupPolicy: Not able to get retention scheduler for backupPolicy {0}", backupPolicyName);
throw new KeyNotFoundException();
}
retentionScheduler.RearmTimer(false, retentionMetadata);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);
}
}
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " DisposeRetentionScheduler ended with Timeout as the operation was cencelled: {0}", backupPolicyName);
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " It could be because the retention was in progress and user tried to remove the retention policy during UpdatePolicy: {0}", backupPolicyName);
}
else
{
// Check for the exceptions of DisposeRetentionScheduler Timeout Retries.
BackupRestoreTrace.TraceSource.WriteError(TraceType, " UpdateBackupPolicy resulted in the exception : {0}", backupPolicyName);
}
throw;
}
}
/// <summary>
/// Gets invoked when the RetentionScheduler's timer's time is elapsed
/// </summary>
internal void timerCallback(string backupPolicyName)
{
RetentionScheduler retentionScheduler;
if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler))
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "RetentionScheduler object is not found in retentionScheduler dict for policy: {0}", backupPolicyName);
return;
}
bool toRearm = true;
RetentionMetadata retentionMetadata = this.RetentionStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult();
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " TimerCallback running for backupPolicy : {0}", backupPolicyName);
var backupPolicyStore = BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService).GetAwaiter().GetResult();
BackupPolicy backupPolicy = backupPolicyStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult();
if (backupPolicy == null)
{
/**
* This is used to garbage collect the retention scheduler. It could happen while deleting\updating policy retention scheduler object was not
* stopped properly. So, timer will call its callback again just to find that the backup policy does not exists.
* In this case, we should stop the retentionScheduler.
* **/
RetentionScheduler retentionSchedulerToDelete;
if (RetentionSchedulerDict.TryRemove(backupPolicyName, out retentionSchedulerToDelete))
{
retentionSchedulerToDelete.Stop();
}
else
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "TimerCallback TryRemove retentionSchedulerDict failed of policy {0}. It could happen" +
" when residual callback already scheduled on threadpool thread is getting invoked.", backupPolicyName);
}
toRearm = false;
return;
}
HashSet <string> partitionsEnabledByPolicy = FindParititionsEnabledByPolicy(backupPolicy).GetAwaiter().GetResult();
retentionMetadata.UpdateLastRetentionStartTime();
this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null).GetAwaiter().GetResult();
var derivedRetentionPolicy = (BasicRetentionPolicy)retentionMetadata.CurrentRetentionPolicy;
TimeSpan retentionDuration = derivedRetentionPolicy.RetentionDuration;
int minimumNumberOfBackups = derivedRetentionPolicy.MinimumNumberOfBackups;
foreach (var partitionId in partitionsEnabledByPolicy)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Deleting backups for partitionId : {0}", partitionId);
List <string> partitionList = this.CleanupStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult();
if (!(partitionList == null || partitionList.Count == 0) && partitionList.Contains(partitionId))
{
// Cleanup is going on for this partition.
continue;
}
DateTime endDateFilter = DateTime.UtcNow - retentionDuration;
BackupStorage storage = backupPolicy.Storage;
DeleteFilesForPartition(partitionId, endDateFilter, minimumNumberOfBackups, storage, backupPolicyName).GetAwaiter().GetResult();
this.tokenSource.Token.ThrowIfCancellationRequested();
}
// --> Introduce some wait time here so that CompletionTime is never equal to StartTime.
retentionMetadata.UpdateLastRetentionCompletionTime();
this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata).GetAwaiter().GetResult();
retentionScheduler.ArmTimer(retentionMetadata);
toRearm = false;
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
// Since, timercallback was cancelled, therefore, there is no need to rearm the timer.
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " TimerCallback for backupPolicy : {0} was cancelled ", backupPolicyName);
toRearm = false;
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " TimerCallback for backupPolicy : {0} has thrown an exception : {1}", backupPolicyName, ex);
}
}
finally
{
if (toRearm)
{
retentionScheduler.RearmTimer(true, retentionMetadata);
}
}
}
internal TimeSpan GetTimeSinceLastRetentionCompletion(DateTime currentDateTime, RetentionMetadata retentionMetadata)
{
TimeSpan timeSinceLastRetentionCompletion = TimeSpan.Zero; // Returning zero means we will wait for retention time.
if (retentionMetadata.LastRetentionCompletionTime != DateTime.MinValue)
{
// It means retention completed atleast once. => We need to wait on the basis of time.
timeSinceLastRetentionCompletion = currentDateTime - retentionMetadata.LastRetentionCompletionTime;
}
else if (retentionMetadata.LastRetentionStartTime != DateTime.MinValue)
{
// It means retention never completed once, but we have waited => We need retention right now.
timeSinceLastRetentionCompletion = currentDateTime - retentionMetadata.LastRetentionCompletionTime;
}
return(timeSinceLastRetentionCompletion);
}
