internal async Task CleanUpTaskUtil(string partitionId, BackupPolicy backupPolicy)
{
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Running CleanUp task for partitionId : {0} and backupPolicy: {1}", partitionId, backupPolicy.Name);
await IntroduceRandomDelay();
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicy.Name, this.timeout, this.tokenSource.Token);
Func <bool> condition = () =>
{
if (retentionMetadata == null || retentionMetadata.LastRetentionStartTime == DateTime.MinValue || retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime ||
!retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId))
{
return(true);
}
return(false);
};
bool cleanUpComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
(partition, token) =>
{
return(CleanUpTask(partition, backupPolicy, token));
}, condition, partitionId,
this.tokenSource.Token,
minimumWaitTimeForRetryOperationInMs,
maxRetryCountForCleanup,
maximumWaitTimeForCleanUpRetryOperationInMs
);
if (!cleanUpComplete)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} was not completed." +
"It could be because retention of the partition was going on from a long time which did not complete 40 seconds.", partitionId, backupPolicy.Name);
}
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} is canceled", partitionId, backupPolicy.Name);
}
else
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} has thrown an exception : {2}", partitionId, backupPolicy.Name, ex);
}
}
}
/// <summary>
/// Gets called when a policy is updated
/// </summary>
internal async Task UpdatePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction)
{
/**
* Cases:
* 1. UpdateRetention ----> Re-arm the timer.
* 0. Add Retention.
* a. Remove retention --> dispose the timer.
* b. update the retention.
* 2. UpdateStorage ---->
* 3. Update Backup Schedule. --> no need to do anything/
*
*/
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In UpdatePolicyAsync of Retention Manager for Backup policy: {0}", backupPolicyName);
var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService);
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
BackupPolicy backupPolicy = await backupPolicyStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
if (retentionMetadata == null)
{
if (backupPolicy.RetentionPolicy != null)
{
retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy);
await this.RetentionStore.AddAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);
RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback);
// Here we have updated the policy to introduce retention.
retentionScheduler.ArmTimer(retentionMetadata, true);
if (!RetentionSchedulerDict.TryAdd(backupPolicyName, retentionScheduler))
{
// If we are not able to add retention in RetentionSchedulerDict then, stop timer in retention scheduler
retentionScheduler.Stop();
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "UpdatePolicyAsync, Not able to add retentionScheduler to dict for policy : {0} ", backupPolicyName);
throw new InvalidOperationException(string.Format("{0}: Key already exists.", backupPolicyName));
}
}
}
else if (backupPolicy.RetentionPolicy == null)
{
// need to remove the retentionScheduler from RetentionStore.
Func <bool> condition = () =>
{
if (retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || retentionMetadata.LastRetentionStartTime == DateTime.MinValue)
{
return(true);
}
return(false);
};
bool disposeComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
(policyName, token) =>
{
return(DisposeRetentionScheduler(policyName, timeout, token, transaction));
},
condition,
backupPolicyName,
cancellationToken,
minimumWaitTimeForRetryOperationInMs,
maxRetryCountForDisposingRetention,
minimumWaitTimeForRetryOperationInMs);
if (!disposeComplete)
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "Dispose retention did not complete for {0}", backupPolicyName);
throw new FabricBackupRestoreLocalRetryException();
}
}
else
{
if (retentionMetadata.UpdateRetentionPolicyMetadata(backupPolicy.RetentionPolicy))
{
// Update the retention Scheduler store and arm the timer.
RetentionScheduler retentionScheduler;
if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler))
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "UpdateBackupPolicy: Not able to get retention scheduler for backupPolicy {0}", backupPolicyName);
throw new KeyNotFoundException();
}
retentionScheduler.RearmTimer(false, retentionMetadata);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);
}
}
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " DisposeRetentionScheduler ended with Timeout as the operation was cencelled: {0}", backupPolicyName);
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " It could be because the retention was in progress and user tried to remove the retention policy during UpdatePolicy: {0}", backupPolicyName);
}
else
{
// Check for the exceptions of DisposeRetentionScheduler Timeout Retries.
BackupRestoreTrace.TraceSource.WriteError(TraceType, " UpdateBackupPolicy resulted in the exception : {0}", backupPolicyName);
}
throw;
}
}