internal async Task CleanUpTaskUtil(string partitionId, BackupPolicy backupPolicy) { try { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Running CleanUp task for partitionId : {0} and backupPolicy: {1}", partitionId, backupPolicy.Name); await IntroduceRandomDelay(); RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicy.Name, this.timeout, this.tokenSource.Token); Func <bool> condition = () => { if (retentionMetadata == null || retentionMetadata.LastRetentionStartTime == DateTime.MinValue || retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || !retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId)) { return(true); } return(false); }; bool cleanUpComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync( (partition, token) => { return(CleanUpTask(partition, backupPolicy, token)); }, condition, partitionId, this.tokenSource.Token, minimumWaitTimeForRetryOperationInMs, maxRetryCountForCleanup, maximumWaitTimeForCleanUpRetryOperationInMs ); if (!cleanUpComplete) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} was not completed." + "It could be because retention of the partition was going on from a long time which did not complete 40 seconds.", partitionId, backupPolicy.Name); } } catch (Exception ex) { if (ex is OperationCanceledException) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} is canceled", partitionId, backupPolicy.Name); } else { BackupRestoreTrace.TraceSource.WriteError(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} has thrown an exception : {2}", partitionId, backupPolicy.Name, ex); } } }
/// <summary> /// Gets called when a policy is updated /// </summary> internal async Task UpdatePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction) { /** * Cases: * 1. UpdateRetention ----> Re-arm the timer. * 0. Add Retention. * a. Remove retention --> dispose the timer. * b. update the retention. * 2. UpdateStorage ----> * 3. Update Backup Schedule. --> no need to do anything/ * */ try { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In UpdatePolicyAsync of Retention Manager for Backup policy: {0}", backupPolicyName); var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService); RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction); BackupPolicy backupPolicy = await backupPolicyStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction); if (retentionMetadata == null) { if (backupPolicy.RetentionPolicy != null) { retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy); await this.RetentionStore.AddAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction); RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback); // Here we have updated the policy to introduce retention. retentionScheduler.ArmTimer(retentionMetadata, true); if (!RetentionSchedulerDict.TryAdd(backupPolicyName, retentionScheduler)) { // If we are not able to add retention in RetentionSchedulerDict then, stop timer in retention scheduler retentionScheduler.Stop(); BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "UpdatePolicyAsync, Not able to add retentionScheduler to dict for policy : {0} ", backupPolicyName); throw new InvalidOperationException(string.Format("{0}: Key already exists.", backupPolicyName)); } } } else if (backupPolicy.RetentionPolicy == null) { // need to remove the retentionScheduler from RetentionStore. Func <bool> condition = () => { if (retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || retentionMetadata.LastRetentionStartTime == DateTime.MinValue) { return(true); } return(false); }; bool disposeComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync( (policyName, token) => { return(DisposeRetentionScheduler(policyName, timeout, token, transaction)); }, condition, backupPolicyName, cancellationToken, minimumWaitTimeForRetryOperationInMs, maxRetryCountForDisposingRetention, minimumWaitTimeForRetryOperationInMs); if (!disposeComplete) { BackupRestoreTrace.TraceSource.WriteError(TraceType, "Dispose retention did not complete for {0}", backupPolicyName); throw new FabricBackupRestoreLocalRetryException(); } } else { if (retentionMetadata.UpdateRetentionPolicyMetadata(backupPolicy.RetentionPolicy)) { // Update the retention Scheduler store and arm the timer. RetentionScheduler retentionScheduler; if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler)) { BackupRestoreTrace.TraceSource.WriteError(TraceType, "UpdateBackupPolicy: Not able to get retention scheduler for backupPolicy {0}", backupPolicyName); throw new KeyNotFoundException(); } retentionScheduler.RearmTimer(false, retentionMetadata); await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction); } } } catch (Exception ex) { if (ex is OperationCanceledException) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " DisposeRetentionScheduler ended with Timeout as the operation was cencelled: {0}", backupPolicyName); BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " It could be because the retention was in progress and user tried to remove the retention policy during UpdatePolicy: {0}", backupPolicyName); } else { // Check for the exceptions of DisposeRetentionScheduler Timeout Retries. BackupRestoreTrace.TraceSource.WriteError(TraceType, " UpdateBackupPolicy resulted in the exception : {0}", backupPolicyName); } throw; } }