예제 #1
0
        internal async Task CleanUpTaskUtil(string partitionId, BackupPolicy backupPolicy)
        {
            try
            {
                BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Running CleanUp task for partitionId : {0} and backupPolicy: {1}", partitionId, backupPolicy.Name);
                await IntroduceRandomDelay();

                RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicy.Name, this.timeout, this.tokenSource.Token);

                Func <bool> condition = () =>
                {
                    if (retentionMetadata == null || retentionMetadata.LastRetentionStartTime == DateTime.MinValue || retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime ||
                        !retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId))
                    {
                        return(true);
                    }
                    return(false);
                };

                bool cleanUpComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
                    (partition, token) =>
                {
                    return(CleanUpTask(partition, backupPolicy, token));
                }, condition, partitionId,
                    this.tokenSource.Token,
                    minimumWaitTimeForRetryOperationInMs,
                    maxRetryCountForCleanup,
                    maximumWaitTimeForCleanUpRetryOperationInMs
                    );

                if (!cleanUpComplete)
                {
                    BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} was not completed." +
                                                                "It could be because retention of the partition was going on from a long time which did not complete 40 seconds.", partitionId, backupPolicy.Name);
                }
            }
            catch (Exception ex)
            {
                if (ex is OperationCanceledException)
                {
                    BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} is canceled", partitionId, backupPolicy.Name);
                }
                else
                {
                    BackupRestoreTrace.TraceSource.WriteError(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} has thrown an exception : {2}", partitionId, backupPolicy.Name, ex);
                }
            }
        }
예제 #2
0
        /// <summary>
        /// Gets called when a policy is updated
        /// </summary>
        internal async Task UpdatePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction)
        {
            /**
             * Cases:
             * 1. UpdateRetention ----> Re-arm the timer.
             *    0. Add Retention.
             *    a. Remove retention --> dispose the timer.
             *    b. update the retention.
             * 2. UpdateStorage ---->
             * 3. Update Backup Schedule. --> no need to do anything/
             *
             */
            try
            {
                BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In UpdatePolicyAsync of Retention Manager for Backup policy: {0}", backupPolicyName);
                var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService);

                RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);

                BackupPolicy backupPolicy = await backupPolicyStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);

                if (retentionMetadata == null)
                {
                    if (backupPolicy.RetentionPolicy != null)
                    {
                        retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy);
                        await this.RetentionStore.AddAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);

                        RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback);

                        // Here we have updated the policy to introduce retention.
                        retentionScheduler.ArmTimer(retentionMetadata, true);
                        if (!RetentionSchedulerDict.TryAdd(backupPolicyName, retentionScheduler))
                        {
                            // If we are not able to add retention in RetentionSchedulerDict then, stop timer in retention scheduler
                            retentionScheduler.Stop();
                            BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "UpdatePolicyAsync, Not able to add retentionScheduler to dict for policy : {0} ", backupPolicyName);
                            throw new InvalidOperationException(string.Format("{0}: Key already exists.", backupPolicyName));
                        }
                    }
                }
                else if (backupPolicy.RetentionPolicy == null)
                {
                    // need to remove the retentionScheduler from RetentionStore.
                    Func <bool> condition = () =>
                    {
                        if (retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || retentionMetadata.LastRetentionStartTime == DateTime.MinValue)
                        {
                            return(true);
                        }
                        return(false);
                    };

                    bool disposeComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
                        (policyName, token) =>
                    {
                        return(DisposeRetentionScheduler(policyName, timeout, token, transaction));
                    },
                        condition,
                        backupPolicyName,
                        cancellationToken,
                        minimumWaitTimeForRetryOperationInMs,
                        maxRetryCountForDisposingRetention,
                        minimumWaitTimeForRetryOperationInMs);

                    if (!disposeComplete)
                    {
                        BackupRestoreTrace.TraceSource.WriteError(TraceType, "Dispose retention did not complete for {0}", backupPolicyName);
                        throw new FabricBackupRestoreLocalRetryException();
                    }
                }
                else
                {
                    if (retentionMetadata.UpdateRetentionPolicyMetadata(backupPolicy.RetentionPolicy))
                    {
                        // Update the retention Scheduler store and arm the timer.
                        RetentionScheduler retentionScheduler;
                        if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler))
                        {
                            BackupRestoreTrace.TraceSource.WriteError(TraceType, "UpdateBackupPolicy: Not able to get retention scheduler for backupPolicy {0}", backupPolicyName);
                            throw new KeyNotFoundException();
                        }

                        retentionScheduler.RearmTimer(false, retentionMetadata);
                        await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);
                    }
                }
            }
            catch (Exception ex)
            {
                if (ex is OperationCanceledException)
                {
                    BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " DisposeRetentionScheduler ended with Timeout as the operation was cencelled: {0}", backupPolicyName);
                    BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " It could be because the retention was in progress and user tried to remove the retention policy during UpdatePolicy: {0}", backupPolicyName);
                }
                else
                {
                    // Check for the exceptions of DisposeRetentionScheduler Timeout Retries.
                    BackupRestoreTrace.TraceSource.WriteError(TraceType, " UpdateBackupPolicy resulted in the exception : {0}", backupPolicyName);
                }
                throw;
            }
        }