/// <summary> /// Gets Invoked When a policy is created. /// </summary> /// <returns></returns> internal async Task CreatePolicyAsync(BackupPolicy backupPolicy, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction) { // create a metadata and initialise the RetentionMetadata. BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Scheduling retention for backupPolicy : {0}", backupPolicy.Name); RetentionPolicy retentionPolicy = backupPolicy.RetentionPolicy; if (null == retentionPolicy) { // there is nothing to be done. BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "RetentionPolicy for backupPolicy : {0} is not defined.", backupPolicy.Name); } else { RetentionMetadata retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy); await this.RetentionStore.AddAsync(backupPolicy.Name, retentionMetadata, timeout, cancellationToken, transaction); RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback); if (!RetentionSchedulerDict.TryAdd(backupPolicy.Name, retentionScheduler)) { BackupRestoreTrace.TraceSource.WriteError(TraceType, "CreatePolicyAsync: Not able to add retention scheduler for backupPolicy : {0}", backupPolicy.Name); throw new InvalidOperationException(string.Format("{0}: Key already exists ", backupPolicy.Name)); } retentionScheduler.ArmTimer(retentionMetadata); } }
internal TimeSpan GetDueTimeForNextRetentionSchedule(RetentionMetadata retentionMetadata) { if (this.retentionPolicy.RetentionPolicyType == RetentionPolicyType.Basic) { var derivedRetentionPolicy = (BasicRetentionPolicy)this.retentionPolicy; var retentionDuration = derivedRetentionPolicy.RetentionDuration; if (retentionMetadata.LastRetentionStartTime == DateTime.MinValue) { return(TimeSpan.Zero); } if ((int)retentionDuration.TotalDays != 0) { return(TimeSpan.FromDays(1)); } else if ((int)retentionDuration.TotalHours != 0) { return(TimeSpan.FromHours(1)); } else { throw new ArgumentException(StringResources.InvalidRetentionInterval); } } else { // Implement the timer code for advanced. throw new NotImplementedException(); } }
internal void ArmTimer(RetentionMetadata retentionMetadata, bool updatePolicyToAddRetention = false) { if (!this.isStopped) { this.retentionTimer.armTimer(false, retentionMetadata, updatePolicyToAddRetention); this.retentionRescheduleTimer.Reset(); } }
internal async Task DeletePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction) { /** * cases: * 1. If cleanup going on, then stall deleting the policy. * * need to change cleanup store. * Cleanup store change ---> {"backupPolicyName": "PartitionList"}--> this should have been there at the first place. * * Calling function should call it again and again and timeout if it true is not returned. * * */ try { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DeletePolicyAsync : {0} is getting deleted", backupPolicyName); while (true) { cancellationToken.ThrowIfCancellationRequested(); List <string> partitionList = await this.CleanupStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction); if (partitionList == null || partitionList.Count == 0) { // Finally,remove retentionMetadata from retentionStore for the backupPolicyName. RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction); if (retentionMetadata != null) { await this.RetentionStore.DeleteValueAsync(backupPolicyName, timeout, cancellationToken, transaction); BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Stopping RetentionScheduler for backupPolicy: {0}", backupPolicyName); RetentionScheduler retentionScheduler; if (!this.RetentionSchedulerDict.TryRemove(backupPolicyName, out retentionScheduler)) { // We wanted to delete the value, we did not find it, so moving on. return; } retentionScheduler.Stop(); } return; } await Task.Delay(waitTimeInMilliSecondForDeleteOp); } } catch (Exception ex) { if (ex is OperationCanceledException) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "{0} Policy cannot be deleted because the operation is cancelled because of timeout ", backupPolicyName); } else { BackupRestoreTrace.TraceSource.WriteError(TraceType, "{0} Policy cannot be deleted because of the exception: {1} ", backupPolicyName, ex); } throw ex; } }
internal async Task CleanUpTaskUtil(string partitionId, BackupPolicy backupPolicy) { try { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Running CleanUp task for partitionId : {0} and backupPolicy: {1}", partitionId, backupPolicy.Name); await IntroduceRandomDelay(); RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicy.Name, this.timeout, this.tokenSource.Token); Func <bool> condition = () => { if (retentionMetadata == null || retentionMetadata.LastRetentionStartTime == DateTime.MinValue || retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || !retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId)) { return(true); } return(false); }; bool cleanUpComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync( (partition, token) => { return(CleanUpTask(partition, backupPolicy, token)); }, condition, partitionId, this.tokenSource.Token, minimumWaitTimeForRetryOperationInMs, maxRetryCountForCleanup, maximumWaitTimeForCleanUpRetryOperationInMs ); if (!cleanUpComplete) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} was not completed." + "It could be because retention of the partition was going on from a long time which did not complete 40 seconds.", partitionId, backupPolicy.Name); } } catch (Exception ex) { if (ex is OperationCanceledException) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} is canceled", partitionId, backupPolicy.Name); } else { BackupRestoreTrace.TraceSource.WriteError(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} has thrown an exception : {2}", partitionId, backupPolicy.Name, ex); } } }
internal void RearmTimer(bool postFailedScheduled, RetentionMetadata retentionMetadata) { if (postFailedScheduled && !this.retentionRescheduleTimer.IsReschduleCountExhausted()) { this.retentionRescheduleTimer.Reschedule(); } else if (postFailedScheduled && this.retentionRescheduleTimer.IsReschduleCountExhausted()) { this.retentionRescheduleTimer.Reset(); this.retentionTimer.armTimer(true, retentionMetadata); } else { this.retentionTimer.armTimer(false, retentionMetadata); } }
private async Task DisposeRetentionScheduler(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction) { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DisposeRetentionScheduler for backupPolicy {0}", backupPolicyName); cancellationToken.ThrowIfCancellationRequested(); RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction); await this.RetentionStore.DeleteValueAsync(backupPolicyName, timeout, cancellationToken, transaction); // It means timer callback is not running. RetentionScheduler retentionScheduler; if (!this.RetentionSchedulerDict.TryRemove(backupPolicyName, out retentionScheduler)) { // This means key does not exists and we are just trying to remove the key. BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Key: {0} not found in retentionSchedulerDict.", backupPolicyName); return; } retentionScheduler.Stop(); }
internal void armTimer(bool postFailedReschedule, RetentionMetadata retentionMetadata, bool updatePolicyToAddRetention = false) { TimeSpan dueTime = TimeSpan.Zero; Random rand = new Random(); dueTime = dueTime + TimeSpan.FromMilliseconds((double)rand.Next(jitterLow, jitterHigh)); if (updatePolicyToAddRetention) { this.timer.ArmTimer(dueTime.TotalMilliseconds); return; } TimeSpan timeSinceLastRetention = postFailedReschedule == true ? TimeSpan.Zero : GetTimeSinceLastRetentionCompletion(DateTime.Now, retentionMetadata); TimeSpan dueTimeForNextRetention = GetDueTimeForNextRetentionSchedule(retentionMetadata); if (dueTimeForNextRetention >= timeSinceLastRetention) { dueTime = dueTime + (dueTimeForNextRetention - timeSinceLastRetention); } BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Timer armed to run after {0} milliseconds ", dueTime.TotalMilliseconds); this.timer.ArmTimer(dueTime.TotalMilliseconds); }
private async Task DeleteFilesForPartition(string partitionId, DateTime endDateFilter, int minimumNumberOfBackups, BackupStorage storage, string backupPolicyName, bool isCleanupTask = false) { // Design: /** * Need out Two API's from Azure share and file share * 1. List of all backups before a certain date for the partition * 2. Start deleting the files one by one.(one file at a time with some random wait time.) * **/ BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DeleteFilesForPartition for partitionId : {0} with endDateFilter {1}", partitionId, endDateFilter); string applicationName = null; string serviceName = null; string partitionID = null; string fabricUri = await UtilityHelper.GetFabricUriFromPartitionId(partitionId, this.timeout, this.tokenSource.Token); var FabricBackupResourceType = UtilityHelper.GetApplicationAndServicePartitionName(fabricUri, out applicationName, out serviceName, out partitionID); Debug.Assert(partitionId == partitionID); var storeManager = RecoveryPointManagerFactory.GetRecoveryPointManager(storage); this.tokenSource.Token.ThrowIfCancellationRequested(); BRSContinuationToken bRSContinuationToken = new BRSContinuationToken(); List <string> allBackupFiles = storeManager.EnumerateRecoveryPointMetadataFiles(storage.GetRelativeStorePath(applicationName, serviceName, partitionId), DateTime.MinValue, DateTime.MaxValue, false, bRSContinuationToken, 0); List <RestorePoint> restorePoints = new List <RestorePoint>(); if ((allBackupFiles != null) && (allBackupFiles.Count > 0)) { restorePoints = await storeManager.GetRecoveryPointDetailsAsync(allBackupFiles, this.tokenSource.Token); } this.tokenSource.Token.ThrowIfCancellationRequested(); int backupCount = allBackupFiles.Count; if (backupCount <= minimumNumberOfBackups) { return; } restorePoints = restorePoints.OrderBy(restorePoint => restorePoint.CreationTimeUtc).ToList(); restorePoints = restorePoints.Where(restorePoint => restorePoint.CreationTimeUtc < endDateFilter).ToList(); restorePoints.Reverse(); bool lastFullBackupFound = false; RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token); if (isCleanupTask) { lastFullBackupFound = true; } if (!isCleanupTask && retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId) && retentionMetadata.OnGoingRetentionFile[partitionId] != null) { storeManager.DeleteBackupFiles(retentionMetadata.OnGoingRetentionFile[partitionId]); } foreach (var restorePoint in restorePoints) { this.tokenSource.Token.ThrowIfCancellationRequested(); BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Trying to Delete Restore point with backupLocation: {0}", restorePoint.BackupLocation); if (!isCleanupTask) { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0} with partitionId : {1} and BackupLocation {2}" , backupPolicyName, partitionId, restorePoint.BackupLocation); retentionMetadata.UpdateOnGoingRetentionFile(partitionId, restorePoint.BackupLocation); await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null); } if (!lastFullBackupFound) { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Skipping RestorePoint with location : {0} as full backup is not found", restorePoint.BackupLocation); if (restorePoint.BackupType == BackupOptionType.Full) { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Full Backup found so, after restore points after :{0} will be deleted. ", restorePoint.BackupLocation); lastFullBackupFound = true; } } else { if (backupCount <= minimumNumberOfBackups) { break; } if (storeManager.DeleteBackupFiles(restorePoint.BackupLocation)) { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Deleted Restore point with backupLocation: {0}", restorePoint.BackupLocation); backupCount--; } else { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " Not able to delete Restore point with backupLocation: {0}", restorePoint.BackupLocation); } } await IntroduceRandomDelay(); } // Deletion for a partition is completed. if (!isCleanupTask) { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0}", backupPolicyName); retentionMetadata.UpdateOnGoingRetentionFile(partitionId); await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null); } }
/// <summary> /// Gets called when a policy is updated /// </summary> internal async Task UpdatePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction) { /** * Cases: * 1. UpdateRetention ----> Re-arm the timer. * 0. Add Retention. * a. Remove retention --> dispose the timer. * b. update the retention. * 2. UpdateStorage ----> * 3. Update Backup Schedule. --> no need to do anything/ * */ try { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In UpdatePolicyAsync of Retention Manager for Backup policy: {0}", backupPolicyName); var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService); RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction); BackupPolicy backupPolicy = await backupPolicyStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction); if (retentionMetadata == null) { if (backupPolicy.RetentionPolicy != null) { retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy); await this.RetentionStore.AddAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction); RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback); // Here we have updated the policy to introduce retention. retentionScheduler.ArmTimer(retentionMetadata, true); if (!RetentionSchedulerDict.TryAdd(backupPolicyName, retentionScheduler)) { // If we are not able to add retention in RetentionSchedulerDict then, stop timer in retention scheduler retentionScheduler.Stop(); BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "UpdatePolicyAsync, Not able to add retentionScheduler to dict for policy : {0} ", backupPolicyName); throw new InvalidOperationException(string.Format("{0}: Key already exists.", backupPolicyName)); } } } else if (backupPolicy.RetentionPolicy == null) { // need to remove the retentionScheduler from RetentionStore. Func <bool> condition = () => { if (retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || retentionMetadata.LastRetentionStartTime == DateTime.MinValue) { return(true); } return(false); }; bool disposeComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync( (policyName, token) => { return(DisposeRetentionScheduler(policyName, timeout, token, transaction)); }, condition, backupPolicyName, cancellationToken, minimumWaitTimeForRetryOperationInMs, maxRetryCountForDisposingRetention, minimumWaitTimeForRetryOperationInMs); if (!disposeComplete) { BackupRestoreTrace.TraceSource.WriteError(TraceType, "Dispose retention did not complete for {0}", backupPolicyName); throw new FabricBackupRestoreLocalRetryException(); } } else { if (retentionMetadata.UpdateRetentionPolicyMetadata(backupPolicy.RetentionPolicy)) { // Update the retention Scheduler store and arm the timer. RetentionScheduler retentionScheduler; if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler)) { BackupRestoreTrace.TraceSource.WriteError(TraceType, "UpdateBackupPolicy: Not able to get retention scheduler for backupPolicy {0}", backupPolicyName); throw new KeyNotFoundException(); } retentionScheduler.RearmTimer(false, retentionMetadata); await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction); } } } catch (Exception ex) { if (ex is OperationCanceledException) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " DisposeRetentionScheduler ended with Timeout as the operation was cencelled: {0}", backupPolicyName); BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " It could be because the retention was in progress and user tried to remove the retention policy during UpdatePolicy: {0}", backupPolicyName); } else { // Check for the exceptions of DisposeRetentionScheduler Timeout Retries. BackupRestoreTrace.TraceSource.WriteError(TraceType, " UpdateBackupPolicy resulted in the exception : {0}", backupPolicyName); } throw; } }
/// <summary> /// Gets invoked when the RetentionScheduler's timer's time is elapsed /// </summary> internal void timerCallback(string backupPolicyName) { RetentionScheduler retentionScheduler; if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler)) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "RetentionScheduler object is not found in retentionScheduler dict for policy: {0}", backupPolicyName); return; } bool toRearm = true; RetentionMetadata retentionMetadata = this.RetentionStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult(); try { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " TimerCallback running for backupPolicy : {0}", backupPolicyName); var backupPolicyStore = BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService).GetAwaiter().GetResult(); BackupPolicy backupPolicy = backupPolicyStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult(); if (backupPolicy == null) { /** * This is used to garbage collect the retention scheduler. It could happen while deleting\updating policy retention scheduler object was not * stopped properly. So, timer will call its callback again just to find that the backup policy does not exists. * In this case, we should stop the retentionScheduler. * **/ RetentionScheduler retentionSchedulerToDelete; if (RetentionSchedulerDict.TryRemove(backupPolicyName, out retentionSchedulerToDelete)) { retentionSchedulerToDelete.Stop(); } else { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "TimerCallback TryRemove retentionSchedulerDict failed of policy {0}. It could happen" + " when residual callback already scheduled on threadpool thread is getting invoked.", backupPolicyName); } toRearm = false; return; } HashSet <string> partitionsEnabledByPolicy = FindParititionsEnabledByPolicy(backupPolicy).GetAwaiter().GetResult(); retentionMetadata.UpdateLastRetentionStartTime(); this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null).GetAwaiter().GetResult(); var derivedRetentionPolicy = (BasicRetentionPolicy)retentionMetadata.CurrentRetentionPolicy; TimeSpan retentionDuration = derivedRetentionPolicy.RetentionDuration; int minimumNumberOfBackups = derivedRetentionPolicy.MinimumNumberOfBackups; foreach (var partitionId in partitionsEnabledByPolicy) { BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Deleting backups for partitionId : {0}", partitionId); List <string> partitionList = this.CleanupStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult(); if (!(partitionList == null || partitionList.Count == 0) && partitionList.Contains(partitionId)) { // Cleanup is going on for this partition. continue; } DateTime endDateFilter = DateTime.UtcNow - retentionDuration; BackupStorage storage = backupPolicy.Storage; DeleteFilesForPartition(partitionId, endDateFilter, minimumNumberOfBackups, storage, backupPolicyName).GetAwaiter().GetResult(); this.tokenSource.Token.ThrowIfCancellationRequested(); } // --> Introduce some wait time here so that CompletionTime is never equal to StartTime. retentionMetadata.UpdateLastRetentionCompletionTime(); this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata).GetAwaiter().GetResult(); retentionScheduler.ArmTimer(retentionMetadata); toRearm = false; } catch (Exception ex) { if (ex is OperationCanceledException) { // Since, timercallback was cancelled, therefore, there is no need to rearm the timer. BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " TimerCallback for backupPolicy : {0} was cancelled ", backupPolicyName); toRearm = false; } else { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " TimerCallback for backupPolicy : {0} has thrown an exception : {1}", backupPolicyName, ex); } } finally { if (toRearm) { retentionScheduler.RearmTimer(true, retentionMetadata); } } }
internal TimeSpan GetTimeSinceLastRetentionCompletion(DateTime currentDateTime, RetentionMetadata retentionMetadata) { TimeSpan timeSinceLastRetentionCompletion = TimeSpan.Zero; // Returning zero means we will wait for retention time. if (retentionMetadata.LastRetentionCompletionTime != DateTime.MinValue) { // It means retention completed atleast once. => We need to wait on the basis of time. timeSinceLastRetentionCompletion = currentDateTime - retentionMetadata.LastRetentionCompletionTime; } else if (retentionMetadata.LastRetentionStartTime != DateTime.MinValue) { // It means retention never completed once, but we have waited => We need retention right now. timeSinceLastRetentionCompletion = currentDateTime - retentionMetadata.LastRetentionCompletionTime; } return(timeSinceLastRetentionCompletion); }